同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { activeAnalysisTurn, analysisIsRunning, beginAnalysisSelection, defaultAnalysisSession, rememberAnalysisSession, rememberPendingAnalysisSession } from '../web/src/analysisConversationState.ts';
|
||||
|
||||
const memory = () => { const data = new Map(); return { getItem: k => data.get(k) || null, setItem: (k, v) => data.set(k, v), removeItem: k => data.delete(k) }; };
|
||||
test('last viewed session wins over the most recently updated conversation', () => {
|
||||
const storage = memory(); rememberAnalysisSession('alice', 'older-viewed', storage);
|
||||
assert.equal(defaultAnalysisSession('alice', [{ id: 'newer' }, { id: 'older-viewed' }], storage), 'older-viewed');
|
||||
});
|
||||
test('another user or deleted session never restores inaccessible history', () => {
|
||||
const storage = memory(); rememberAnalysisSession('alice', 'a', storage);
|
||||
assert.equal(defaultAnalysisSession('bob', [{ id: 'b' }], storage), 'b');
|
||||
assert.equal(defaultAnalysisSession('alice', [{ id: 'replacement' }], storage), 'replacement');
|
||||
assert.equal(defaultAnalysisSession('alice', [], storage), '');
|
||||
});
|
||||
test('disabled storage falls back to accessible server history', () => {
|
||||
const storage = { getItem() { throw Error(); }, setItem() { throw Error(); }, removeItem() { throw Error(); } };
|
||||
rememberAnalysisSession('a', 'x', storage);
|
||||
assert.equal(defaultAnalysisSession('a', [{ id: 'server' }], storage), 'server');
|
||||
});
|
||||
test('restores latest active turn without restarting completed analysis', () => {
|
||||
const rows = [{ id: 'old', status: 'COMPLETED' }, { id: 'queued', status: 'QUEUED' }, { id: 'new', status: 'RUNNING' }];
|
||||
assert.equal(activeAnalysisTurn(rows).id, 'new');
|
||||
assert.equal(activeAnalysisTurn([{ id: 'done', status: 'COMPLETED' }]), undefined);
|
||||
assert.equal(analysisIsRunning('CANCELLED'), false);
|
||||
});
|
||||
test('a late background response cannot replace a newer viewed session', () => {
|
||||
const storage = memory();
|
||||
const oldSelection = beginAnalysisSelection('alice');
|
||||
const newSelection = beginAnalysisSelection('alice');
|
||||
rememberPendingAnalysisSession('alice', 'new', newSelection, storage);
|
||||
rememberPendingAnalysisSession('alice', 'old', oldSelection, storage);
|
||||
assert.equal(defaultAnalysisSession('alice', [{ id: 'old' }, { id: 'new' }], storage), 'new');
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { settledAnalysisTurn } from '../web/src/analysisConversationState.ts';
|
||||
|
||||
test('a stream closing mid-sentence cannot replace the completed saved answer', () => {
|
||||
const full = '小结:新能源汽车磁钢下降。完整的后续分析与关注方向。';
|
||||
const partial = '小结:新能源汽车磁钢下';
|
||||
const turn = settledAnalysisTurn('t', '问题', { status: 'COMPLETED', answer: full, steps: [1, 2] }, { answer: partial, steps: [1] }, '');
|
||||
assert.equal(turn.answer, full);
|
||||
assert.equal(turn.result.answer, full);
|
||||
assert.deepEqual(turn.steps, [1, 2]);
|
||||
});
|
||||
|
||||
test('a nested final result and an intentionally empty final answer remain authoritative', () => {
|
||||
for (const answer of ['完整最终回答', '']) {
|
||||
const turn = settledAnalysisTurn('t', '问题', { status: 'COMPLETED', result: { answer } }, { answer: '过时的中间内容' }, 'COMPLETED');
|
||||
assert.equal(turn.result.answer, answer);
|
||||
}
|
||||
});
|
||||
|
||||
test('a terminal event supplies the answer if the final detail read is unavailable', () => {
|
||||
const turn = settledAnalysisTurn('t', '问题', {}, { answer: '完成事件的完整回答' }, 'COMPLETED');
|
||||
assert.equal(turn.status, 'COMPLETED');
|
||||
assert.equal(turn.result.answer, '完成事件的完整回答');
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { consumeEventStream } from '../web/src/eventStream.ts';
|
||||
const consume=async chunks=>{const events=[];const encoder=new TextEncoder();await consumeEventStream(new ReadableStream({start(c){for(const chunk of chunks)c.enqueue(encoder.encode(chunk));c.close();}}),event=>events.push(event));return events;};
|
||||
test('a comment before an event cannot hide the recorded result',async()=>{
|
||||
const events=await consume([': heartbeat\nevent: progress\nid: 12\ndata: {"message":"来源已返回"}\n\n']);
|
||||
assert.equal(events.length,1);assert.equal(events[0].id,'12');assert.equal(events[0].data.message,'来源已返回');
|
||||
});
|
||||
test('an interrupted final frame cannot falsely complete a running turn',async()=>{
|
||||
const events=await consume(['event: progress\nid: 1\ndata: {"status":"RUNNING"}\n\n','event: completed\nid: 2\ndata: {"status":"COMPL']);
|
||||
assert.equal(events.length,1);assert.equal(events[0].event,'progress');
|
||||
});
|
||||
test('CRLF and chunk boundaries preserve events and multi-line data',async()=>{
|
||||
const events=await consume(['id: 4\r\nevent: answer\r\ndat','a: {"answer":\r\ndata: "完整回答"}\r\n\r','\n']);
|
||||
assert.equal(events[0].data.answer,'完整回答');
|
||||
});
|
||||
test('a malformed structured terminal event is rejected before it changes task status',async()=>{
|
||||
await assert.rejects(consume(['event: completed\ndata: {broken}\n\n']),/事件格式异常/);
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { publicErrorMessage, readableMetricFormula } from '../web/src/publicDataDisplay.ts';
|
||||
import * as conversation from '../web/src/analysisConversationState.ts';
|
||||
import { sanitizeAnalysisDisplayText } from '../web/src/analysisDisplayText.ts';
|
||||
|
||||
test('gateway HTML is a service failure, never rendered as its raw payload', () => {
|
||||
const error = {message: publicErrorMessage('<html><h1>502 Bad Gateway</h1></html>', 502)};
|
||||
assert.match(error.message, /服务.*暂不可用/);
|
||||
assert.doesNotMatch(error.message, /<html>/);
|
||||
});
|
||||
test('replay duplicates and old snapshots cannot move the event cursor backwards', () => {
|
||||
assert.equal(conversation.acceptAnalysisEvent('12', '12'), false);
|
||||
assert.equal(conversation.acceptAnalysisEvent('11', '12'), false);
|
||||
assert.equal(conversation.acceptAnalysisEvent('13', '12'), true);
|
||||
assert.equal(conversation.acceptAnalysisEvent('9007199254740994', '9007199254740993'), true);
|
||||
});
|
||||
|
||||
test('formula wording translates the service policy without replacing its business calculation', () => {
|
||||
assert.equal(readableMetricFormula('blackRate * brightRate;运算空值=PROPAGATE;零分母=ONE;空结果=NULL'), '黑片合格率 × 亮片合格率;运算中遇到空值:运算结果保持空值;分母为0:取1(率值100%);计算结果为空:保留空值');
|
||||
assert.equal(readableMetricFormula('blackGood / blackTotal * 0.99'), '黑片合格重量 ÷ 黑片总重量 × 0.99');
|
||||
assert.equal(readableMetricFormula(undefined), '当前指标服务未提供公式说明');
|
||||
});
|
||||
|
||||
test('an old contradictory report retains its text and gains an explicit evidence-scope warning', () => {
|
||||
const answer='具体缺陷线索(来源细分,未与正式大类完全对账)。责废缺角增量是黑片报废增长主因。';
|
||||
assert.equal(conversation.hasUnreconciledEvidence({answer}), true);
|
||||
assert.equal(conversation.hasUnreconciledEvidence({answer:'未发现对账差异,已完成核对。'}), false);
|
||||
assert.equal(conversation.hasUnreconciledEvidence({plan:{timeline:[{text:'细分与正式大类未对账:仅作为变化线索'}]}}),true);
|
||||
});
|
||||
|
||||
test('provider whitespace does not duplicate the saved final answer in its event stream', () => {
|
||||
const answer='**2026年9月正式合格率**\n\n综合76.5%,黑片85%,亮片90%。';
|
||||
assert.equal(conversation.duplicatesFinalAnalysisAnswer('\n\n'+answer,answer),true);
|
||||
assert.equal(conversation.duplicatesFinalAnalysisAnswer(answer.slice(0,12),answer),true);
|
||||
assert.equal(conversation.duplicatesFinalAnalysisAnswer('正在核查合成指标。',answer),false);
|
||||
assert.equal(conversation.duplicatesFinalAnalysisAnswer(' ',answer),false);
|
||||
assert.equal(conversation.duplicatesFinalAnalysisAnswer('较早的独立进展',''),false);
|
||||
});
|
||||
|
||||
test('verified VCM coverage supersedes the same scope warning but preserves other unresolved scopes', () => {
|
||||
const key='0123456789abcdef', other='fedcba9876543210';
|
||||
const event=(checks)=>({kind:'tool_result',status:'completed',evidenceChecks:checks});
|
||||
assert.equal(conversation.hasUnreconciledEvidence({answer:'VCM细分未对账',timeline:[event({[key]:'unresolved'}),event({[key]:'expected'})]}),false);
|
||||
assert.equal(conversation.hasUnreconciledEvidence({timeline:[event({[other]:'unresolved'}),event({[key]:'expected'})]}),true);
|
||||
assert.equal(conversation.hasUnreconciledEvidence({timeline:[{kind:'assistant',evidenceChecks:{[other]:'checked'},text:'未对账'}]}),true);
|
||||
});
|
||||
|
||||
test('partial pre-tool notes stay in chronology instead of being repeated as final answers', () => {
|
||||
const note={kind:'assistant',text:'正在核查原表。'}, tool={kind:'tool_result',status:'completed'};
|
||||
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:note.text,timeline:[note,tool]}),true);
|
||||
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:'已完成部分比较。',timeline:[tool,{kind:'assistant',text:'已完成部分比较。'}]}),false);
|
||||
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'COMPLETED',answer:note.text,timeline:[note,tool]}),false);
|
||||
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:note.text}),false);
|
||||
});
|
||||
|
||||
test('each tool start/result pair renders one folded row without merging different calls', () => {
|
||||
const input={mode:'compare',period:{month:9}};
|
||||
const events=[{kind:'tool_start',sequence:1,toolCallId:'a',label:'查询',input},
|
||||
{kind:'tool_start',sequence:2,toolCallId:'b',label:'查询'},
|
||||
{kind:'assistant',sequence:3,text:'继续核查'},
|
||||
{kind:'tool_result',sequence:4,toolCallId:'a',status:'completed',text:'已返回'},
|
||||
{kind:'tool_result',sequence:5,toolCallId:'b',status:'failed',text:'需重试'}];
|
||||
const compact=conversation.compactAnalysisTimeline(events);
|
||||
assert.equal(compact.length,3);
|
||||
assert.deepEqual(compact.map(row=>row.kind),['tool','tool','assistant']);
|
||||
assert.equal(compact[0].status,'completed');assert.deepEqual(compact[0].input,input);
|
||||
assert.equal(compact[1].status,'failed');
|
||||
assert.equal(events[0].kind,'tool_start');
|
||||
const replay=conversation.compactAnalysisTimeline([...events,events[0]]);
|
||||
assert.equal(replay[0].status,'completed');assert.equal(replay.length,3);
|
||||
});
|
||||
|
||||
test('stream updates respect an upward scroll and resume following only at the bottom', () => {
|
||||
assert.equal(conversation.analysisScrollFollows(40,true,'up'),false);
|
||||
assert.equal(conversation.analysisScrollFollows(150,false),false);
|
||||
assert.equal(conversation.analysisScrollFollows(130,false,'down'),false);
|
||||
assert.equal(conversation.analysisScrollFollows(10,false,'down'),true);
|
||||
assert.equal(conversation.analysisScrollFollows(120,true),true);
|
||||
assert.equal(conversation.analysisScrollFollows(10,false),false);
|
||||
});
|
||||
|
||||
test('partial completion explains tool failures separately from model limits', () => {
|
||||
assert.match(conversation.partialAnalysisNotice('mcp_error'),/数据查询未成功/);
|
||||
assert.match(conversation.partialAnalysisNotice('length'),/长度上限/);
|
||||
assert.match(conversation.partialAnalysisNotice('timeout'),/处理时限/);
|
||||
assert.match(conversation.partialAnalysisNotice('evidence_incomplete'),/被标记为部分完成/);
|
||||
assert.doesNotMatch(conversation.partialAnalysisNotice('evidence_incomplete'),/数据尚未齐全/);
|
||||
});
|
||||
|
||||
|
||||
test('provider tool-call markup is hidden while business text stays readable', () => {
|
||||
assert.equal(sanitizeAnalysisDisplayText('<tool_call><function=read><parameter_path>/RULES.md</parameter></function></tool_call>'), '');
|
||||
assert.equal(sanitizeAnalysisDisplayText('已核对。<tool_call><function=read>'), '已核对。');
|
||||
assert.equal(sanitizeAnalysisDisplayText('产品VCM739+Z重量10kg'), '产品VCM739+Z重量10kg');
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {withRequestDeadline,RequestTimeoutError} from '../web/src/requestDeadline.ts';
|
||||
test('a silent connection expires and aborts its network request',async()=>{
|
||||
let signal;const start=Date.now();
|
||||
await assert.rejects(withRequestDeadline(s=>{signal=s;return new Promise(()=>{});},20),RequestTimeoutError);
|
||||
assert.equal(signal.aborted,true);assert.ok(Date.now()-start<500);
|
||||
});
|
||||
test('deadline includes reading a stalled response body',async()=>{
|
||||
const run=async signal=>{await Promise.resolve();return new Promise(()=>{});};
|
||||
await assert.rejects(withRequestDeadline(run,20),RequestTimeoutError);
|
||||
});
|
||||
test('a completed response cancels its deadline',async()=>{
|
||||
let signal;assert.equal(await withRequestDeadline(s=>{signal=s;return Promise.resolve('ok');},20),'ok');
|
||||
await new Promise(r=>setTimeout(r,30));assert.equal(signal.aborted,false);
|
||||
});
|
||||
test('HTTP and authentication errors retain their original identity',async()=>{
|
||||
const original=Object.assign(new Error('unauthorized'),{status:401});
|
||||
await assert.rejects(withRequestDeadline(async()=>{throw original;},100),e=>e===original);
|
||||
});
|
||||
@@ -0,0 +1,350 @@
|
||||
"""Evidence-bound composed answers; uses only mocked OpenAI-compatible responses."""
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import tempfile
|
||||
from urllib.error import HTTPError
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_evidence_finalizer as finalizer
|
||||
|
||||
|
||||
def evidence(ref='Q00000001', rate=.95):
|
||||
return [{'evidenceRef': ref, 'mode': 'compare', 'result': {
|
||||
'kind': 'quality_comparison', 'filterBinding': {},
|
||||
'current': {'period': {'year': 2026, 'month': 9}, 'compositeRate': rate,
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': 9,
|
||||
'blackRows': 2, 'brightRows': 2}]}},
|
||||
'comparison': {'period': {'year': 2026, 'month': 8}, 'compositeRate': .96,
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': 8,
|
||||
'blackRows': 2, 'brightRows': 2}]}},
|
||||
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
|
||||
}}]
|
||||
|
||||
|
||||
def response(parsed):
|
||||
payload = {'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]}
|
||||
return io.BytesIO(json.dumps(payload).encode())
|
||||
|
||||
|
||||
class AgentAnswerTest(unittest.TestCase):
|
||||
def test_product_drilldown_fallback_keeps_each_queried_specification(self):
|
||||
from unittest.mock import patch
|
||||
facts = [{'id':'F1','kind':'overall','text':'总体合格率上升。','evidenceRef':'Q00000001'}]
|
||||
docs = evidence()
|
||||
for index, product in enumerate(('规格甲','规格乙','规格丙'),2):
|
||||
ref = f'Q{index:08X}'
|
||||
facts.append({'id':f'F{index}','kind':'cross','text':product+'来源缺角变化。','evidenceRef':ref})
|
||||
docs.append({'evidenceRef':ref,'result':{'filters':{'product':product}}})
|
||||
report = {'facts':facts,'sections':['不应输出冗长全表'],'headline':'总体','policy':'关联不作为生产根因。','missingEvidence':[]}
|
||||
invalid = {'answerBlocks':[{'type':'bullet','text':'{{F1}} 虚构改善50%'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]}
|
||||
with patch('nao_agent_report.build_agent_report',return_value=report):
|
||||
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question='fixture',evidence=docs,cancel_event=threading.Event(),
|
||||
context={'_agentTask':{'version':1,'kind':'analysis','defectScope':'products','defectProductCount':3}},
|
||||
opener=lambda *args,**kwargs:response(invalid))
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertTrue(result['compositionRepairAttempted'])
|
||||
for product in ('规格甲','规格乙','规格丙'):
|
||||
self.assertIn(product,result['answer'])
|
||||
self.assertIn('建议核查',result['answer'])
|
||||
self.assertNotIn('50%',result['answer'])
|
||||
self.assertNotIn('不应输出冗长全表',result['answer'])
|
||||
|
||||
def test_invalid_composition_gets_one_model_repair_with_same_global_budget(self):
|
||||
calls = []
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data)
|
||||
calls.append(body)
|
||||
if len(calls) == 1:
|
||||
return response({'reportTitle':'合格率变化分析','answerBlocks':[{'type':'bullet','text':'本期改善50% {{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})
|
||||
return response({'answerBlocks':[{'type':'bullet','text':'{{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})
|
||||
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question='比较质量',evidence=evidence(),cancel_event=threading.Event(),opener=provider)
|
||||
self.assertEqual(len(calls),2)
|
||||
self.assertNotIn('50%',result['answer'])
|
||||
self.assertFalse(result.get('selectionFallback',False))
|
||||
self.assertEqual(result['reportTitle'],'合格率变化分析')
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
||||
self.assertTrue(all(body['max_completion_tokens']==MAX_OUTPUT_TOKENS for body in calls))
|
||||
self.assertEqual(len(calls[1]['messages']),4)
|
||||
|
||||
def test_report_uses_model_selected_priorities_without_dumping_all_tables(self):
|
||||
result = self.synthesize({'reportFactIds': ['F1'], 'reportTitle': '合格率变化分析',
|
||||
'evidenceSufficient': True, 'missingEvidence': []},
|
||||
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
|
||||
'question': '综合以上生成报告'}})
|
||||
self.assertIn('## 核心结论', result['answer'])
|
||||
self.assertIn('下降1.00个百分点', result['answer'])
|
||||
self.assertIn('## 下一步核查', result['answer'])
|
||||
self.assertNotIn('| ---', result['answer'])
|
||||
self.assertLess(len(result['answer']), 1000)
|
||||
self.assertFalse(result.get('selectionFallback', False))
|
||||
|
||||
def test_report_rejects_unknown_focus_id_and_still_has_bounded_fallback(self):
|
||||
result = self.synthesize({'reportFactIds': ['F999'],
|
||||
'evidenceSufficient': True, 'missingEvidence': []},
|
||||
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
|
||||
'question': '综合以上生成报告'}})
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertNotIn('F999', result['answer'])
|
||||
self.assertNotIn('| ---', result['answer'])
|
||||
|
||||
def synthesize(self, parsed, *, docs=None, model='vendor/compatible-chat', context=None, capture=None):
|
||||
def opener(request, timeout):
|
||||
if capture is not None:
|
||||
capture['body'] = json.loads(request.data)
|
||||
capture['timeout'] = timeout
|
||||
return response(parsed)
|
||||
|
||||
return finalizer.synthesize(
|
||||
base_url='https://offline.invalid/v1', api_key='synthetic-only', model=model,
|
||||
question='请概括质量变化', evidence=evidence() if docs is None else docs,
|
||||
cancel_event=threading.Event(), context=context, opener=opener)
|
||||
|
||||
def test_model_composes_prose_around_exact_server_fact_and_reference(self):
|
||||
captured = {}
|
||||
result = self.synthesize({
|
||||
'answerBlocks': [
|
||||
{'type': 'heading', 'text': '变化概览'},
|
||||
{'type': 'paragraph', 'text': '建议结合现场记录开展复核;当前结论只限于本次查询指标:{{F1}}'},
|
||||
],
|
||||
'evidenceSufficient': True, 'missingEvidence': [],
|
||||
}, context={'_agentTask': {'version': 1, 'kind': 'analysis', 'responseStyle': 'brief',
|
||||
'dimensions': ['group'], 'question': '简要回答'}}, capture=captured)
|
||||
|
||||
self.assertIn('### 变化概览', result['answer'])
|
||||
self.assertIn('建议结合现场记录开展复核', result['answer'])
|
||||
self.assertIn('下降1.00个百分点', result['answer'])
|
||||
self.assertIn('[Q00000001]', result['answer'])
|
||||
self.assertEqual(result['selectedFactIds'], ['F1'])
|
||||
self.assertFalse(result.get('selectionFallback', False))
|
||||
self.assertEqual(captured['body']['model'], 'vendor/compatible-chat')
|
||||
self.assertEqual(captured['body']['thinking'], {'type':'disabled'})
|
||||
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
||||
self.assertEqual(captured['body']['max_completion_tokens'], MAX_OUTPUT_TOKENS)
|
||||
selector = json.loads(captured['body']['messages'][1]['content'])
|
||||
self.assertEqual(selector['agentTask']['responseStyle'], 'brief')
|
||||
self.assertTrue(selector['facts'])
|
||||
|
||||
def test_unsupported_digits_force_verified_report_fallback(self):
|
||||
result = self.synthesize({
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '本期合格率达到99%,依据{{F1}}'}],
|
||||
'evidenceSufficient': True, 'missingEvidence': [],
|
||||
})
|
||||
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertNotIn('99%', result['answer'])
|
||||
self.assertIn('下降1.00个百分点', result['answer'])
|
||||
|
||||
def test_unknown_fact_reference_forces_fallback(self):
|
||||
result = self.synthesize({
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '结论{{F99}}'}],
|
||||
'evidenceSufficient': True, 'missingEvidence': [],
|
||||
})
|
||||
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertNotIn('{{F99}}', result['answer'])
|
||||
self.assertIn('下降1.00个百分点', result['answer'])
|
||||
|
||||
def test_unsupported_direction_and_cause_prose_forces_fallback(self):
|
||||
result=self.synthesize({
|
||||
'answerBlocks':[{'type':'paragraph','text':'质量已持续改善,且确认由设备磨损导致。{{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertNotIn('设备磨损',result['answer'])
|
||||
|
||||
def test_cause_hypothesis_passes_only_when_explicitly_unconfirmed(self):
|
||||
result=self.synthesize({
|
||||
'answerBlocks':[{'type':'paragraph','text':'待验证:设备磨损是否可能导致报废增加,建议检查。{{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})
|
||||
self.assertFalse(result.get('selectionFallback',False))
|
||||
self.assertIn('待验证:设备磨损是否可能导致报废增加',result['answer'])
|
||||
|
||||
absolute=self.synthesize({
|
||||
'answerBlocks':[{'type':'paragraph','text':'所有缺陷都已完全排除。{{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})
|
||||
self.assertTrue(absolute['selectionFallback'])
|
||||
|
||||
def test_thinking_capability_is_retried_without_model_name_branch(self):
|
||||
bodies=[]
|
||||
parsed={'answerBlocks':[{'type':'paragraph','text':'总体变化:{{F1}}'}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]}
|
||||
def opener(request,timeout):
|
||||
bodies.append(json.loads(request.data))
|
||||
if len(bodies)==1:
|
||||
raise HTTPError(request.full_url,400,'thinking unsupported',None,io.BytesIO(b''))
|
||||
return response(parsed)
|
||||
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic-only',
|
||||
model='some/openai-compatible-model',question='请概括质量变化',evidence=evidence(),
|
||||
cancel_event=threading.Event(),opener=opener)
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertEqual(len(bodies),2)
|
||||
self.assertEqual(bodies[0]['thinking'],{'type':'disabled'})
|
||||
self.assertNotIn('thinking',bodies[1])
|
||||
|
||||
def test_non_mimo_model_is_eligible_and_incomplete_server_evidence_stays_partial(self):
|
||||
calls = [{'status': 'completed'}]
|
||||
self.assertTrue(finalizer.eligible('provider/model-x', 'PARTIAL', 'timeout', calls, False, True))
|
||||
self.assertFalse(finalizer.eligible(' ', 'PARTIAL', 'timeout', calls, False, True))
|
||||
|
||||
from test_nao_evidence_report import coverage_comparison
|
||||
doc=coverage_comparison()
|
||||
doc['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
|
||||
result = self.synthesize({
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '当前正式变化:{{F1}}'}],
|
||||
'evidenceSufficient': True, 'missingEvidence': [],
|
||||
}, docs=[doc])
|
||||
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertTrue(result['missingEvidence'])
|
||||
self.assertTrue(any('没有可引用的正式综合合格率' in item for item in result['missingEvidence']))
|
||||
self.assertIn('未覆盖内容', result['answer'])
|
||||
|
||||
def test_legacy_selector_response_still_uses_verified_report(self):
|
||||
result = self.synthesize({
|
||||
'otherFactIds': ['F1'], 'groupFactIds': [], 'productFactIds': [], 'defectFactIds': [],
|
||||
'evidenceSufficient': True, 'missingEvidence': [],
|
||||
})
|
||||
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertIn('下降1.00个百分点', result['answer'])
|
||||
self.assertIn('### 重点关注', result['answer'])
|
||||
self.assertFalse(result.get('selectionFallback', False))
|
||||
|
||||
def test_source_verification_task_requires_complete_source_proof(self):
|
||||
question = '请核对来源'
|
||||
period={'year':2026,'month':9}
|
||||
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'source_verify',
|
||||
'verifySource': True, 'fullDetails': False,'period':period,
|
||||
'comparisonPeriod':None,'filters':{'product':'P'},'stages':['bright']}}
|
||||
query = {'mode': 'compare', 'result': {'kind': 'quality_query'}}
|
||||
proof = {'complete': True, 'expectedRows':2, 'verifiedRows': 2, 'missingRows':0,
|
||||
'mismatchRows':0,'allSelectedValuesMatchStored': True}
|
||||
side_context={'releaseId':'r1','year':2026,'month':9,'stage':'bright','category':None,
|
||||
'product':'P','includeUnassigned':False,'extraFilters':{}}
|
||||
source = {'mode': 'investigate', 'result': {'kind':'quality_investigation','releaseId':'r1',
|
||||
'period':period,'filters':{'product':'P'},'stage':'bright','action':'source','current': {
|
||||
'context':side_context,'totalRows': 2,'aggregateComplete':True,'sourceVerification': proof}}}
|
||||
self.assertFalse(finalizer.coverage_sufficient([query], question, context))
|
||||
self.assertTrue(finalizer.coverage_sufficient([source], question, context))
|
||||
source['result']['current']['sourceVerification'] = {**proof, 'complete': False}
|
||||
self.assertFalse(finalizer.coverage_sufficient([source], question, context))
|
||||
|
||||
def test_source_proofs_must_cover_every_requested_stage_side_and_scope(self):
|
||||
question='核验9月与8月原表'
|
||||
current={'year':2026,'month':9};previous={'year':2026,'month':8}
|
||||
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'source_verify',
|
||||
'verifySource':True,'fullDetails':False,'period':current,'comparisonPeriod':previous,
|
||||
'filters':{'product':'P'},'stages':['black','bright']}}
|
||||
query={'mode':'compare','result':{'kind':'quality_query'}}
|
||||
def source(stage, product='P'):
|
||||
proof={'complete':True,'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0,
|
||||
'allSelectedValuesMatchStored':True}
|
||||
def side(month):
|
||||
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
|
||||
'product':product,'includeUnassigned':False,'extraFilters':{}},
|
||||
'totalRows':1,'aggregateComplete':True,'sourceVerification':proof}
|
||||
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
||||
'period':current,'comparisonPeriod':previous,'filters':{'product':product},'stage':stage,
|
||||
'category':None,'action':'source','current':side(9),'comparison':side(8)}}
|
||||
wrong_scope=source('black',product='OTHER')
|
||||
self.assertFalse(finalizer.coverage_sufficient([query,wrong_scope],question,context))
|
||||
black=source('black');bright=source('bright')
|
||||
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
|
||||
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
|
||||
|
||||
def test_full_details_task_requires_contiguous_complete_pages(self):
|
||||
question = '把所有记录列出来'
|
||||
period={'year':2026,'month':9}
|
||||
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'details',
|
||||
'verifySource': False, 'fullDetails': True,'period':period,
|
||||
'comparisonPeriod':None,'filters':{},'stages':['black']}}
|
||||
side={'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black','category':None,
|
||||
'includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
|
||||
'matchingRows':2,'offset':0,'returnedRows':2,'hasMore':False,'differenceRecords':[{},{}]}
|
||||
rows={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1','period':period,
|
||||
'filters':{},'stage':'black','category':None,'action':'rows','current':side}}
|
||||
self.assertTrue(finalizer.coverage_sufficient([rows], question, context))
|
||||
rows['result']['current']['hasMore'] = True
|
||||
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
|
||||
rows['result']['current']['hasMore'] = False
|
||||
rows['result']['current']['recordFilter'] = 'differences'
|
||||
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
|
||||
|
||||
def test_full_details_require_every_requested_stage_and_both_periods(self):
|
||||
question='列出黑片和亮片两期的全部明细'
|
||||
current={'year':2026,'month':9};previous={'year':2026,'month':8}
|
||||
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
|
||||
'verifySource':False,'fullDetails':True,'period':current,'comparisonPeriod':previous,
|
||||
'filters':{'group':'G'},'stages':['black','bright']}}
|
||||
def rows(stage):
|
||||
def side(month):
|
||||
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
|
||||
'group':'G','includeUnassigned':False,'extraFilters':{}},
|
||||
'recordFilter':'all','matchingRows':1,'offset':0,'returnedRows':1,
|
||||
'hasMore':False,'differenceRecords':[{}]}
|
||||
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
||||
'period':current,'comparisonPeriod':previous,'filters':{'group':'G'},'stage':stage,
|
||||
'category':None,'action':'rows','current':side(9),'comparison':side(8)}}
|
||||
black=rows('black');bright=rows('bright')
|
||||
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
|
||||
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
|
||||
bright['result']['comparison']['recordFilter']='differences'
|
||||
self.assertFalse(finalizer.coverage_sufficient([black,bright],question,context))
|
||||
|
||||
def test_unbound_task_period_requires_a_matching_query_root(self):
|
||||
question='列出当前范围的全部明细'
|
||||
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
|
||||
'verifySource':False,'fullDetails':True,'period':None,'comparisonPeriod':None,
|
||||
'filters':{'group':'G'},'stages':['black']}}
|
||||
query={'mode':'compare','result':{'kind':'quality_comparison','releaseId':'r1',
|
||||
'filterBinding':{'group':'G'},'current':{'period':{'year':2026,'month':9}},
|
||||
'comparison':{'period':{'year':2026,'month':8}}}}
|
||||
page={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
||||
'period':{'year':2026,'month':9},'filters':{'group':'G'},'stage':'black','category':None,
|
||||
'action':'rows','current':{'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black',
|
||||
'category':None,'group':'G','includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
|
||||
'matchingRows':1,'offset':0,'returnedRows':1,'hasMore':False,'differenceRecords':[{}]}}}
|
||||
self.assertTrue(finalizer.coverage_sufficient([query,page],question,context))
|
||||
self.assertFalse(finalizer.coverage_sufficient([page],question,context))
|
||||
query['result']['filterBinding']={'group':'OTHER'}
|
||||
self.assertFalse(finalizer.coverage_sufficient([query,page],question,context))
|
||||
|
||||
def test_empty_evidence_is_allowed_only_for_planning_callers(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||||
finalizer.evidence_summaries(Path(folder),'release-a')
|
||||
self.assertEqual(finalizer.evidence_summaries(Path(folder),'release-a',allow_empty=True),[])
|
||||
|
||||
def test_semantic_metrics_and_single_period_dimensions_gate_completion(self):
|
||||
from test_nao_metric_requests import metric_evidence, QUESTION
|
||||
task={'version':1,'question':QUESTION,'kind':'metrics','period':{'year':2026,'month':9},
|
||||
'comparisonPeriod':{'year':2026,'month':8},'filters':{'product':'SYNTH-16X15+T'},
|
||||
'stages':['black','bright'],'fields':['rate','totalWeight','scrapWeight']}
|
||||
context={'releaseId':'release-a','_agentTask':task}
|
||||
docs=metric_evidence()
|
||||
self.assertTrue(finalizer.coverage_sufficient(docs,QUESTION,context))
|
||||
docs[0]['result']['current']['bright'].pop('totalWeight')
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs,QUESTION,context))
|
||||
|
||||
question='本期产品情况'
|
||||
single={'version':1,'question':question,'kind':'analysis','period':{'year':2026,'month':9},
|
||||
'comparisonPeriod':None,'filters':{},'dimensions':['product'],'stages':[]}
|
||||
frame={'releaseId':'r1','_agentTask':single}
|
||||
query={'mode':'query','result':{'kind':'quality_metrics','releaseId':'r1',
|
||||
'period':{'year':2026,'month':9},'filters':{},'view':'overall'}}
|
||||
self.assertFalse(finalizer.coverage_sufficient([query],question,frame))
|
||||
query['result']['view']='product'
|
||||
self.assertTrue(finalizer.coverage_sufficient([query],question,frame))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Regressions for calendar-bound defect TOP and evidence-bound model contracts."""
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_task import normalize_task, bind_task, plan_task, TaskUnavailable
|
||||
from nao_agent_report import build_agent_report
|
||||
from nao_analysis_coverage import coverage_gaps
|
||||
from nao_evidence_finalizer import coverage_sufficient, eligible, _compose_answer_blocks, _agent_task_summary, CompositionError
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
Q = '帮我看下上月黑片缺陷TOP'
|
||||
|
||||
def fixture():
|
||||
task = normalize_task(task_value(dimensions=['defect'], dimensionStages={'defect':['black']},
|
||||
stages=['black'], fields=['scrapWeight'], period={'year':2026,'month':9}, comparisonRequired=False), Q)
|
||||
ctx = bind_task({'releaseId':'release-a'}, task)
|
||||
doc = {'evidenceRef':'Q12345678','mode':'defects','result':{
|
||||
'kind':'quality_fine_defects','releaseId':'release-a','period':task['period'],
|
||||
'filters':{},'dimension':'defect','stage':'black','category':None,'defect':None,
|
||||
'coverage':{'available':True,'hasValidWeights':True}, 'rankingComplete':True,
|
||||
'sourceRows':43172,'rowsReturned':2,'rows':[
|
||||
{'key':'责废缺角重量','label':'责废缺角重量','category':'责废','weight':17513.162},
|
||||
{'key':'料废缺角重量','label':'料废缺角重量','category':'料废','weight':13949.63}]}}
|
||||
return task,ctx,[doc]
|
||||
|
||||
class ContractRegressions(unittest.TestCase):
|
||||
def test_single_period_defect_top_has_verified_answer_without_formal_comparison(self):
|
||||
task,ctx,docs=fixture()
|
||||
self.assertEqual(coverage_gaps(docs,Q,ctx),[])
|
||||
self.assertTrue(coverage_sufficient(docs,Q,ctx))
|
||||
report=build_agent_report(docs,ctx,Q)
|
||||
self.assertEqual(len(report['facts']),2)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
answer,ids=_compose_answer_blocks([{'factIds':['F1','F2']}],report['facts'])
|
||||
self.assertEqual(ids,['F1','F2'])
|
||||
self.assertIn('17,513.16',answer)
|
||||
self.assertIn('[Q12345678]',answer)
|
||||
self.assertNotIn('八成',answer)
|
||||
prefetched={'mode':'query','evidenceRef':'Q87654321','result':{
|
||||
'kind':'quality_metrics','releaseId':'release-a','period':task['period'],
|
||||
'filters':{},'summary':{'black':{'scrapWeight':999}}}}
|
||||
self.assertEqual({fact['kind'] for fact in build_agent_report([prefetched,*docs],ctx,Q)['facts']},{'defect'})
|
||||
ctx['_agentTask']['fields']=['totalWeight']
|
||||
self.assertFalse(coverage_sufficient(docs,Q,ctx))
|
||||
|
||||
def test_wrong_or_unusable_evidence_never_satisfies_defect_top(self):
|
||||
for mutation in ('release','month','filters','stage','coverage','weight','ranking','source'):
|
||||
with self.subTest(mutation=mutation):
|
||||
_,ctx,docs=fixture();r=docs[0]['result']
|
||||
if mutation=='release':r['releaseId']='other'
|
||||
if mutation=='month':r['period']={'year':2026,'month':8}
|
||||
if mutation=='filters':r['filters']={'group':'OTHER'}
|
||||
if mutation=='stage':r['stage']='bright'
|
||||
if mutation=='coverage':r['coverage']['available']=False
|
||||
if mutation=='weight':r['rows'][0]['weight']=None
|
||||
if mutation=='ranking':r['rankingComplete']=False
|
||||
if mutation=='source':r['sourceRows']=0
|
||||
self.assertFalse(coverage_sufficient(docs,Q,ctx))
|
||||
|
||||
def test_failed_supplement_cannot_discard_facts_or_promote_missing_stage(self):
|
||||
task,ctx,docs=fixture()
|
||||
calls=[{'status':'completed'},{'status':'failed'}]
|
||||
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',calls,False,None))
|
||||
self.assertTrue(eligible('m','PARTIAL','evidence_incomplete',calls,False,None,True))
|
||||
self.assertTrue(eligible('m','FAILED','mcp_error',calls,False,None,True))
|
||||
for cancelled,stopped in ((True,None),(False,False)):
|
||||
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',calls,cancelled,stopped,True))
|
||||
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',[{'status':'started'}],False,None,True))
|
||||
ctx['_agentTask']['dimensionStages']['defect'].append('bright')
|
||||
self.assertFalse(coverage_sufficient(docs,Q,ctx))
|
||||
self.assertTrue(build_agent_report(docs,ctx,Q)['missingEvidence'])
|
||||
|
||||
def test_null_nonclarification_and_independent_review_use_trusted_calendar(self):
|
||||
payloads=[]
|
||||
value=task_value(dimensions=['defect'],dimensionStages={'defect':['black']},stages=['black'],fields=[],
|
||||
period={'year':2026,'month':9},clarification=None,defectScope=None,defectProductCount=None)
|
||||
def opener(req,timeout):
|
||||
payloads.append(json.loads(req.data))
|
||||
result=value if len(payloads)==1 else {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(result)}}]}).encode())
|
||||
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question=Q,
|
||||
context={'_serverHistory':[{'question':'你好,你是谁,能做什么','evidenceRefs':[]}]},
|
||||
previous={'kind':'explain','goal':'问候','rootGoal':'问候'},cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(len(payloads),2)
|
||||
self.assertEqual(result['clarification'],'')
|
||||
self.assertEqual(result['rootGoal'],result['goal'])
|
||||
payload=json.loads(payloads[0]['messages'][1]['content'])
|
||||
self.assertEqual(payload['serverCalendar']['date'],datetime.now(ZoneInfo('Asia/Shanghai')).date().isoformat())
|
||||
self.assertFalse(payload['platformCapabilities']['charts']['drawingToolRequired'])
|
||||
with self.assertRaises(TaskUnavailable):normalize_task({**value,'needsClarification':True},Q)
|
||||
|
||||
def test_fact_ids_keep_all_numeric_and_claim_guards(self):
|
||||
facts=build_agent_report(fixture()[2],fixture()[1],Q)['facts']
|
||||
self.assertIn('[Q12345678]',_compose_answer_blocks([{'text':'{{F1}}'}],facts)[0])
|
||||
for block in ({'factIds':['F999']},{'factIds':['F1'],'lead':'上升20%'},
|
||||
{'factIds':['F1'],'advice':'证明工艺是根因'},
|
||||
{'factIds':['F1'],'lead':'占八成以上'},
|
||||
{'factIds':['F1'],'text':'额外正文'}):
|
||||
with self.subTest(block=block),self.assertRaises(CompositionError) as caught:
|
||||
_compose_answer_blocks([block],facts)
|
||||
self.assertIn('answerBlocks[0]',caught.exception.field)
|
||||
self.assertIn('safeRepair',caught.exception.details())
|
||||
|
||||
def test_structured_focus_does_not_repeat_facts_across_answer_blocks(self):
|
||||
_,ctx,docs=fixture();facts=build_agent_report(docs,ctx,Q)['facts']
|
||||
answer,selected=_compose_answer_blocks([
|
||||
{'factIds':['F1']}, {'type':'bullet','factIds':['F1','F2']},
|
||||
{'type':'bullet','factIds':['F1']}],facts)
|
||||
self.assertEqual(selected,['F1','F2'])
|
||||
self.assertEqual(answer.count('17,513.16'),1)
|
||||
self.assertEqual(answer.count('13,949.63'),1)
|
||||
|
||||
def test_finalizer_retains_dimension_stage_and_exact_period(self):
|
||||
task,ctx,_=fixture();summary=_agent_task_summary(ctx)
|
||||
self.assertEqual(summary['dimensionStages'],{'defect':['black']})
|
||||
self.assertEqual(summary['period'],task['period'])
|
||||
self.assertEqual(summary['filters'],{})
|
||||
self.assertFalse(summary['comparisonRequired'])
|
||||
|
||||
def test_source_defect_top_does_not_prefetch_unrequested_formal_kpis(self):
|
||||
from nao_agent_contract import primary_query_required
|
||||
task,_,_=fixture()
|
||||
self.assertFalse(primary_query_required(task))
|
||||
self.assertTrue(primary_query_required({**task,'comparisonRequired':True}))
|
||||
self.assertTrue(primary_query_required({**task,'kind':'metrics'}))
|
||||
self.assertTrue(primary_query_required({**task,'dimensions':['defect','product']}))
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,37 @@
|
||||
import sys,json,io,tempfile,uuid,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch,MagicMock
|
||||
from contextlib import ExitStack
|
||||
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'scripts'),str(ROOT/'verification')]
|
||||
import nao_quality_runtime as runtime
|
||||
from test_agent_contract_regressions import fixture,Q
|
||||
class RuntimeFinalizationRegression(unittest.TestCase):
|
||||
def test_verified_defect_answer_survives_failed_auxiliary_renderer(self):
|
||||
with tempfile.TemporaryDirectory() as folder,ExitStack() as stack:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
request={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')};request.update(question=Q,context={},accessToken='synthetic')
|
||||
task,ctx,docs=fixture();docs[0]['result']['releaseId']=request['releaseId'];doc={**docs[0],'tool':'quality_data','releaseId':request['releaseId']}
|
||||
manager=MagicMock();manager.acquire_actor.return_value=(root,None);manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','synthetic','m','http://127.0.0.1:5000')
|
||||
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1';gate.request_audit=[]
|
||||
def native(**kwargs):
|
||||
p=root/'evidence'/('nao-quality-'+'a'*32+'.json');p.write_text(json.dumps(doc));p.chmod(0o600)
|
||||
stream=runtime.ConversationAccumulator();stream.blocks['draft']='原始草稿';stream.block_order.append('draft');stream.record_event('assistant',text='原始草稿')
|
||||
stream.tool_calls=[{'tool':'quality_data','status':'completed'},{'tool':'display_chart','status':'failed'}]
|
||||
return stream,'PARTIAL','mcp_error',None
|
||||
def synthesize(**kwargs):return {'answer':'责废缺角来源重量17,513.16 kg。[Q12345678]','model':'m','usage':{},'finishReason':'stop','evidenceSufficient':True,'missingEvidence':[]}
|
||||
replacements={'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
|
||||
'_authenticate':MagicMock(),'_input_cancel_listener':MagicMock(),'SessionState':MagicMock(),
|
||||
'_sse_request':native,'_story_files':MagicMock(return_value=(None,None)),
|
||||
'_model_settings':MagicMock(return_value=('https://model.test/v1','synthetic','m',{'enabled':True,'effort':'medium','protocol':'thinking'})),
|
||||
'plan_agent_task':MagicMock(return_value=task),'_model_gate_settings':MagicMock(return_value=(49191,'synthetic')),
|
||||
'DelegatedApi':MagicMock(),'read_primary_evidence':MagicMock(return_value=None)}
|
||||
for name,value in replacements.items():stack.enter_context(patch.object(runtime,name,value))
|
||||
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate));stack.enter_context(patch('nao_evidence_finalizer.synthesize',side_effect=synthesize))
|
||||
result=runtime.run_turn(request,stdout=io.StringIO(),stdin=io.StringIO())
|
||||
error=root/'private/last-runtime-error.json'
|
||||
if error.exists():raise AssertionError(json.loads(error.read_text())['trace'])
|
||||
assert result['status']=='COMPLETED'
|
||||
assert result['timeline'][-1]['kind']=='assistant'
|
||||
assert not any(entry.get('text')=='原始草稿' for entry in result['timeline'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,445 @@
|
||||
"""Single-period and mixed historical report composition tests."""
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_report import build_agent_report, build_noncomparison_report
|
||||
import nao_evidence_finalizer as finalizer
|
||||
|
||||
|
||||
RELEASE = 'release-verified'
|
||||
|
||||
|
||||
def task(question, *, kind='metrics', period=None, filters=None, stages=None, fields=None,
|
||||
history_mode='none'):
|
||||
return {'version': 1, 'question': question, 'kind': kind,
|
||||
'historyMode': history_mode, 'filters': filters or {}, 'period': period,
|
||||
'comparisonPeriod': None, 'dimensions': [], 'stages': stages or [],
|
||||
'fields': fields or [], 'verifySource': False, 'fullDetails': False}
|
||||
|
||||
|
||||
def metric_doc(*, month=9, product='产品甲', rate=.81, black_rate=.84,
|
||||
bright_rate=.78, black_rows=2, bright_rows=2, evidence='A'):
|
||||
period = {'year': 2026, 'month': month}
|
||||
filters = {'product': product} if product else {}
|
||||
coverage = {'sourceMonths': [{'year': 2026, 'month': month,
|
||||
'blackRows': black_rows, 'brightRows': bright_rows}]}
|
||||
summary = {'compositeRate': rate,
|
||||
'black': {'rate': black_rate, 'totalWeight': 123.25, 'scrapWeight': 3.5},
|
||||
'bright': {'rate': bright_rate, 'totalWeight': 88.75, 'scrapWeight': 2.25}}
|
||||
return {'evidenceRef': 'Q' + evidence * 8, 'mode': 'query',
|
||||
'result': {'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
|
||||
'filters': filters, 'view': 'overall', 'summary': summary,
|
||||
'coverage': coverage}}
|
||||
|
||||
|
||||
def comparison_doc(*, month=9, product='产品甲', evidence='B'):
|
||||
def snapshot(selected_month, rate):
|
||||
return {'period': {'year': 2026, 'month': selected_month}, 'compositeRate': rate,
|
||||
'black': {'rate': .9, 'totalWeight': 100, 'scrapWeight': 2},
|
||||
'bright': {'rate': .9, 'totalWeight': 100, 'scrapWeight': 2},
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': selected_month,
|
||||
'blackRows': 2, 'brightRows': 2}]}}
|
||||
return {'evidenceRef': 'Q' + evidence * 8, 'mode': 'compare',
|
||||
'result': {'kind': 'quality_comparison', 'releaseId': RELEASE,
|
||||
'filterBinding': {'product': product},
|
||||
'current': snapshot(month, .92), 'comparison': snapshot(month - 1, .9),
|
||||
'delta': {'available': True, 'percentagePoints': 2},
|
||||
'stageRateChangesPP': {'black': 0, 'bright': 0}}}
|
||||
|
||||
|
||||
class NoncomparisonReportTest(unittest.TestCase):
|
||||
def test_leaf_only_report_has_a_real_core_conclusion(self):
|
||||
facts=[{'id':'F1','kind':'defect','text':'缺陷甲来源重量12.50 kg。','evidenceRef':'QAAAAAAAA'},
|
||||
{'id':'F2','kind':'defect','text':'缺陷乙来源重量8.00 kg。','evidenceRef':'QAAAAAAAA'}]
|
||||
answer,selected=finalizer._focused_report_answer(facts,['F1','F2'])
|
||||
self.assertIn('## 核心结论\n\n- 缺陷甲',answer)
|
||||
self.assertIn('## 优先关注\n\n- 缺陷乙',answer)
|
||||
self.assertEqual(selected,['F1','F2'])
|
||||
|
||||
def context(self, question, **updates):
|
||||
value = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
|
||||
'filters': {'product': '产品甲'},
|
||||
'_agentTask': task(question, period={'year': 2026, 'month': 9},
|
||||
filters={'product': '产品甲'}, fields=['rate'])}
|
||||
value['_agentTask'].update(updates)
|
||||
return value
|
||||
|
||||
def test_single_query_uses_bound_quality_metrics_not_a_stale_generic_query(self):
|
||||
question = '查询产品甲本月合格率'
|
||||
fresh = metric_doc(rate=.81)
|
||||
stale = {'evidenceRef': 'Q' + 'C' * 8, 'mode': 'query',
|
||||
'result': {'kind': 'quality_query', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9},
|
||||
'filters': {'product': '产品甲'}, 'summary': {'compositeRate': .99}}}
|
||||
report = build_noncomparison_report([stale, fresh], self.context(question), question)
|
||||
rendered = report['headline'] + '\n' + '\n'.join(report['sections'])
|
||||
self.assertIn('81.00%', rendered)
|
||||
self.assertNotIn('99.00%', rendered)
|
||||
self.assertEqual(report['facts'][0]['evidenceRef'], fresh['evidenceRef'])
|
||||
self.assertEqual(report['coverageSufficient'], True)
|
||||
|
||||
def test_explicit_stage_and_fields_do_not_add_unrequested_numeric_facts(self):
|
||||
question = '只看黑片投入重量'
|
||||
context = self.context(question)
|
||||
context['_agentTask'].update({'stages': ['black'], 'fields': ['totalWeight']})
|
||||
report = build_noncomparison_report([metric_doc()], context, question)
|
||||
texts = [fact['text'] for fact in report['facts']]
|
||||
self.assertEqual(len(texts), 1)
|
||||
self.assertIn('黑片投入重量:123.25 kg', texts[0])
|
||||
self.assertNotIn('亮片', texts[0])
|
||||
self.assertNotIn('综合', texts[0])
|
||||
|
||||
def test_fieldless_report_includes_composite_and_stage_input_scrap(self):
|
||||
question = '生成质量指标报告'
|
||||
context = {'releaseId': RELEASE, 'filters': {'product': '产品甲'},
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
report = build_noncomparison_report([metric_doc()], context, question)
|
||||
labels = [fact['label'] for fact in report['facts']]
|
||||
self.assertEqual(labels, ['综合合格率', '黑片合格率', '黑片投入重量', '黑片报废重量',
|
||||
'亮片合格率', '亮片投入重量', '亮片报废重量'])
|
||||
self.assertIn('81.00%', report['sections'][0])
|
||||
self.assertIn('123.25 kg', report['sections'][0])
|
||||
|
||||
def test_quarter_frame_uses_all_three_server_coverage_months(self):
|
||||
question = '查询产品甲第三季度合格率'
|
||||
doc = metric_doc(month=9, rate=.81)
|
||||
doc['result']['period'] = {'year': 2026, 'quarter': 3}
|
||||
doc['result']['coverage']['sourceMonths'] = [
|
||||
{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}
|
||||
for month in (7, 8, 9)]
|
||||
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'quarter': 3},
|
||||
'filters': {'product': '产品甲'},
|
||||
'_agentTask': task(question, period={'year': 2026, 'quarter': 3},
|
||||
filters={'product': '产品甲'}, fields=['rate'])}
|
||||
report = build_noncomparison_report([doc], context, question)
|
||||
self.assertIn('2026年第3季度', report['sections'][0])
|
||||
self.assertIn('81.00%', report['facts'][0]['text'])
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
|
||||
def test_multiple_noncomparison_frames_remain_separately_scoped(self):
|
||||
question = '整合历史统计生成报告'
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
first = metric_doc(month=9, product='产品甲', rate=.81, evidence='1')
|
||||
second = metric_doc(month=7, product='产品乙', rate=.76, evidence='2')
|
||||
report = build_noncomparison_report([first, second], context, question)
|
||||
self.assertEqual(len(report['sections']), 2)
|
||||
text = '\n'.join(report['sections'])
|
||||
self.assertIn('产品甲', text)
|
||||
self.assertIn('产品乙', text)
|
||||
self.assertIn('81.00%', text)
|
||||
self.assertIn('76.00%', text)
|
||||
|
||||
def test_missing_stage_coverage_remains_partial_and_does_not_show_rate(self):
|
||||
question = '查询黑片合格率'
|
||||
context = self.context(question)
|
||||
context['_agentTask'].update({'stages': ['black'], 'fields': ['rate']})
|
||||
doc = metric_doc(black_rows=0)
|
||||
report = build_noncomparison_report([doc], context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(report['missingEvidence'])
|
||||
self.assertIn('黑片合格率:未提供', report['facts'][0]['text'])
|
||||
self.assertNotIn('84.00%', report['facts'][0]['text'])
|
||||
|
||||
def test_period_and_scope_mismatch_is_not_replaced_with_another_frame(self):
|
||||
question = '查询产品甲本月合格率'
|
||||
context = self.context(question)
|
||||
with self.assertRaises(ValueError):
|
||||
build_noncomparison_report([metric_doc(month=8)], context, question)
|
||||
|
||||
|
||||
class MixedHistoryReportTest(unittest.TestCase):
|
||||
def test_comparison_history_appends_a_distinct_single_period_scope(self):
|
||||
question = '整合前述分析生成报告'
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
comparison = comparison_doc(product='产品甲')
|
||||
historic_metric = metric_doc(month=7, product='产品乙', rate=.81, evidence='D')
|
||||
report = build_agent_report([comparison, historic_metric], context, question)
|
||||
text = report['headline'] + '\n' + '\n'.join(report['sections'])
|
||||
self.assertIn('产品甲', text)
|
||||
self.assertIn('产品乙', text)
|
||||
self.assertIn('81.00%', text)
|
||||
self.assertTrue(any(fact['evidenceRef'] == historic_metric['evidenceRef'] for fact in report['facts']))
|
||||
|
||||
def test_metrics_already_inside_comparison_are_not_duplicated(self):
|
||||
question = '整合前述分析生成报告'
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
comparison = comparison_doc(product='产品甲')
|
||||
duplicate = metric_doc(month=9, product='产品甲', rate=.92, evidence='E')
|
||||
report = build_agent_report([comparison, duplicate], context, question)
|
||||
refs = [fact['evidenceRef'] for fact in report['facts']]
|
||||
self.assertNotIn(duplicate['evidenceRef'], refs)
|
||||
self.assertEqual(report['coverageSufficient'], True)
|
||||
|
||||
def test_verified_source_and_rows_summaries_are_appended_without_raw_paths(self):
|
||||
question = '整合前述分析生成报告'
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
comparison = comparison_doc()
|
||||
investigations = [
|
||||
{'evidenceRef': 'Q' + 'F' * 8, 'mode': 'investigate',
|
||||
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'stage': 'black',
|
||||
'filters': {}, 'action': 'source',
|
||||
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
||||
'stage': 'black'},
|
||||
'totalRows': 205, 'aggregateComplete': True,
|
||||
'sourceVerification': {'complete': True,
|
||||
'allSelectedValuesMatchStored': True,
|
||||
'expectedRows': 205, 'verifiedRows': 205,
|
||||
'missingRows': 0, 'mismatchRows': 0,
|
||||
'files': [{'sourceFile': '/private/raw.xlsx'}]}}}},
|
||||
{'evidenceRef': 'Q' + 'A' * 8, 'mode': 'investigate',
|
||||
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'stage': 'bright',
|
||||
'filters': {}, 'action': 'rows',
|
||||
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
||||
'stage': 'bright'},
|
||||
'matchingRows': 12, 'offset': 0, 'returnedRows': 10,
|
||||
'hasMore': True, 'recordFilter': 'differences',
|
||||
'rows': [{'sensitive': 'must not render'}]}}}
|
||||
]
|
||||
report = build_agent_report([comparison, *investigations], context, question)
|
||||
rendered = '\n'.join(report['sections'])
|
||||
self.assertIn('原表值全部匹配且核验完整', rendered)
|
||||
self.assertIn('已核验205行/应核验205行', rendered)
|
||||
self.assertIn('共12条', rendered)
|
||||
self.assertIn('差异记录', rendered)
|
||||
self.assertNotIn('/private/raw.xlsx', rendered)
|
||||
self.assertNotIn('must not render', rendered)
|
||||
self.assertIn('QFFFFFFFF', rendered)
|
||||
self.assertIn('QAAAAAAAA', rendered)
|
||||
table = rendered.split('### 原表核验与明细读取', 1)[1].split('\n### ', 1)[0]
|
||||
table_rows = [line for line in table.splitlines() if line.startswith('|')]
|
||||
self.assertTrue(table_rows)
|
||||
self.assertEqual(len(table_rows[0].split('|')), 7)
|
||||
self.assertTrue(all(len(line.split('|')) == 7 for line in table_rows[1:]))
|
||||
|
||||
def test_full_details_preserve_all_four_rows_and_blank_values_without_paths(self):
|
||||
question = '读取9月黑片全部明细'
|
||||
detail_task = task(question, kind='details', period={'year': 2026, 'month': 9},
|
||||
filters={}, stages=['black'], fields=[])
|
||||
detail_task['fullDetails'] = True
|
||||
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
|
||||
'filters': {}, '_agentTask': detail_task}
|
||||
records = [
|
||||
{'rowNumber': 11, 'product': '规格甲', 'sourceFile': '/private/source.xlsx',
|
||||
'values': {'数值字段': 0}},
|
||||
{'rowNumber': 12, 'product': '规格乙', 'sourceFile': '/private/source.xlsx',
|
||||
'values': {'空值字段': None}},
|
||||
{'rowNumber': 13, 'product': '规格丙', 'sourceFile': '/private/source.xlsx',
|
||||
'values': {'文本字段': '已保留'}},
|
||||
{'rowNumber': 14, 'product': '规格丁', 'sourceFile': '/private/source.xlsx',
|
||||
'values': {'重量字段': 2.5}},
|
||||
]
|
||||
document = {'evidenceRef': 'Q' + '6' * 8, 'mode': 'investigate',
|
||||
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': {},
|
||||
'stage': 'black', 'category': None, 'action': 'rows',
|
||||
'current': {'context': {'releaseId': RELEASE, 'year': 2026,
|
||||
'month': 9, 'stage': 'black',
|
||||
'category': None, 'includeUnassigned': False},
|
||||
'matchingRows': 4, 'offset': 0, 'returnedRows': 4,
|
||||
'hasMore': False, 'recordFilter': 'all',
|
||||
'rows': records}}}
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
evidence_path = Path(temp_dir) / ('nao-quality-' + '6' * 32 + '.json')
|
||||
evidence_path.write_text(json.dumps({'tool': 'quality_data', 'mode': 'investigate',
|
||||
'releaseId': RELEASE, 'evidenceRef': document['evidenceRef'],
|
||||
'apiSource': {'base': '/private/source.xlsx'},
|
||||
'result': document['result']}, ensure_ascii=False))
|
||||
evidence_path.chmod(0o600)
|
||||
summaries = finalizer.evidence_summaries(Path(temp_dir), RELEASE)
|
||||
|
||||
page = summaries[0]['result']['current']
|
||||
self.assertEqual(len(page['differenceRecords']), 4)
|
||||
self.assertEqual(page['differenceRecords'][0]['values'], {'数值字段': 0})
|
||||
self.assertEqual(page['differenceRecords'][1]['values'], {'空值字段': None})
|
||||
self.assertNotIn('sourceFile', json.dumps(summaries, ensure_ascii=False))
|
||||
self.assertNotIn('/private/source.xlsx', json.dumps(summaries, ensure_ascii=False))
|
||||
|
||||
summarized_doc = {**document, 'result': summaries[0]['result']}
|
||||
report = build_agent_report([summarized_doc], context, question)
|
||||
rendered = '\n'.join(report['sections'])
|
||||
detail_table = rendered.split('### 2026年9月 · 黑片原表行明细', 1)[1]
|
||||
table_lines = [line for line in detail_table.splitlines() if line.startswith('|')]
|
||||
self.assertEqual(len(table_lines[0].split('|')), 6)
|
||||
self.assertEqual(len(table_lines), 6)
|
||||
for row_number in ('11', '12', '13', '14'):
|
||||
self.assertIn(row_number + ' [Q66666666]', detail_table)
|
||||
self.assertIn('| 0 |', detail_table)
|
||||
self.assertIn('| 未提供 |', detail_table)
|
||||
self.assertIn('已保留', detail_table)
|
||||
self.assertIn('2.5', detail_table)
|
||||
self.assertNotIn('/private/source.xlsx', rendered)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
|
||||
def test_source_only_request_can_render_exact_verified_proof_without_kpi(self):
|
||||
question = '核对本月黑片原表'
|
||||
source_task = task(question, kind='source_verify', period={'year': 2026, 'month': 9},
|
||||
filters={}, stages=['black'], fields=[], history_mode='none')
|
||||
source_task['verifySource'] = True
|
||||
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
|
||||
'filters': {}, '_agentTask': source_task}
|
||||
proof = {'complete': True, 'allSelectedValuesMatchStored': True,
|
||||
'expectedRows': 12, 'verifiedRows': 12, 'missingRows': 0, 'mismatchRows': 0}
|
||||
evidence = [{'evidenceRef': 'Q' + '9' * 8, 'mode': 'investigate',
|
||||
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': {},
|
||||
'stage': 'black', 'action': 'source',
|
||||
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
||||
'stage': 'black'},
|
||||
'totalRows': 12, 'aggregateComplete': True,
|
||||
'sourceVerification': proof}}}]
|
||||
report = build_agent_report(evidence, context, question)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('已核验12行/应核验12行', report['sections'][0])
|
||||
self.assertFalse(report['rawMetrics'])
|
||||
|
||||
def test_source_only_history_with_mismatched_proof_stays_partial(self):
|
||||
question = '整合前述分析生成报告'
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(question, kind='report', history_mode='all')}
|
||||
evidence = [{'evidenceRef': 'Q' + '8' * 8, 'mode': 'investigate',
|
||||
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': {},
|
||||
'stage': 'black', 'action': 'source',
|
||||
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
||||
'stage': 'black'},
|
||||
'totalRows': 12, 'aggregateComplete': True,
|
||||
'sourceVerification': {'complete': True,
|
||||
'allSelectedValuesMatchStored': True,
|
||||
'expectedRows': 12, 'verifiedRows': 11,
|
||||
'missingRows': 1, 'mismatchRows': 0}}}}]
|
||||
report = build_agent_report(evidence, context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn('原表核验未完整通过', report['sections'][0])
|
||||
self.assertTrue(report['missingEvidence'])
|
||||
|
||||
|
||||
class ReportTitleTest(unittest.TestCase):
|
||||
question = '整合前述分析生成报告'
|
||||
|
||||
def run_synthesis(self, parsed=None, *, timed_out=False, capture=None):
|
||||
context = {'releaseId': RELEASE,
|
||||
'_agentTask': task(self.question, kind='report', history_mode='all')}
|
||||
|
||||
def opener(request, timeout):
|
||||
if timed_out:
|
||||
raise TimeoutError('synthetic timeout')
|
||||
if capture is not None:
|
||||
capture['body'] = json.loads(request.data)
|
||||
body = {'choices': [{'finish_reason': 'stop',
|
||||
'message': {'content': json.dumps(parsed or {}, ensure_ascii=False)}}]}
|
||||
return io.BytesIO(json.dumps(body, ensure_ascii=False).encode())
|
||||
|
||||
return finalizer.synthesize(base_url='https://offline.invalid/v1', api_key='synthetic',
|
||||
model='compatible/test', question=self.question,
|
||||
evidence=[comparison_doc()], context=context,
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
|
||||
@staticmethod
|
||||
def composed(title):
|
||||
return {'reportTitle': title,
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '{{F1}}'}],
|
||||
'evidenceSufficient': True, 'missingEvidence': []}
|
||||
|
||||
def test_model_title_is_accepted_when_bound_to_scope_period_and_topic(self):
|
||||
title = '2026年8月至9月 产品甲 合格率变化'
|
||||
captured = {}
|
||||
result = self.run_synthesis(self.composed(title), capture=captured)
|
||||
self.assertEqual(result['reportTitle'], title)
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
selector = json.loads(captured['body']['messages'][1]['content'])
|
||||
self.assertIn('2026年8月', selector['reportTitleGuide']['actualPeriods'])
|
||||
self.assertIn('规格=产品甲', selector['reportTitleGuide']['actualScopes'])
|
||||
self.assertIn('合格率', selector['reportTitleGuide']['topics'])
|
||||
self.assertIn('reportTitle', captured['body']['messages'][0]['content'])
|
||||
|
||||
def test_question_as_title_is_replaced_with_neutral_evidence_title(self):
|
||||
result = self.run_synthesis(self.composed(self.question))
|
||||
self.assertNotEqual(result['reportTitle'], self.question)
|
||||
self.assertFalse(result['titleGeneratedByModel'])
|
||||
self.assertIn('2026年', result['reportTitle'])
|
||||
|
||||
def test_natural_title_paraphrase_does_not_require_fixed_topic_words(self):
|
||||
title = '2026年8—9月质量变化与重点规格报告'
|
||||
result = self.run_synthesis(self.composed(title))
|
||||
self.assertEqual(result['reportTitle'], title)
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
|
||||
def test_malformed_unsafe_or_unbound_numeric_titles_fall_back(self):
|
||||
for title in ('<script>2026年8月产品甲合格率</script>',
|
||||
'https://example.test 2026年产品甲合格率',
|
||||
'sk-abcdef0123456789 2026年产品甲合格率',
|
||||
'2026年8月\n产品甲合格率',
|
||||
'2027年8月产品甲合格率'):
|
||||
with self.subTest(title=title):
|
||||
result = self.run_synthesis(self.composed(title))
|
||||
self.assertFalse(result['titleGeneratedByModel'])
|
||||
self.assertNotIn('<', result['reportTitle'])
|
||||
self.assertNotIn('https://', result['reportTitle'])
|
||||
self.assertNotIn('2027', result['reportTitle'])
|
||||
|
||||
def test_valid_model_title_survives_answer_block_fallback(self):
|
||||
title = '2026年8月至9月 产品甲 合格率变化'
|
||||
result = self.run_synthesis({
|
||||
**self.composed(title),
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '未绑定的99%结论{{F1}}'}],
|
||||
})
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertEqual(result['reportTitle'], title)
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
|
||||
def test_brief_metric_fallback_renders_only_requested_bound_fact(self):
|
||||
question = '查询产品甲本月综合合格率'
|
||||
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
|
||||
'filters': {'product': '产品甲'},
|
||||
'_agentTask': task(question, period={'year': 2026, 'month': 9},
|
||||
filters={'product': '产品甲'}, fields=['rate'])}
|
||||
context['_agentTask'].update({'kind': 'metrics', 'responseStyle': 'brief'})
|
||||
evidence = [metric_doc(rate=.81, evidence='7')]
|
||||
parsed = {'reportTitle': '2026年9月 产品甲 合格率',
|
||||
'answerBlocks': [{'type': 'paragraph', 'text': '未绑定结论99%:{{F1}}'}],
|
||||
'evidenceSufficient': True, 'missingEvidence': []}
|
||||
|
||||
def opener(request, timeout):
|
||||
body = {'choices': [{'finish_reason': 'stop',
|
||||
'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]}
|
||||
return io.BytesIO(json.dumps(body, ensure_ascii=False).encode())
|
||||
|
||||
result = finalizer.synthesize(base_url='https://offline.invalid/v1', api_key='synthetic',
|
||||
model='compatible/test', question=question, evidence=evidence,
|
||||
context=context, cancel_event=threading.Event(), opener=opener)
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertEqual(result['reportTitle'], parsed['reportTitle'])
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
self.assertIn('81.00%', result['answer'])
|
||||
self.assertIn('[Q77777777]', result['answer'])
|
||||
self.assertNotIn('| 指标 |', result['answer'])
|
||||
self.assertNotIn('重点关注', result['answer'])
|
||||
self.assertNotIn('重量单位', result['answer'])
|
||||
|
||||
def test_timeout_and_legacy_response_have_neutral_nonquestion_titles(self):
|
||||
timeout_result = self.run_synthesis(timed_out=True)
|
||||
self.assertFalse(timeout_result['titleGeneratedByModel'])
|
||||
self.assertNotIn(self.question, timeout_result['reportTitle'])
|
||||
self.assertIn('产品甲', timeout_result['reportTitle'])
|
||||
legacy = self.run_synthesis({'otherFactIds': ['F1'], 'groupFactIds': [],
|
||||
'productFactIds': [], 'defectFactIds': [],
|
||||
'evidenceSufficient': True, 'missingEvidence': []})
|
||||
self.assertFalse(legacy['titleGeneratedByModel'])
|
||||
self.assertIn('合格率', legacy['reportTitle'])
|
||||
self.assertNotIn(self.question, legacy['reportTitle'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,571 @@
|
||||
"""Runtime/task integration regressions; all model, native and delegated I/O is mocked."""
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_evidence_finalizer as finalizer
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_agent_task import TaskUnavailable, normalize_task, plan_repairs
|
||||
from nao_quality_tools import QualityTools
|
||||
|
||||
|
||||
ACTOR = '11111111-1111-4111-8111-111111111111'
|
||||
SESSION = '22222222-2222-4222-8222-222222222222'
|
||||
RELEASE = '33333333-3333-4333-8333-333333333333'
|
||||
OTHER_RELEASE = '44444444-4444-4444-8444-444444444444'
|
||||
TURN = '55555555-5555-4555-8555-555555555555'
|
||||
|
||||
|
||||
def task(question='查询本月综合合格率', **updates):
|
||||
value = {'kind': 'metrics', 'goal': question, 'freshQuery': True,
|
||||
'historyMode': 'none', 'dimensions': [], 'stages': [], 'fields': ['rate'],
|
||||
'verifySource': False, 'fullDetails': False, 'visualPreference': 'none',
|
||||
'visualType': None, 'responseStyle': 'standard', 'needsClarification': False,
|
||||
'clarification': '', 'filterUpdates': {}, 'period': {'year': 2026, 'month': 9}}
|
||||
value.update(updates)
|
||||
return normalize_task(value, question, {'releaseId': RELEASE})
|
||||
|
||||
|
||||
def snapshot(month, *, rate=.95):
|
||||
return {'period': {'year': 2026, 'month': month}, 'compositeRate': rate,
|
||||
'black': {'rate': .97, 'totalWeight': 100, 'scrapWeight': 3},
|
||||
'bright': {'rate': .98, 'totalWeight': 100, 'scrapWeight': 2},
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
|
||||
'blackRows': 2, 'brightRows': 2}]}}
|
||||
|
||||
|
||||
def metrics(*, month=9, product=None, rate=.95):
|
||||
side = snapshot(month, rate=rate)
|
||||
return {'kind': 'quality_metrics', 'releaseId': RELEASE,
|
||||
'period': side['period'], 'filters': {'product': product} if product else {},
|
||||
'view': 'overall', 'summary': {k: v for k, v in side.items() if k != 'period'},
|
||||
'coverage': side['coverage'], 'rows': [], 'trend': []}
|
||||
|
||||
|
||||
def comparison(*, month=9, product=None, rate=.95):
|
||||
return {'kind': 'quality_comparison', 'releaseId': RELEASE,
|
||||
'filterBinding': {'product': product} if product else {},
|
||||
'current': snapshot(month, rate=rate), 'comparison': snapshot(month - 1, rate=.96),
|
||||
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
|
||||
'stageRateChangesPP': {'black': 0, 'bright': 0}}
|
||||
|
||||
|
||||
def write_doc(root, result, identifier='a' * 32, *, mode=None):
|
||||
mode = mode or ('compare' if result['kind'] == 'quality_comparison' else 'query')
|
||||
ref = 'Q' + identifier[:8].upper()
|
||||
document = {'tool': 'quality_data', 'mode': mode, 'releaseId': result['releaseId'],
|
||||
'evidenceRef': ref, 'createdAt': '2026-10-25T00:00:00Z', 'result': result}
|
||||
path = root / 'evidence' / ('nao-quality-' + identifier + '.json')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(document), encoding='utf-8')
|
||||
path.chmod(0o600)
|
||||
return document, path
|
||||
|
||||
|
||||
def model_response(request, timeout):
|
||||
"""A provider response, not an evidence/coverage mock."""
|
||||
selector = json.loads(json.loads(request.data)['messages'][1]['content'])
|
||||
facts = selector.get('facts', [])
|
||||
payload = {'answerBlocks': [{'type': 'paragraph', 'text': '{{' + f['id'] + '}}'}
|
||||
for f in facts[:12]],
|
||||
'evidenceSufficient': True, 'missingEvidence': []}
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
|
||||
'message': {'content': json.dumps(payload)}}]}).encode())
|
||||
|
||||
|
||||
class AgentRuntimeIntegrationTest(unittest.TestCase):
|
||||
def test_repeated_question_does_not_inherit_previous_wrong_answer_or_goal(self):
|
||||
context={'_serverQuestions':[{'turnId':'first','question':'9月合格率为什么下降','status':'PARTIAL'},
|
||||
{'turnId':'bad1','question':'主要是国内还是国外','status':'COMPLETED'},
|
||||
{'turnId':'bad2','question':'主要是国内还是国外','status':'COMPLETED'}],
|
||||
'_serverHistory':[{'turnId':'bad1','answer':'单期报废构成','evidenceRefs':['QAAAAAAAA']},
|
||||
{'turnId':'bad2','answer':'本期总体','evidenceRefs':['QBBBBBBBB']}],
|
||||
'lastVisibleAnswer':'错误的单期总体','filters':{'department':'新能源汽车磁钢'}}
|
||||
clean,repeated=runtime._drop_repeated_followup_context('主要是国内还是国外',context)
|
||||
self.assertTrue(repeated)
|
||||
self.assertEqual([q['turnId'] for q in clean['_serverQuestions']],['first'])
|
||||
self.assertEqual(clean['_serverHistory'],[])
|
||||
self.assertNotIn('lastVisibleAnswer',clean)
|
||||
self.assertEqual(clean['filters'],context['filters'])
|
||||
self.assertEqual(len(context['_serverQuestions']),3)
|
||||
single,reset=runtime._drop_repeated_followup_context('同一问题',{'_serverQuestions':[{'turnId':'one','question':'同一问题'}],
|
||||
'_serverHistory':[{'turnId':'one','answer':'旧结论'}]})
|
||||
self.assertTrue(reset);self.assertEqual(single['_serverQuestions'],[]);self.assertEqual(single['_serverHistory'],[])
|
||||
|
||||
def test_repeat_context_handles_truncation_and_older_same_question_answers(self):
|
||||
question='比较某产品质量变化及具体分组差异'*24
|
||||
truncated=question[:219]+'…'
|
||||
context={'_serverQuestions':[{'turnId':'root','question':'原来的总体分析','status':'PARTIAL'},
|
||||
{'turnId':'recent','question':truncated,'status':'COMPLETED','truncated':True}],
|
||||
'_serverHistory':[{'turnId':'older','question':truncated,'answer':'过早的错误结论'},
|
||||
{'turnId':'recent','question':truncated,'answer':'最近的错误结论'}]}
|
||||
clean,repeated=runtime._drop_repeated_followup_context(question,context)
|
||||
self.assertTrue(repeated);self.assertEqual(clean['_serverHistory'],[])
|
||||
self.assertEqual([x['turnId'] for x in clean['_serverQuestions']],['root'])
|
||||
|
||||
def run_mocked(self, root, planned, native, *, repair=None, primary=None, history=None):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
(root / 'evidence').mkdir(exist_ok=True)
|
||||
(root / 'private').mkdir(exist_ok=True)
|
||||
payload = {'turnId': TURN, 'sessionId': SESSION, 'actorId': ACTOR,
|
||||
'releaseId': RELEASE, 'accessToken': 'synthetic-delegation',
|
||||
'question': planned['question'], 'context': {'releaseId': RELEASE}}
|
||||
if history is not None: payload['context']['_serverHistory'] = history
|
||||
manager = MagicMock()
|
||||
manager.acquire_actor.return_value = (root, None)
|
||||
manager.ensure_instance.return_value = ('http://127.0.0.1:1/v1', 'synthetic-key',
|
||||
'synthetic-model', 'http://127.0.0.1:2')
|
||||
state = MagicMock()
|
||||
state.get_chat_id.return_value = None
|
||||
gate = MagicMock()
|
||||
gate.base_url = 'http://127.0.0.1:1/v1'
|
||||
with ExitStack() as stack:
|
||||
replacements = {
|
||||
'RuntimeManager': MagicMock(return_value=manager),
|
||||
'SessionState': MagicMock(return_value=state),
|
||||
'write_access_file': MagicMock(return_value=root / 'access.json'),
|
||||
'_authenticate': MagicMock(), '_input_cancel_listener': MagicMock(),
|
||||
'_model_settings': MagicMock(return_value=('https://offline.invalid/v1', 'fake-key', 'synthetic-model',{'enabled':False,'effort':'auto','protocol':'thinking'})),
|
||||
'_model_gate_settings': MagicMock(return_value=(1, 'synthetic-key')),
|
||||
'DelegatedApi': MagicMock(), '_sse_request': native,
|
||||
'_story_files': MagicMock(return_value=(None, None)),
|
||||
'plan_agent_task': MagicMock(return_value=planned),
|
||||
'read_primary_evidence': primary or MagicMock(return_value=None),
|
||||
}
|
||||
for name, replacement in replacements.items():
|
||||
stack.enter_context(patch.object(runtime, name, replacement))
|
||||
stack.enter_context(patch('nao_model_gate.ModelGate', return_value=gate))
|
||||
stack.enter_context(patch.object(finalizer, '_open_model', side_effect=model_response))
|
||||
if repair is not None:
|
||||
stack.enter_context(patch.object(QualityTools, 'quality_data', side_effect=repair))
|
||||
result = runtime.run_turn(payload, stdout=io.StringIO(), stdin=io.StringIO())
|
||||
gate.close.assert_called_once()
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def accumulator(answer):
|
||||
value = runtime.ConversationAccumulator()
|
||||
value.blocks['native'] = answer
|
||||
value.block_order.append('native')
|
||||
value.tool_calls.append({'tool': 'quality_data', 'status': 'completed'})
|
||||
return value
|
||||
|
||||
def test_real_scalar_query_kind_is_fresh_and_completes_with_matching_coverage(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
def native(**kwargs):
|
||||
write_doc(root, metrics())
|
||||
return self.accumulator('综合合格率95.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
|
||||
result = self.run_mocked(root, task(), native)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertNotIn('未取得新的查询证据', result['answer'])
|
||||
|
||||
def test_scalar_wrong_product_period_and_missing_requested_field_remain_partial(self):
|
||||
for defect in ('product', 'period', 'field'):
|
||||
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
result_doc = metrics()
|
||||
if defect == 'product': result_doc['filters'] = {'product': 'other'}
|
||||
if defect == 'period': result_doc = metrics(month=8)
|
||||
if defect == 'field': result_doc['summary'].pop('compositeRate')
|
||||
def native(**kwargs):
|
||||
write_doc(root, result_doc)
|
||||
return self.accumulator('原生回答'), 'COMPLETED', 'stop', None
|
||||
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
||||
result = self.run_mocked(root, task(), native)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
|
||||
def test_no_initial_evidence_enters_repair_and_recovers_comparison(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
planned = task(kind='analysis', fields=[], comparisonPeriod={'year': 2026, 'month': 8})
|
||||
def native(**kwargs):
|
||||
return self.accumulator('尚未取得数据'), 'COMPLETED', 'stop', None
|
||||
def repair(arguments):
|
||||
document, _ = write_doc(root, comparison())
|
||||
return {**document['result'], 'evidenceRef': document['evidenceRef']}
|
||||
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'compare', 'releaseId': RELEASE,
|
||||
'period': planned['period'], 'comparisonPeriod': planned['comparisonPeriod'], 'filters': {}}]) as planner:
|
||||
result = self.run_mocked(root, planned, native, repair=repair)
|
||||
self.assertTrue(planner.called)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['completionMethod'], 'bounded_evidence_finalization')
|
||||
self.assertNotIn('尚未取得数据', result['answer'])
|
||||
self.assertIn('[QAAAAAAAA]', result['answer'])
|
||||
|
||||
def test_scalar_repair_must_replace_the_native_no_data_answer(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
def native(**kwargs):
|
||||
return self.accumulator('尚未取得数据,无法给出合格率。'), 'COMPLETED', 'stop', None
|
||||
def repair(arguments):
|
||||
document, _ = write_doc(root, metrics())
|
||||
return {**document['result'], 'evidenceRef': document['evidenceRef']}
|
||||
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'query', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': {}}]):
|
||||
result = self.run_mocked(root, task(), native, repair=repair)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertNotIn('尚未取得数据', result['answer'])
|
||||
self.assertIn('95.00%', result['answer'])
|
||||
|
||||
def test_scalar_native_claim_must_agree_with_queried_value(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
def native(**kwargs):
|
||||
write_doc(root, metrics(rate=.95))
|
||||
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
|
||||
result = self.run_mocked(root, task(), native)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertNotIn('99.00%', result['answer'])
|
||||
self.assertIn('95.00%', result['answer'])
|
||||
|
||||
def test_ai_selected_primary_frame_is_queried_before_local_investigation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
planned = task(filterUpdates={'product': 'primary-product'})
|
||||
def primary(**kwargs):
|
||||
self.assertEqual(kwargs['arguments']['mode'], 'query')
|
||||
self.assertEqual(kwargs['arguments']['filters'], {'product': 'primary-product'})
|
||||
self.assertEqual(kwargs['arguments']['period'], {'year': 2026, 'month': 9})
|
||||
result = metrics(product='primary-product', rate=.95)
|
||||
document, _ = write_doc(root, result)
|
||||
return {**result, 'evidenceRef': document['evidenceRef']}
|
||||
def native(**kwargs):
|
||||
self.assertEqual(kwargs['context']['primaryEvidence']['summary']['compositeRate'], .95)
|
||||
write_doc(root, metrics(product='unrelated-child', rate=.81), identifier='b' * 32)
|
||||
return self.accumulator('错误局部81%'), 'COMPLETED', 'stop', None
|
||||
result = self.run_mocked(root, planned, native, primary=primary)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('95.00%', result['answer'])
|
||||
self.assertNotIn('81.00%', result['answer'])
|
||||
|
||||
def test_single_period_analysis_without_breakdowns_still_requires_matching_frame(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
def native(**kwargs):
|
||||
write_doc(root, metrics(month=8))
|
||||
return self.accumulator('来自八月的旧期间结论'), 'COMPLETED', 'stop', None
|
||||
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
||||
result = self.run_mocked(root, task(kind='analysis', dimensions=[]), native)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
|
||||
def test_inconsistent_nonfresh_metrics_plan_cannot_complete_without_any_proof(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
def native(**kwargs):
|
||||
accumulator = self.accumulator('未经查询的综合合格率99.00%')
|
||||
accumulator.tool_calls.clear()
|
||||
return accumulator, 'COMPLETED', 'stop', None
|
||||
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
||||
result = self.run_mocked(root, task(freshQuery=False), native)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
|
||||
def test_off_target_native_answer_is_not_completed_when_final_synthesis_fails(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
planned=task(question='主要是国内还是国外的',kind='analysis',dimensions=['group'],
|
||||
dimensionStages={'group':[]},comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
||||
def native(**kwargs):
|
||||
write_doc(root,comparison())
|
||||
changes={'kind':'quality_changes','releaseId':RELEASE,'dimension':'group','stage':None,'filters':{},
|
||||
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
|
||||
'sort':'scrap_delta','totalRows':2,'returnedRows':2,'truncated':False,
|
||||
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
|
||||
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
|
||||
write_doc(root,changes,identifier='b'*32,mode='changes')
|
||||
return self.accumulator('仅有总体指标,没有回答国内或海外。'), 'COMPLETED','stop',None
|
||||
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
|
||||
result=self.run_mocked(root,planned,native)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('答复尚未完成核验',result['answer'])
|
||||
|
||||
def test_wrong_native_metric_is_not_completed_when_final_synthesis_fails(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
def native(**kwargs):
|
||||
write_doc(root,metrics(rate=.95))
|
||||
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'),'COMPLETED','stop',None
|
||||
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
|
||||
result=self.run_mocked(root,task(),native)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertNotIn('99.00%',result['answer'])
|
||||
|
||||
def test_incomplete_tool_call_cannot_skip_answer_validation(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
def native(**kwargs):
|
||||
write_doc(root,metrics(rate=.95))
|
||||
value=self.accumulator('综合合格率99.00% [QAAAAAAAA]')
|
||||
value.tool_calls[0]['status']='started'
|
||||
return value,'COMPLETED','stop',None
|
||||
result=self.run_mocked(root,task(),native)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertNotIn('99.00%',result['answer'])
|
||||
|
||||
def test_nonfresh_business_restatement_without_verified_history_is_partial(self):
|
||||
for kind in ('explain','restate'):
|
||||
with self.subTest(kind=kind),tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
planned=task(question='主要是国内还是国外的',kind=kind,freshQuery=False,
|
||||
historyMode='last',comparisonRequired=True,
|
||||
comparisonPeriod={'year':2026,'month':8})
|
||||
def native(**kwargs):
|
||||
result=self.accumulator('主要是国外,沿用上一轮未完成的结论。')
|
||||
result.tool_calls.clear()
|
||||
return result,'COMPLETED','stop',None
|
||||
with patch('nao_agent_task.plan_repairs',return_value=[]):
|
||||
result=self.run_mocked(root,planned,native)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('未找到同范围、同期间的已核实历史依据',result['answer'])
|
||||
|
||||
def test_nonfresh_restatement_rejects_answer_that_does_not_match_question(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);write_doc(root,comparison())
|
||||
planned=task(question='主要是国内还是国外的',kind='explain',freshQuery=False,
|
||||
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
||||
history=[{'turnId':'prior','question':'为什么下降','answer':'综合率下降','evidenceRefs':['QAAAAAAAA'],
|
||||
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
|
||||
def native(**kwargs):
|
||||
value=self.accumulator('只是总体下降,没有回答分组。');value.tool_calls.clear()
|
||||
return value,'COMPLETED','stop',None
|
||||
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=False,create=True):
|
||||
result=self.run_mocked(root,planned,native,history=history)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('答复尚未完成核验',result['answer'])
|
||||
|
||||
def test_nonfresh_restatement_can_use_matching_verified_history(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);write_doc(root,comparison())
|
||||
planned=task(question='综合合格率变化多少',kind='explain',freshQuery=False,
|
||||
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
||||
history=[{'turnId':'prior','question':'比较两期','answer':'综合率96%到95%','evidenceRefs':['QAAAAAAAA'],
|
||||
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
|
||||
def native(**kwargs):
|
||||
value=self.accumulator('综合合格率由96%降到95%。');value.tool_calls.clear()
|
||||
return value,'COMPLETED','stop',None
|
||||
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=True):
|
||||
result=self.run_mocked(root,planned,native,history=history)
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
|
||||
|
||||
class RepairScopeBindingTest(unittest.TestCase):
|
||||
def plan(self, query, planned):
|
||||
with patch('nao_agent_task._model_json', return_value={'queries': [query]}):
|
||||
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='synthetic',
|
||||
task=planned, evidence=[], gaps=['missing'], failures=[], release_id=RELEASE,
|
||||
cancel_event=threading.Event())
|
||||
|
||||
def test_changes_inherits_the_only_requested_stage(self):
|
||||
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
|
||||
comparisonPeriod={'year': 2026, 'month': 8})
|
||||
accepted = self.plan({'mode': 'changes', 'dimension': 'product', 'filters': {}}, planned)
|
||||
self.assertEqual(accepted[0]['stage'], 'black')
|
||||
self.assertEqual(accepted[0]['period'], planned['period'])
|
||||
self.assertEqual(accepted[0]['comparisonPeriod'], planned['comparisonPeriod'])
|
||||
|
||||
def test_changes_cannot_switch_to_unrequested_stage(self):
|
||||
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
|
||||
comparisonPeriod={'year': 2026, 'month': 8})
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
self.plan({'mode': 'changes', 'dimension': 'product', 'stage': 'bright', 'filters': {}}, planned)
|
||||
|
||||
|
||||
class TrustedHistoricalReportTest(unittest.TestCase):
|
||||
def test_verified_full_report_fallback_is_not_appended_twice(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
document, _ = write_doc(root, comparison())
|
||||
with patch.object(finalizer, 'synthesize', return_value={
|
||||
'answer': '正式完整统计及明细', 'evidenceSufficient': True,
|
||||
'reportTitle': '质量变化分析', 'titleGeneratedByModel': True,
|
||||
'selectionFallback': True}):
|
||||
result = self.run_report(root, [self.entry(document['evidenceRef'])])
|
||||
self.assertTrue(result['reportMarkdown'].startswith('正式完整统计及明细\n\n## 数据明细'))
|
||||
self.assertEqual(result['reportMarkdown'].count('正式完整统计及明细'), 1)
|
||||
|
||||
def run_report(self, root, entries, *, last=False):
|
||||
question = '整合前面分析生成报告'
|
||||
planned = task(question, kind='report', freshQuery=False, historyMode='last' if last else 'all',
|
||||
fields=[], period=None)
|
||||
request = {'question': question, 'releaseId': RELEASE, 'turnId': TURN,
|
||||
'actorId': ACTOR, 'sessionId': SESSION,
|
||||
'context': {'_serverHistory': entries}}
|
||||
with patch.object(finalizer, '_open_model', side_effect=model_response):
|
||||
return runtime.historical_report(request=request, task=planned, actor_root=root,
|
||||
connection=('https://offline.invalid/v1', 'synthetic-only', 'synthetic-model'),
|
||||
cancel_event=threading.Event(), emit=lambda value: None)
|
||||
|
||||
@staticmethod
|
||||
def entry(ref, turn=TURN):
|
||||
return {'turnId': turn, 'evidenceRefs': [ref]}
|
||||
|
||||
def test_all_reopens_both_trusted_frames_and_excludes_unreferenced_files(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
a, _ = write_doc(root, comparison(product='产品甲'), 'a' * 32)
|
||||
b, _ = write_doc(root, comparison(month=7, product='产品乙'), 'b' * 32)
|
||||
write_doc(root, comparison(product='未授权引用产品'), 'c' * 32)
|
||||
# An unselected corrupt file must not poison the selected report.
|
||||
bad = root / 'evidence' / ('nao-quality-' + 'd' * 32 + '.json')
|
||||
bad.write_text('not-json')
|
||||
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('产品甲', result['reportMarkdown'])
|
||||
self.assertEqual(result['agentContext']['filters'], {}, 'all-history report must clear the previous local-only display scope')
|
||||
self.assertIn('产品乙', result['reportMarkdown'])
|
||||
self.assertNotIn('未授权引用产品', result['reportMarkdown'])
|
||||
|
||||
def test_missing_other_session_ref_cannot_be_reopened(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder) / 'current'
|
||||
other = Path(folder) / 'other'
|
||||
a, _ = write_doc(root, comparison(), 'a' * 32)
|
||||
b, _ = write_doc(other, comparison(product='另一会话'), 'b' * 32)
|
||||
with self.assertRaises(runtime.RuntimeErrorSafe):
|
||||
self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
||||
|
||||
def test_report_rejects_wrong_release_public_permissions_and_symlink(self):
|
||||
for defect in ('release', 'permissions', 'symlink'):
|
||||
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder) / 'current'
|
||||
result = comparison()
|
||||
if defect == 'release': result['releaseId'] = OTHER_RELEASE
|
||||
doc, path = write_doc(root, result)
|
||||
if defect == 'permissions': path.chmod(0o644)
|
||||
if defect == 'symlink':
|
||||
target = Path(folder) / 'external.json'
|
||||
path.rename(target)
|
||||
path.symlink_to(target)
|
||||
with self.assertRaises((runtime.RuntimeErrorSafe, finalizer.FinalizationUnavailable)):
|
||||
self.run_report(root, [self.entry(doc['evidenceRef'])])
|
||||
|
||||
def test_last_does_not_reopen_an_older_unavailable_entry(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
a, _ = write_doc(root, comparison(product='最新范围'))
|
||||
result = self.run_report(root, [self.entry('QFFFFFFFF'), self.entry(a['evidenceRef'])], last=True)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('最新范围', result['reportMarkdown'])
|
||||
|
||||
def test_all_must_not_claim_a_reference_free_turn_was_verified(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
a, _ = write_doc(root, comparison())
|
||||
entries = [self.entry(a['evidenceRef']), {'turnId': 'unverified-turn', 'evidenceRefs': []}]
|
||||
result=self.run_report(root, entries)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('未纳入本次报告统计',result['answer'])
|
||||
self.assertTrue(result['reportMarkdown'])
|
||||
self.assertEqual(result['unverifiedHistoryTurnIds'],['unverified-turn'])
|
||||
|
||||
def test_recorded_non_query_turns_do_not_block_all_or_last_analysis_reports(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);a,_=write_doc(root,comparison())
|
||||
chat_turn='66666666-6666-4666-8666-666666666666'
|
||||
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':RELEASE})
|
||||
entries=[{'turnId':chat_turn,'evidenceRefs':[]},self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}]
|
||||
for last in (False,True):
|
||||
result=self.run_report(root,entries,last=last)
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.assertEqual(result['unverifiedHistoryTurnIds'],[])
|
||||
|
||||
def test_non_query_marker_from_another_scope_cannot_hide_unverified_analysis(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);a,_=write_doc(root,comparison())
|
||||
chat_turn='66666666-6666-4666-8666-666666666666'
|
||||
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':OTHER_RELEASE})
|
||||
result=self.run_report(root,[self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}])
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
|
||||
def test_all_includes_single_period_turn_in_addition_to_comparison_turn(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
a, _ = write_doc(root, comparison(product='比较范围'), 'a' * 32)
|
||||
b, _ = write_doc(root, metrics(month=7, product='单期范围', rate=.81), 'b' * 32)
|
||||
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('单期范围', result['reportMarkdown'])
|
||||
self.assertIn('81.00%', result['reportMarkdown'])
|
||||
|
||||
def test_history_can_report_a_verified_source_turn_without_unrelated_metric_query(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
source = {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': {}, 'stage': 'black',
|
||||
'action': 'source', 'current': {
|
||||
'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
||||
'stage': 'black', 'includeUnassigned': False},
|
||||
'aggregateComplete': True, 'totalRows': 2,
|
||||
'sourceVerification': {'complete': True, 'allSelectedValuesMatchStored': True,
|
||||
'expectedRows': 2, 'verifiedRows': 2,
|
||||
'missingRows': 0, 'mismatchRows': 0}}}
|
||||
doc, _ = write_doc(root, source, mode='investigate')
|
||||
result = self.run_report(root, [self.entry(doc['evidenceRef'])])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('原表', result['reportMarkdown'])
|
||||
self.assertIn('[QAAAAAAAA]', result['answer'])
|
||||
|
||||
@staticmethod
|
||||
def defect(month=9, product=None):
|
||||
return {'kind':'quality_fine_defects','releaseId':RELEASE,'dimension':'defect',
|
||||
'period':{'year':2026,'month':month},'comparisonPeriod':None,
|
||||
'filters':{'product':product} if product else {},'stage':'black',
|
||||
'sourceRows':2,'rankingComplete':True,'coverage':{'available':True,'hasValidWeights':True},
|
||||
'rows':[{'key':'corner','label':'责废缺角','category':'责废','weight':12.5}]}
|
||||
|
||||
def test_history_reports_single_period_defects_without_parent_metrics(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);doc,_=write_doc(root,self.defect(),mode='defects')
|
||||
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.assertIn('责废缺角',result['reportMarkdown'])
|
||||
self.assertIn('12.50',result['reportMarkdown'])
|
||||
self.assertNotIn('综合合格率',result['answer'])
|
||||
|
||||
def test_history_keeps_defect_periods_scopes_and_mixed_metrics(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
a,_=write_doc(root,self.defect(product='范围甲'),'a'*32,mode='defects')
|
||||
b,_=write_doc(root,self.defect(month=8,product='范围乙'),'b'*32,mode='defects')
|
||||
c,_=write_doc(root,metrics(product='指标范围'),'c'*32)
|
||||
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b,c)])
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
for text in ('范围甲','范围乙','指标范围','2026年9月','2026年8月','12.50'):
|
||||
self.assertIn(text,result['reportMarkdown'])
|
||||
|
||||
def test_incomplete_historical_defects_are_not_marked_complete(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);good=self.defect();bad=self.defect(month=8);bad['rankingComplete']=False
|
||||
a,_=write_doc(root,good,'a'*32,mode='defects');b,_=write_doc(root,bad,'b'*32,mode='defects')
|
||||
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b)])
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('12.50',result['reportMarkdown'])
|
||||
|
||||
def test_history_reports_weight_ranking_without_parent_metrics(self):
|
||||
from nao_weight_ranking import rank_response
|
||||
response={'rows':[{'key':'规格甲','black':{'totalWeight':60}},{'key':'规格乙','black':{'totalWeight':40}}],
|
||||
'summary':{'black':{'totalWeight':100}},'context':{'rowCount':2},
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2}]}}
|
||||
for period in ({'year':2026,'month':9},{'year':2026}):
|
||||
response['coverage']['sourceMonths']=[{'year':2026,'month':month,'blackRows':2}
|
||||
for month in (range(1,10) if 'month' not in period else [9])]
|
||||
ranking=rank_response(response,{'id':RELEASE},period,{},'product','black','totalWeight',1)
|
||||
with self.subTest(period=period),tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);doc,_=write_doc(root,ranking,mode='rank')
|
||||
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.assertIn('规格甲',result['reportMarkdown'])
|
||||
self.assertIn('60.00%',result['reportMarkdown'])
|
||||
self.assertIn('100.00',result['reportMarkdown'])
|
||||
if 'month' not in period:self.assertIn('1—9月已发布数据',result['reportMarkdown'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Evidence-bound monthly answers must not turn an annual snapshot into a trend."""
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_report import build_noncomparison_report
|
||||
from nao_evidence_finalizer import coverage_sufficient, synthesize
|
||||
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
|
||||
|
||||
|
||||
QUESTION = '查看客户甲今年的综合合格率变化趋势'
|
||||
RELEASE = 'release-trend'
|
||||
|
||||
|
||||
def task(**updates):
|
||||
value = {'version': 1, 'kind': 'metrics', 'question': QUESTION, 'goal': QUESTION,
|
||||
'period': {'year': 2026}, 'comparisonPeriod': None, 'comparisonRequired': False,
|
||||
'filters': {'extraFilters': {'customerCode': ['客户甲']}},
|
||||
'freshQuery': True, 'dimensions': [], 'dimensionStages': {}, 'stages': [],
|
||||
'fields': ['rate'], 'historyMode': 'none', 'trendRequired': True,
|
||||
'periodCoverage': 'available', 'visualPreference': 'auto', 'visualType': None, 'responseStyle': 'brief',
|
||||
'verifySource': False, 'fullDetails': False}
|
||||
value.update(updates)
|
||||
return value
|
||||
|
||||
|
||||
def context(**updates):
|
||||
t = task(**updates)
|
||||
return {'releaseId': RELEASE, 'period': t['period'], 'filters': t['filters'], '_agentTask': t}
|
||||
|
||||
|
||||
def doc(months=(1, 3, 4), customer='客户甲', release=RELEASE, ref='Q12345678'):
|
||||
rows = [{'period': f'2026-{m:02d}', 'compositeRate': value,
|
||||
'black': {'rate': value - .01, 'totalWeight': 100},
|
||||
'bright': {'rate': value + .01, 'totalWeight': 90},
|
||||
'coverage': {'blackRows': 2, 'brightRows': 2}}
|
||||
for m, value in zip(months, (.75, .85, .80, .78))]
|
||||
return {'evidenceRef': ref, 'tool': 'quality_data', 'mode': 'query', 'releaseId': release, 'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': release, 'view': 'overall',
|
||||
'period': {'year': 2026}, 'filters': {'extraFilters': {'customerCode': [customer]}},
|
||||
'summary': {'compositeRate': .81}, 'trend': rows, 'trendCount': len(rows),
|
||||
'trendTruncated': False, 'coverage': {'sourceMonths': [
|
||||
{'year': 2026, 'month': m, 'blackRows': 2, 'brightRows': 2} for m in months]}}}
|
||||
|
||||
|
||||
class TrendEvidenceTests(unittest.TestCase):
|
||||
def test_annual_trend_uses_formal_months_and_keeps_missing_month_empty(self):
|
||||
report = build_noncomparison_report([doc()], context(), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertTrue(coverage_sufficient([doc()], QUESTION, context()))
|
||||
text = '\n'.join(f['text'] for f in report['facts'])
|
||||
self.assertIn('1月75.00%', text)
|
||||
self.assertIn('2月未提供', text)
|
||||
self.assertIn('3月85.00%', text)
|
||||
self.assertIn('4月80.00%', text)
|
||||
self.assertIn('首末对比,4月较1月上升', text)
|
||||
self.assertNotIn('81.00%', text)
|
||||
self.assertEqual(report['facts'][0]['evidenceRef'], 'Q12345678')
|
||||
|
||||
def test_explicit_comparison_still_requires_matching_comparison_evidence(self):
|
||||
comparison = context(comparisonRequired=True, comparisonPeriod={'year': 2025})
|
||||
self.assertFalse(coverage_sufficient([doc()], QUESTION, comparison))
|
||||
self.assertTrue(coverage_gaps([doc()], QUESTION, comparison))
|
||||
|
||||
def test_client_filter_and_release_must_match_exactly(self):
|
||||
wrong = doc(customer='客户乙')
|
||||
right = doc(ref='Q87654321')
|
||||
stale = doc(release='old', ref='QABCDEF12')
|
||||
for docs in ([wrong], [stale]):
|
||||
report = build_noncomparison_report(docs, context(), QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertFalse(coverage_sufficient(docs, QUESTION, context()))
|
||||
self.assertFalse(report['facts'])
|
||||
report = build_noncomparison_report([wrong, stale, right], context(), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertEqual([f['evidenceRef'] for f in report['facts']], ['Q87654321'])
|
||||
|
||||
def test_history_report_never_relabels_another_customer_as_the_current_scope(self):
|
||||
historical = context(kind='report', historyMode='all', freshQuery=False)
|
||||
wrong = doc(customer='客户乙')
|
||||
report = build_noncomparison_report([wrong], historical, QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertFalse(report['facts'])
|
||||
self.assertFalse(coverage_sufficient([wrong], QUESTION, historical))
|
||||
|
||||
def test_empty_truncated_wrong_year_and_duplicate_month_are_incomplete(self):
|
||||
cases = []
|
||||
empty = doc(months=()); cases.append(empty)
|
||||
truncated = doc(); truncated['result']['trendTruncated'] = True; cases.append(truncated)
|
||||
count = doc(); count['result']['trendCount'] += 1; cases.append(count)
|
||||
wrong_year = doc(); wrong_year['result']['trend'][0]['period'] = '2025-01'; cases.append(wrong_year)
|
||||
duplicate = doc(); duplicate['result']['trend'][1]['period'] = '2026-01'; cases.append(duplicate)
|
||||
for item in cases:
|
||||
with self.subTest(item=item):
|
||||
self.assertFalse(build_noncomparison_report([item], context(), QUESTION)['coverageSufficient'])
|
||||
self.assertFalse(coverage_sufficient([item], QUESTION, context()))
|
||||
|
||||
def test_no_trend_data_returns_an_explanation_without_asking_model_to_invent(self):
|
||||
empty = doc(months=())
|
||||
def opener(request, timeout):
|
||||
raise AssertionError('No factual content should be sent for narration')
|
||||
result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question=QUESTION, evidence=[empty], cancel_event=threading.Event(),
|
||||
context=context(), opener=opener)
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertIn('未取得符合本轮范围和期间的可用月度趋势数据', result['answer'])
|
||||
self.assertIn('不足以判断趋势', result['answer'])
|
||||
|
||||
def test_single_stage_composite_retains_formal_business_rate(self):
|
||||
item = doc()
|
||||
for row in item['result']['trend'][:2]: row['coverage']['brightRows'] = 0
|
||||
report = build_noncomparison_report([item], context(), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertTrue(coverage_sufficient([item], QUESTION, context()))
|
||||
self.assertIn('75.00%', str(report['facts']))
|
||||
self.assertIn('85.00%', str(report['facts']))
|
||||
|
||||
def test_stage_reference_both_directions_matches_chart_and_customer_scope(self):
|
||||
from analysis_visuals import build_visuals
|
||||
item = doc(months=(1, 2, 3, 4))
|
||||
rows = item['result']['trend']
|
||||
rows[0]['coverage']['blackRows'] = 0
|
||||
rows[1]['coverage']['blackRows'] = 0
|
||||
rows[1]['coverage']['brightRows'] = 0
|
||||
rows[2]['coverage']['brightRows'] = 0
|
||||
report = build_noncomparison_report([item], context(), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
text = report['facts'][0]['text']
|
||||
for value in ('1月75.00%', '2月未提供', '3月80.00%', '4月78.00%'):
|
||||
self.assertIn(value, text)
|
||||
visual = build_visuals([item], QUESTION, 'turn-customer-stage', context=context())
|
||||
self.assertEqual(visual['visualState'], 'ready')
|
||||
chart = next(c for c in visual['visuals'] if c.get('metric') == 'composite_rate')
|
||||
self.assertEqual(chart['series'][0]['values'], [.75, None, .80, .78])
|
||||
self.assertEqual(chart['scope']['extraFilters'], {'customerCode': ['客户甲']})
|
||||
self.assertEqual(chart['evidenceRefs'], ['Q12345678'])
|
||||
self.assertEqual(build_visuals([doc(customer='客户乙')], QUESTION, 'turn-wrong-client',
|
||||
context=context())['visuals'], [])
|
||||
|
||||
def test_both_stages_missing_does_not_create_observations(self):
|
||||
item = doc()
|
||||
for row in item['result']['trend'][:2]:
|
||||
row['coverage']['blackRows'] = 0
|
||||
row['coverage']['brightRows'] = 0
|
||||
report = build_noncomparison_report([item], context(), QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertFalse(coverage_sufficient([item], QUESTION, context()))
|
||||
self.assertTrue(any('不足以判断趋势' in value for value in report['missingEvidence']))
|
||||
self.assertNotIn('75.00%', str(report['facts']))
|
||||
self.assertNotIn('85.00%', str(report['facts']))
|
||||
|
||||
def test_two_valid_months_explain_chart_limit_and_required_chart_is_partial(self):
|
||||
item = doc(months=(1, 2))
|
||||
report = build_noncomparison_report([item], context(), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('不足3个有效月份', report['facts'][0]['text'])
|
||||
required = context(visualPreference='required')
|
||||
self.assertFalse(build_noncomparison_report([item], required, QUESTION)['coverageSufficient'])
|
||||
self.assertFalse(coverage_sufficient([item], QUESTION, required))
|
||||
|
||||
def test_missing_trend_is_requeried_only_in_exact_requested_frame(self):
|
||||
ctx = context()
|
||||
wrong = doc(customer='客户乙')
|
||||
query = {'mode': 'query', 'releaseId': RELEASE, 'period': {'year': 2026},
|
||||
'filters': {'extraFilters': {'customerCode': ['客户甲']}}, 'view': 'overall'}
|
||||
self.assertEqual(plan_missing_queries([wrong], QUESTION, ctx), [query])
|
||||
self.assertTrue(coverage_gaps([wrong], QUESTION, ctx))
|
||||
self.assertEqual(plan_missing_queries([doc()], QUESTION, ctx), [])
|
||||
self.assertEqual(coverage_gaps([doc()], QUESTION, ctx), [])
|
||||
truncated = doc(); truncated['result']['trendTruncated'] = True
|
||||
self.assertEqual(plan_missing_queries([truncated], QUESTION, ctx), [])
|
||||
self.assertTrue(coverage_gaps([truncated], QUESTION, ctx))
|
||||
|
||||
def test_missing_trend_does_not_guess_period_or_drop_scope(self):
|
||||
for ctx in (context(period=None), context(period={'year': 2026, 'month': 9}),
|
||||
context(filters={'extraFilters': {'customerCode': ['客户甲']},
|
||||
'exclusions': {'product': ['X']}}),
|
||||
context(trendRequired=False), context(needsClarification=True)):
|
||||
with self.subTest(ctx=ctx):
|
||||
self.assertEqual(plan_missing_queries([], QUESTION, ctx), [])
|
||||
no_release = context(); no_release.pop('releaseId')
|
||||
self.assertEqual(plan_missing_queries([], QUESTION, no_release), [])
|
||||
|
||||
def test_runtime_fills_missing_trend_and_keeps_failed_reads_incomplete(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
import nao_quality_runtime as runtime
|
||||
ctx = context()
|
||||
request = {'actorId': 'actor', 'sessionId': 'session', 'releaseId': RELEASE,
|
||||
'question': QUESTION, 'context': ctx}
|
||||
for fails in (False, True):
|
||||
with self.subTest(fails=fails):
|
||||
docs = [doc(customer='客户乙')]
|
||||
seen = []
|
||||
def read(arguments):
|
||||
seen.append(arguments)
|
||||
if fails:
|
||||
raise RuntimeError('no data')
|
||||
evidence = doc()
|
||||
docs.append(evidence)
|
||||
return evidence
|
||||
with patch('nao_evidence_finalizer.evidence_summaries', side_effect=lambda *a, **k: list(docs)), \
|
||||
patch('nao_agent_task.plan_repairs', return_value=[]), \
|
||||
patch.object(runtime, 'DelegatedApi') as delegated, \
|
||||
patch('nao_quality_tools.QualityTools') as quality:
|
||||
delegated.return_value.login.return_value = MagicMock()
|
||||
quality.return_value.quality_data.side_effect = read
|
||||
runtime.complete_agent_evidence(task=ctx['_agentTask'], request=request,
|
||||
actor_root=Path('/offline/session'), access_path=Path('/offline/access'),
|
||||
manager=MagicMock(), connection=('offline', 'synthetic', 'm'),
|
||||
accumulator=runtime.ConversationAccumulator(), prior_files=frozenset(),
|
||||
cancel_event=threading.Event(), emit=lambda event: None)
|
||||
self.assertEqual(len(seen), 1)
|
||||
self.assertEqual(seen[0]['filters'], {'extraFilters': {'customerCode': ['客户甲']}})
|
||||
self.assertEqual(coverage_sufficient(docs, QUESTION, ctx), not fails)
|
||||
|
||||
def test_finalizer_includes_both_requested_stage_trends_even_when_model_omits_one(self):
|
||||
ctx = context(stages=['black', 'bright'])
|
||||
item = doc()
|
||||
def opener(request, timeout):
|
||||
answer = {'answerBlocks': [{'type': 'paragraph', 'factIds': ['F1']}],
|
||||
'evidenceSufficient': True, 'missingEvidence': []}
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
|
||||
'message': {'content': json.dumps(answer)}}]}).encode())
|
||||
result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question=QUESTION, evidence=[item], cancel_event=threading.Event(),
|
||||
context=ctx, opener=opener)
|
||||
self.assertIn('黑片合格率月度趋势', result['answer'])
|
||||
self.assertIn('亮片合格率月度趋势', result['answer'])
|
||||
self.assertEqual(result['selectedFactIds'], ['F1', 'F2'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,179 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
from analysis_visual_evidence import visual_result
|
||||
from analysis_visuals import build_visuals
|
||||
|
||||
|
||||
def evidence(result=None):
|
||||
result = result or {
|
||||
"kind": "quality_comparison",
|
||||
"releaseId": "r-1",
|
||||
"filterBinding": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
|
||||
"current": {
|
||||
"period": {"year": 2026, "month": 9},
|
||||
"compositeRate": 0.98,
|
||||
"black": {"rate": 0.97, "scrapWeight": 12.0},
|
||||
"bright": {"rate": 0.99, "scrapWeight": 5.0},
|
||||
},
|
||||
"comparison": {
|
||||
"period": {"year": 2026, "month": 8},
|
||||
"compositeRate": 0.96,
|
||||
"black": {"rate": 0.95, "scrapWeight": 10.0},
|
||||
"bright": {"rate": 0.98, "scrapWeight": 7.0},
|
||||
},
|
||||
"delta": {"available": True, "percentagePoints": 2.0},
|
||||
"stageRateChangesPP": {"black": 2.0, "bright": 1.0},
|
||||
}
|
||||
return {
|
||||
"tool": "quality_data",
|
||||
"mode": "compare",
|
||||
"releaseId": "r-1",
|
||||
"evidenceRef": "Q1234ABCD",
|
||||
"result": {"release": {"id": "r-1"}, **result},
|
||||
}
|
||||
|
||||
|
||||
def task(question, *, preference="auto", chart_type=None, kind="analysis", dimensions=None,
|
||||
stages=None, fields=None):
|
||||
return {
|
||||
"version": 1,
|
||||
"question": question,
|
||||
"goal": "compare formal quality evidence",
|
||||
"kind": kind,
|
||||
"dimensions": dimensions or [],
|
||||
"stages": stages or [],
|
||||
"fields": fields or [],
|
||||
"freshQuery": True,
|
||||
"visualPreference": preference,
|
||||
"visualType": chart_type,
|
||||
"responseStyle": "concise",
|
||||
}
|
||||
|
||||
|
||||
def context_for(question, **kwargs):
|
||||
return {
|
||||
"releaseId": "r-1",
|
||||
"filters": {"department": "电机部"},
|
||||
"_agentTask": task(question, **kwargs),
|
||||
}
|
||||
|
||||
|
||||
class AgentVisualTests(unittest.TestCase):
|
||||
def test_semantic_task_makes_chart_choice_independent_of_question_wording(self):
|
||||
questions = ["电机部近期情况如何", "帮我看看这段时间的表现"]
|
||||
outputs = [build_visuals([evidence()], question, "turn-semantic", context=context_for(
|
||||
question, dimensions=["department"], fields=["rate"])) for question in questions]
|
||||
self.assertEqual(outputs[0]["visualState"], "ready")
|
||||
self.assertEqual(outputs[0]["visuals"], outputs[1]["visuals"])
|
||||
self.assertEqual({chart["metric"] for chart in outputs[0]["visuals"]},
|
||||
{"composite_rate", "black_rate", "bright_rate"})
|
||||
|
||||
def test_none_overrides_explicit_chart_words(self):
|
||||
question = "画图比较两期综合合格率"
|
||||
output = build_visuals([evidence()], question, "turn-none",
|
||||
context=context_for(question, preference="none", fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "not_requested")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_required_without_evidence_reports_unavailable_and_invents_nothing(self):
|
||||
question = "检查重点产品"
|
||||
output = build_visuals([], question, "turn-required-empty",
|
||||
context=context_for(question, preference="required", fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "unavailable")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_wrong_question_task_is_ignored(self):
|
||||
question = "比较两个期间变化"
|
||||
stale = task("不要图,只要文字", preference="none")
|
||||
output = build_visuals([evidence()], question, "turn-stale",
|
||||
context={"releaseId": "r-1", "filters": {"department": "电机部"},
|
||||
"_agentTask": stale})
|
||||
self.assertEqual(output["visualState"], "ready")
|
||||
self.assertTrue(output["visuals"])
|
||||
|
||||
def test_semantic_scope_keeps_evidence_binding_and_requested_stage(self):
|
||||
question = "只看本部门黑片"
|
||||
output = build_visuals([evidence()], question, "turn-bound",
|
||||
context=context_for(question, dimensions=["department"], stages=["black"],
|
||||
fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "ready")
|
||||
self.assertEqual(len(output["visuals"]), 1)
|
||||
chart = output["visuals"][0]
|
||||
self.assertEqual(chart["stage"], "black")
|
||||
self.assertEqual(chart["scope"]["department"], "电机部")
|
||||
self.assertEqual(chart["evidenceRefs"], ["Q1234ABCD"])
|
||||
|
||||
def test_supported_semantic_style_is_applied_without_style_words(self):
|
||||
question = "请查看两期质量表现"
|
||||
output = build_visuals([evidence()], question, "turn-style",
|
||||
context=context_for(question, chart_type="column", fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "ready")
|
||||
self.assertTrue(output["visuals"])
|
||||
self.assertTrue(all(chart["type"] == "column" for chart in output["visuals"]))
|
||||
|
||||
def test_unsupported_for_evidence_style_has_no_fallback(self):
|
||||
question = "请查看两期质量表现"
|
||||
output = build_visuals([evidence()], question, "turn-style-unsafe",
|
||||
context=context_for(question, preference="required", chart_type="heatmap", fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "unavailable")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_auto_style_hint_cannot_hide_an_evidence_supported_chart(self):
|
||||
question='请查看两期质量表现'
|
||||
output=build_visuals([evidence()],question,'turn-auto-style',
|
||||
context=context_for(question,chart_type='heatmap',fields=['rate']))
|
||||
self.assertEqual(output['visualState'],'ready')
|
||||
self.assertTrue(output['visuals'])
|
||||
self.assertTrue(all(chart['type'] in ('bar','column') for chart in output['visuals']))
|
||||
|
||||
def test_auto_pareto_for_source_defects_preserves_weights_without_inventing_shares(self):
|
||||
from test_analysis_visuals import fine_field
|
||||
question='帮我看下上月黑片缺陷TOP'
|
||||
keys=['责废缺角重量','料废缺角重量']
|
||||
result={'kind':'quality_fine_defects','releaseId':'r-1','dimension':'defect',
|
||||
'grain':'source_defect_weight','stage':'black','filters':{},'period':{'year':2026,'month':9},
|
||||
'coverage':{'available':True,'sourceRows':2},'sourceRows':2,
|
||||
'sourceFields':[fine_field(key,'责废' if key.startswith('责废') else '料废') for key in keys],
|
||||
'rows':[{'key':key,'label':key,'category':'责废' if key.startswith('责废') else '料废','weight':weight}
|
||||
for key,weight in zip(keys,[17513.162,13949.63])],
|
||||
'totalRows':2,'truncated':False,'rankingComplete':True}
|
||||
context={'releaseId':'r-1','filters':{},'_agentTask':task(question,dimensions=['defect'],stages=['black'],chart_type='pareto')}
|
||||
output=build_visuals([evidence(result)],question,'turn-auto-defect',context=context)
|
||||
self.assertEqual(output['visualState'],'ready')
|
||||
chart=output['visuals'][0]
|
||||
self.assertEqual(chart['type'],'bar')
|
||||
self.assertEqual(chart['series'][0]['values'],[17513.162,13949.63])
|
||||
self.assertFalse(any(series.get('unit')=='%' for series in chart['series']))
|
||||
context['_agentTask']['visualPreference']='required'
|
||||
self.assertEqual(build_visuals([evidence(result)],question,'turn-required-defect',context=context)['visuals'],[])
|
||||
|
||||
def test_auto_does_not_chart_a_single_number_metric(self):
|
||||
question = "本期综合合格率是多少"
|
||||
metric = evidence({
|
||||
"kind": "quality_metrics", "releaseId": "r-1", "view": "overall",
|
||||
"filters": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
|
||||
"period": {"year": 2026, "month": 9},
|
||||
"summary": {"compositeRate": 0.98},
|
||||
})
|
||||
for kind in ("metrics", "restate", "explain"):
|
||||
with self.subTest(kind=kind):
|
||||
output = build_visuals([metric], question, f"turn-single-metric-{kind}",
|
||||
context=context_for(question, kind=kind, fields=["rate"]))
|
||||
self.assertEqual(output["visualState"], "not_requested")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_evidence_adapter_forwards_the_validated_task_unchanged(self):
|
||||
question = "看两期表现"
|
||||
request_context = context_for(question, fields=["rate"])
|
||||
with patch("analysis_visual_evidence.load_visual_evidence", return_value=[evidence()]), \
|
||||
patch("analysis_visuals.build_visuals", return_value={"visuals": []}) as builder:
|
||||
visual_result(Path("."), "r-1", question, "turn-forward", context=request_context)
|
||||
self.assertIs(builder.call_args.kwargs["context"]["_agentTask"], request_context["_agentTask"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,76 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from analysis_visual_evidence import load_visual_evidence, visual_result
|
||||
|
||||
|
||||
class VisualEvidenceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
self.path = self.root / ('nao-quality-' + 'a' * 32 + '.json')
|
||||
self.doc = {'tool': 'quality_data', 'mode': 'compare', 'releaseId': 'release',
|
||||
'evidenceRef': 'QAAAAAAAA', 'result': {'releaseId': 'release', 'kind': 'quality_comparison'}}
|
||||
self.save()
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def save(self):
|
||||
self.path.write_text(json.dumps(self.doc))
|
||||
self.path.chmod(0o600)
|
||||
|
||||
def test_full_document_and_only_this_turn(self):
|
||||
self.assertEqual(load_visual_evidence(self.root, 'release'), [self.doc])
|
||||
self.assertEqual(load_visual_evidence(self.root, 'release', exclude_files={self.path.name}), [])
|
||||
|
||||
def test_release_tool_and_reference_mismatch_fail_closed(self):
|
||||
for key, value in [('releaseId', 'other'), ('tool', 'arbitrary'), ('evidenceRef', 'forged')]:
|
||||
with self.subTest(key=key):
|
||||
original = self.doc[key]
|
||||
self.doc[key] = value
|
||||
self.save()
|
||||
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release')
|
||||
self.doc[key] = original
|
||||
|
||||
def test_symlinks_and_shared_files_rejected(self):
|
||||
self.path.chmod(0o644)
|
||||
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release')
|
||||
self.path.unlink()
|
||||
outside = self.root / 'outside.json'
|
||||
outside.write_text(json.dumps(self.doc))
|
||||
self.path.symlink_to(outside)
|
||||
with self.assertRaises(OSError): load_visual_evidence(self.root, 'release')
|
||||
|
||||
def test_cancel_and_deadline_are_bounded(self):
|
||||
cancel = threading.Event(); cancel.set()
|
||||
self.assertEqual(load_visual_evidence(self.root, 'release', cancel_event=cancel), [])
|
||||
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release', budget_seconds=-1)
|
||||
|
||||
def test_large_file_and_failure_never_change_business_status(self):
|
||||
with patch('analysis_visual_evidence.FILE_LIMIT', 10):
|
||||
self.assertEqual(visual_result(self.root, 'release', '比较', 'turn'),
|
||||
{'visualSchemaVersion': 1, 'visualState': 'failed', 'visuals': []})
|
||||
|
||||
def test_feature_switch_does_not_read_data(self):
|
||||
with patch.dict(os.environ, {'QUALITY_ANALYSIS_VISUALS_ENABLED': 'false'}):
|
||||
result = visual_result(Path('/does-not-exist'), 'release', '比较', 'turn')
|
||||
self.assertEqual(result['visualState'], 'not_requested')
|
||||
|
||||
def test_turn_context_is_forwarded_without_reinterpreting_question(self):
|
||||
import inspect
|
||||
context={'year':2026,'month':9,'group':'组A'}
|
||||
kwargs={'context':context} if 'context' in inspect.signature(visual_result).parameters else {}
|
||||
with patch('analysis_visuals.build_visuals',return_value={'visuals':[]}) as build:
|
||||
visual_result(self.root,'release','比较','turn',**kwargs)
|
||||
self.assertEqual(build.call_args.kwargs.get('context'),context)
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,911 @@
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from scripts.analysis_visuals import MAX_BODY_BYTES, build_visuals
|
||||
|
||||
|
||||
def evidence(result, *, ref="Q1234ABCD", release_id="r-1"):
|
||||
return {
|
||||
"tool": "quality_data",
|
||||
"mode": "compare",
|
||||
"releaseId": release_id,
|
||||
"evidenceRef": ref,
|
||||
"result": {
|
||||
"release": {"id": release_id},
|
||||
"releaseId": release_id,
|
||||
**result,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def comparison_result(**overrides):
|
||||
result = {
|
||||
"kind": "quality_comparison",
|
||||
"releaseId": "r-1",
|
||||
"filterBinding": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
|
||||
"current": {
|
||||
"period": {"year": 2026, "month": 9},
|
||||
"compositeRate": 0.98,
|
||||
"black": {"rate": 0.97, "scrapWeight": 12.0},
|
||||
"bright": {"rate": 0.99, "scrapWeight": 5.0},
|
||||
},
|
||||
"comparison": {
|
||||
"period": {"year": 2026, "month": 8},
|
||||
"compositeRate": 0.96,
|
||||
"black": {"rate": 0.95, "scrapWeight": 10.0},
|
||||
"bright": {"rate": 0.98, "scrapWeight": 7.0},
|
||||
},
|
||||
"delta": {"available": True, "percentagePoints": 2.0},
|
||||
"stageRateChangesPP": {"black": 2.0, "bright": 1.0},
|
||||
}
|
||||
result.update(overrides)
|
||||
return result
|
||||
|
||||
|
||||
def decomposed_comparison_result(**overrides):
|
||||
result = comparison_result(
|
||||
current={
|
||||
"period": {"year": 2026, "month": 9}, "compositeRate": 0.9556,
|
||||
"black": {"rate": 0.947, "scrapWeight": 12.0,
|
||||
"defects": [{"name": "责废", "weight": 12.0}, {"name": "料废", "weight": 8.0}, {"name": "外废", "weight": 2.0}]},
|
||||
"bright": {"rate": 0.978, "scrapWeight": 5.0,
|
||||
"defects": [{"name": "镀废", "weight": 5.0}, {"name": "材料废", "weight": 3.0},
|
||||
{"name": "加工废", "weight": 2.0}, {"name": "扩散废", "weight": 1.0}]},
|
||||
},
|
||||
comparison={
|
||||
"period": {"year": 2026, "month": 8}, "compositeRate": 0.96,
|
||||
"black": {"rate": 0.95, "scrapWeight": 10.0,
|
||||
"defects": [{"name": "责废", "weight": 10.0}, {"name": "料废", "weight": 7.0}, {"name": "外废", "weight": 1.0}]},
|
||||
"bright": {"rate": 0.98, "scrapWeight": 7.0,
|
||||
"defects": [{"name": "镀废", "weight": 6.0}, {"name": "材料废", "weight": 3.5},
|
||||
{"name": "加工废", "weight": 2.0}, {"name": "扩散废", "weight": 1.5}]},
|
||||
},
|
||||
delta={"available": True, "rate": -0.0044, "percentagePoints": -0.44},
|
||||
stageRateChangesPP={"black": -0.3, "bright": -0.2},
|
||||
stageDecomposition={"status": "available", "blackContributionToCompositePP": -0.3,
|
||||
"brightContributionToCompositePP": -0.14, "sumPP": -0.44},
|
||||
)
|
||||
result.update(overrides)
|
||||
return result
|
||||
|
||||
|
||||
def changes_result(**overrides):
|
||||
result = {
|
||||
"kind": "quality_changes",
|
||||
"releaseId": "r-1",
|
||||
"filters": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
|
||||
"dimension": "product",
|
||||
"stage": "black",
|
||||
"category": None,
|
||||
"currentPeriod": {"year": 2026, "month": 9},
|
||||
"comparisonPeriod": {"year": 2026, "month": 8},
|
||||
"sort": "scrap_delta",
|
||||
"rows": [
|
||||
{"rank": 1, "key": "p-1", "label": "规格一", "scrapDelta": 4.0, "current": {"scrapWeight": 8.0}},
|
||||
{"rank": 2, "key": "p-2", "label": "规格二", "scrapDelta": -2.0, "current": {"scrapWeight": 3.0}},
|
||||
],
|
||||
"totalRows": 8,
|
||||
"returnedRows": 2,
|
||||
"truncated": True,
|
||||
}
|
||||
result.update(overrides)
|
||||
return result
|
||||
|
||||
|
||||
def metric_result(period, composite_rate, *, release_id="r-1", filters=None):
|
||||
return {
|
||||
"kind": "quality_metrics",
|
||||
"releaseId": release_id,
|
||||
"period": period,
|
||||
"filters": filters or {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
|
||||
"view": "overall",
|
||||
"summary": {"compositeRate": composite_rate, "black": {"rate": composite_rate - 0.01},
|
||||
"bright": {"rate": composite_rate + 0.01}},
|
||||
}
|
||||
|
||||
|
||||
def fine_field(key, category):
|
||||
return {"key": key, "label": key, "category": category, "sourceField": key}
|
||||
|
||||
|
||||
class AnalysisVisualsTest(unittest.TestCase):
|
||||
def test_frozen_followup_scope_selects_same_comparison_as_text(self):
|
||||
import inspect
|
||||
global_result=comparison_result(filterBinding={})
|
||||
local=comparison_result(filterBinding={'group':'组A'})
|
||||
local['current']['compositeRate']=.80
|
||||
docs=[evidence(global_result,ref='Q00000001'),evidence(local,ref='Q00000002')]
|
||||
kwargs={'context':{'year':2026,'month':9,'group':'组A'}} if 'context' in inspect.signature(build_visuals).parameters else {}
|
||||
charts=build_visuals(docs,'比较本期综合率变化','turn-bound-followup',**kwargs)['visuals']
|
||||
overview=[c for c in charts if c.get('metric')=='composite_rate']
|
||||
self.assertEqual(len(overview),1)
|
||||
self.assertEqual(overview[0]['scope'].get('group'),'组A')
|
||||
self.assertEqual(overview[0]['series'][0]['values'],[.96,.80])
|
||||
|
||||
def test_frozen_period_excludes_other_period_comparison_and_defects(self):
|
||||
import inspect
|
||||
selected=comparison_result(filterBinding={})
|
||||
older=comparison_result(filterBinding={})
|
||||
older['current']['period']={'year':2026,'month':7};older['comparison']['period']={'year':2026,'month':6}
|
||||
kwargs={'context':{'year':2026,'month':9}} if 'context' in inspect.signature(build_visuals).parameters else {}
|
||||
charts=build_visuals([evidence(selected,ref='Q00000001'),evidence(older,ref='Q00000002')],
|
||||
'比较综合率变化','turn-bound-period',**kwargs)['visuals']
|
||||
self.assertTrue(charts)
|
||||
self.assertTrue(all(c.get('period',{}).get('month')==9 for c in charts))
|
||||
|
||||
def test_available_requested_stage_is_drawn_when_other_stage_blocks_composite(self):
|
||||
result=comparison_result(delta={'available':False,'reason':'缺少黑片来源'},stageRateChangesPP={'black':None,'bright':1})
|
||||
result['current']['black']['rate']=1
|
||||
charts=build_visuals([evidence(result)],'比较亮片合格率变化','turn-stage-only')['visuals']
|
||||
self.assertEqual([c['metric'] for c in charts],['bright_rate'])
|
||||
scoped=build_visuals([evidence(result)],'比较本期变化','turn-stage-context',context={'year':2026,'month':9,'stage':'bright'})['visuals']
|
||||
self.assertEqual([c['metric'] for c in scoped],['bright_rate'])
|
||||
|
||||
def test_formal_comparison_emits_safe_single_metric_chart_with_zero_and_null(self):
|
||||
result = comparison_result(
|
||||
current={"period": {"year": 2026, "month": 9}, "compositeRate": 0.0,
|
||||
"black": {"rate": None}, "bright": {"rate": 0.99}},
|
||||
comparison={"period": {"year": 2026, "month": 8}, "compositeRate": None,
|
||||
"black": {"rate": 0.95}, "bright": {"rate": 0.98}},
|
||||
stageRateChangesPP={"black": None, "bright": 1.0},
|
||||
)
|
||||
output = build_visuals([evidence(result)], "请比较8月和9月综合合格率", "turn-01")
|
||||
self.assertEqual(output["visualState"], "ready")
|
||||
chart = output["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "composite_rate")
|
||||
self.assertEqual(chart["unit"], "ratio")
|
||||
self.assertEqual(chart["series"][0]["values"], [None, 0.0])
|
||||
self.assertNotIn("stage", chart)
|
||||
self.assertEqual(chart["scope"]["releaseId"], "r-1")
|
||||
self.assertEqual(chart["originTurnId"], "turn-01")
|
||||
self.assertEqual(chart["type"], "column")
|
||||
|
||||
def test_comparison_skips_mismatched_period_or_release(self):
|
||||
bad_period = comparison_result()
|
||||
bad_period["current"]["period"] = {"year": 2026, "month": 8}
|
||||
self.assertEqual(build_visuals([evidence(bad_period)], "比较两个期间变化", "turn-01")["visualState"], "not_requested")
|
||||
self.assertEqual(build_visuals([evidence(comparison_result(), release_id="other")], "比较两个期间变化", "turn-01")["visualState"], "not_requested")
|
||||
|
||||
def test_invalid_rate_is_never_clamped_or_drawn(self):
|
||||
result = comparison_result()
|
||||
result["current"]["compositeRate"] = 1.01
|
||||
output = build_visuals([evidence(result)], "比较综合合格率变化", "turn-01")
|
||||
self.assertEqual(output["visualState"], "not_requested")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_broad_cause_analysis_adds_three_rates_math_waterfall_and_both_category_deltas(self):
|
||||
output = build_visuals([evidence(decomposed_comparison_result())], "分析整体合格率下降原因", "turn-broad-chapters")
|
||||
charts = output["visuals"]
|
||||
self.assertEqual(output["visualState"], "ready")
|
||||
by_section = {}
|
||||
for chart in charts:
|
||||
by_section.setdefault(chart["sectionKey"], []).append(chart)
|
||||
self.assertEqual(len(by_section["overview"]), 2)
|
||||
self.assertEqual({chart["metric"] for chart in by_section["stage"]}, {"black_rate", "bright_rate"})
|
||||
waterfall = next(chart for chart in charts if chart["type"] == "waterfall")
|
||||
self.assertEqual(waterfall["metric"], "composite_impact_pp")
|
||||
self.assertEqual(waterfall["series"][0]["values"], [-0.3, -0.14, -0.44])
|
||||
self.assertEqual(waterfall["requiredFields"], [
|
||||
"black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"
|
||||
])
|
||||
self.assertIn("拆解", waterfall["note"])
|
||||
self.assertNotIn("数学", waterfall["note"])
|
||||
self.assertNotIn("生产根因", waterfall["title"])
|
||||
self.assertEqual({chart["sectionKey"] for chart in charts if chart["sectionKey"].startswith("category_")},
|
||||
{"category_black", "category_bright"})
|
||||
black_categories = next(chart for chart in charts if chart["sectionKey"] == "category_black")
|
||||
self.assertEqual(black_categories["series"][0]["values"], [2.0, 1.0, 1.0])
|
||||
self.assertEqual(black_categories["dimension"], "category")
|
||||
|
||||
def test_followup_product_comparisons_do_not_crowd_out_main_chapters(self):
|
||||
docs = [evidence(decomposed_comparison_result(filterBinding={}), ref="Q00000001")]
|
||||
for index in range(6):
|
||||
docs.append(evidence(decomposed_comparison_result(filterBinding={"product": f"规格-{index}"}), ref=f"Q{index+2:08X}"))
|
||||
for index, dimension in enumerate(("department", "group", "product")):
|
||||
docs.append(evidence(changes_result(filters={}, dimension=dimension), ref=f"Q{index+20:08X}"))
|
||||
for index, stage in enumerate(("black", "bright")):
|
||||
docs.append(evidence(changes_result(filters={}, dimension="department", stage=stage), ref=f"Q{index+30:08X}"))
|
||||
field, category = ("责废其他重量", "责废") if stage == "black" else ("加工磕边重量", "加工废")
|
||||
docs.append(evidence({"kind":"quality_fine_defects","dimension":"defect","grain":"source_defect_weight","stage":stage,"filters":{},
|
||||
"period":{"year":2026,"month":9},"comparisonPeriod":{"year":2026,"month":8},
|
||||
"coverage":{"available":True},"comparisonCoverage":{"available":True},
|
||||
"sourceFields":[fine_field(field,category)],"comparisonSourceFields":[fine_field(field,category)],
|
||||
"pairedRows":[{"key":field,"current":{"label":field,"category":category,"weight":3},"comparison":{"label":field,"category":category,"weight":2},"deltaWeight":1,"deltaAvailable":True}],
|
||||
"pairedTotalRows":1,"pairedTruncated":False,"pairedRankingComplete":True,"sort":"scrap_delta"},ref=f"Q{index+40:08X}"))
|
||||
charts = build_visuals(docs, "分析整体变化,覆盖产业部、业务分组、重点产品与报废类别", "turn-balanced")["visuals"]
|
||||
self.assertEqual(len(charts), 12)
|
||||
self.assertTrue({"department", "group", "product", "category_black", "category_bright", "defect_black", "defect_bright"}.issubset({c["sectionKey"] for c in charts}))
|
||||
self.assertFalse(charts[0]["scope"].get("product"))
|
||||
self.assertEqual({c["metric"] for c in charts if c["sectionKey"] == "stage" and not c["scope"].get("product")}, {"black_rate", "bright_rate"})
|
||||
|
||||
def test_waterfall_requires_exact_finite_decomposition_that_matches_service_delta(self):
|
||||
valid = decomposed_comparison_result()
|
||||
bad_cases = []
|
||||
mismatch_delta = json.loads(json.dumps(valid))
|
||||
mismatch_delta["delta"]["percentagePoints"] = -0.5
|
||||
bad_cases.append(mismatch_delta)
|
||||
mismatch_sum = json.loads(json.dumps(valid))
|
||||
mismatch_sum["stageDecomposition"]["sumPP"] = -0.5
|
||||
bad_cases.append(mismatch_sum)
|
||||
missing_stage = json.loads(json.dumps(valid))
|
||||
missing_stage["stageDecomposition"]["brightContributionToCompositePP"] = None
|
||||
bad_cases.append(missing_stage)
|
||||
non_finite = json.loads(json.dumps(valid))
|
||||
non_finite["stageDecomposition"]["blackContributionToCompositePP"] = float("inf")
|
||||
bad_cases.append(non_finite)
|
||||
for index, result in enumerate(bad_cases):
|
||||
output = build_visuals([evidence(result)], "画瀑布图解释整体合格率下降原因", f"turn-waterfall-invalid-{index}")
|
||||
self.assertEqual(output["visualState"], "unavailable")
|
||||
self.assertEqual(output["visuals"], [])
|
||||
|
||||
def test_formal_category_delta_requires_matching_complete_category_sets(self):
|
||||
result = decomposed_comparison_result()
|
||||
chart = next(chart for chart in build_visuals(
|
||||
[evidence(result)], "分析两阶段正式报废类别增减", "turn-category-delta"
|
||||
)["visuals"] if chart["sectionKey"] == "category_bright")
|
||||
self.assertEqual(chart["metric"], "source_defect_delta_weight")
|
||||
self.assertEqual(chart["stage"], "bright")
|
||||
self.assertEqual(chart["series"][0]["values"], [-1.0, -0.5, 0.0, -0.5])
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 4, "returnedRows": 4, "truncated": False})
|
||||
|
||||
partial = json.loads(json.dumps(result))
|
||||
partial["current"]["bright"]["defects"].pop()
|
||||
output = build_visuals([evidence(partial)], "分析两阶段正式报废类别增减", "turn-category-delta-partial")
|
||||
self.assertFalse(any(chart.get("sectionKey") == "category_bright" for chart in output["visuals"]))
|
||||
|
||||
zero_only = json.loads(json.dumps(result))
|
||||
for side in ("current", "comparison"):
|
||||
for item in zero_only[side]["black"]["defects"]:
|
||||
item["weight"] = 0
|
||||
output = build_visuals([evidence(zero_only)], "分析黑片正式报废类别增减", "turn-category-delta-zero")
|
||||
self.assertFalse(any(chart.get("sectionKey") == "category_black" for chart in output["visuals"]))
|
||||
|
||||
def test_scoped_defect_charts_keep_separate_chapter_scope_from_global_chart(self):
|
||||
def fine_result(filters):
|
||||
return {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "black", "filters": filters,
|
||||
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
|
||||
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
|
||||
"sourceFields": [fine_field("责废其他重量", "责废")],
|
||||
"comparisonSourceFields": [fine_field("责废其他重量", "责废")],
|
||||
"pairedRows": [{
|
||||
"key": "责废其他重量",
|
||||
"current": {"label": "责废其他重量", "category": "责废", "weight": 3.0},
|
||||
"comparison": {"label": "责废其他重量", "category": "责废", "weight": 2.0},
|
||||
"deltaWeight": 1.0, "deltaAvailable": True,
|
||||
}],
|
||||
"sort": "scrap_delta", "pairedTotalRows": 1, "pairedTruncated": False,
|
||||
"pairedRankingComplete": True,
|
||||
}
|
||||
|
||||
docs = [
|
||||
evidence(fine_result({"extraFilters": {}}), ref="Q00000031"),
|
||||
evidence(fine_result({"group": "汽车交通-国内", "extraFilters": {}}), ref="Q00000032"),
|
||||
]
|
||||
charts = build_visuals(docs, "画图看黑片来源具体缺陷增减", "turn-scoped-defects")["visuals"]
|
||||
self.assertEqual(len(charts), 2)
|
||||
self.assertEqual({chart["sectionKey"] for chart in charts}, {"defect_black"})
|
||||
self.assertEqual({chart["dimension"] for chart in charts}, {"defect"})
|
||||
self.assertEqual({chart["scope"].get("group") for chart in charts}, {None, "汽车交通-国内"})
|
||||
self.assertEqual(len({chart["id"] for chart in charts}), 2)
|
||||
scoped_chart = next(chart for chart in charts if chart["scope"].get("group"))
|
||||
self.assertIn("范围 业务分组:汽车交通-国内", scoped_chart["subtitle"])
|
||||
|
||||
many_scopes = [evidence(fine_result({"group": f"业务组-{index}", "extraFilters": {}}),
|
||||
ref=f"Q{index:08X}") for index in range(13)]
|
||||
bounded = build_visuals(many_scopes, "画图看黑片来源具体缺陷增减", "turn-12-limit")
|
||||
self.assertEqual(len(bounded["visuals"]), 12)
|
||||
self.assertEqual(len({chart["id"] for chart in bounded["visuals"]}), 12)
|
||||
self.assertLess(len(json.dumps(bounded, ensure_ascii=False).encode("utf-8")), MAX_BODY_BYTES)
|
||||
|
||||
def test_change_chart_preserves_signed_values_and_returned_order(self):
|
||||
rows = changes_result()["rows"]
|
||||
rows[0]["scrapDelta"] = -1.0
|
||||
rows[1]["scrapDelta"] = 5.0
|
||||
output = build_visuals([evidence(changes_result(rows=rows))], "画图比较产品报废增量", "turn-02")
|
||||
chart = output["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "scrap_delta")
|
||||
self.assertEqual(chart["stage"], "black")
|
||||
self.assertEqual(chart["series"][0]["values"], [-1.0, 5.0])
|
||||
self.assertEqual([row["key"] for row in chart["categories"]], ["p-1", "p-2"])
|
||||
self.assertTrue(chart["coverage"]["truncated"])
|
||||
self.assertEqual(chart["coverage"]["totalRows"], 8)
|
||||
|
||||
def test_rate_drop_is_signed_and_converted_to_percentage_points(self):
|
||||
result = changes_result(sort="rate_drop", rows=[
|
||||
{"rank": 1, "key": "p-1", "label": "规格一", "rateDrop": -0.025},
|
||||
{"rank": 2, "key": "p-2", "label": "规格二", "rateDrop": 0.04},
|
||||
])
|
||||
chart = build_visuals([evidence(result)], "画图比较产品合格率降幅", "turn-03")["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "rate_drop_pp")
|
||||
self.assertEqual(chart["unit"], "pp")
|
||||
self.assertEqual(chart["series"][0]["values"], [-2.5, 4.0])
|
||||
self.assertIn("率降幅线索", chart["title"])
|
||||
self.assertIn("未纳入投入规模", chart["note"])
|
||||
|
||||
def test_broad_product_focus_omits_rate_only_ranking_chart(self):
|
||||
rate = changes_result(stage=None, sort="rate_drop", returnedRows=1, rows=[
|
||||
{"rank": 1, "key": "tiny", "label": "小量规格", "rateDrop": 0.9943,
|
||||
"scrapDelta": 6.74, "current": {"blackInputWeight": 7.1, "brightInputWeight": 39.66}},
|
||||
])
|
||||
scrap = changes_result(stage=None, sort="scrap_delta", returnedRows=1, rows=[
|
||||
{"rank": 1, "key": "large", "label": "报废增量规格", "scrapDelta": 758.76},
|
||||
])
|
||||
charts = build_visuals(
|
||||
[evidence(rate, ref="Q00000001"), evidence(scrap, ref="Q00000002")],
|
||||
"分析下近期要关注的重点产品", "turn-product-focus",
|
||||
)["visuals"]
|
||||
self.assertEqual([chart["metric"] for chart in charts], ["scrap_delta"])
|
||||
raw_only = build_visuals(
|
||||
[evidence(rate)], "分析下近期要关注的重点产品", "turn-product-focus-raw-only",
|
||||
)
|
||||
self.assertEqual(raw_only["visualState"], "not_requested")
|
||||
self.assertEqual(raw_only["visuals"], [])
|
||||
mixed = build_visuals(
|
||||
[evidence(rate)], "分析重点产品的合格率下降和报废占比", "turn-product-focus-mixed",
|
||||
)
|
||||
self.assertEqual(mixed["visuals"], [])
|
||||
|
||||
def test_rank_metadata_must_match_order_and_zero_is_preserved(self):
|
||||
result = changes_result(rows=[
|
||||
{"rank": 2, "key": "p-1", "label": "规格一", "scrapDelta": 0.0},
|
||||
{"rank": 1, "key": "p-2", "label": "规格二", "scrapDelta": 3.0},
|
||||
])
|
||||
self.assertEqual(build_visuals([evidence(result)], "画图比较产品报废增量", "turn-04")["visualState"], "unavailable")
|
||||
|
||||
def test_change_chart_requires_stage_and_matching_filter_shape(self):
|
||||
no_stage = changes_result(stage=None)
|
||||
output = build_visuals([evidence(no_stage)], "画图比较产品报废增量", "turn-05")
|
||||
chart = output["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "scrap_delta")
|
||||
self.assertNotIn("stage", chart)
|
||||
self.assertNotIn("stage", chart["scope"])
|
||||
self.assertEqual(chart["requiredFields"], ["black_scrap_weight", "bright_scrap_weight"])
|
||||
secret_filter = changes_result(filters={"department": "电机部", "apiKey": "secret-value"})
|
||||
self.assertEqual(build_visuals([evidence(secret_filter)], "画图比较产品报废增量", "turn-05")["visualState"], "unavailable")
|
||||
|
||||
def test_overall_rate_drop_uses_canonical_two_stage_fields(self):
|
||||
result = changes_result(stage=None, sort="rate_drop", rows=[
|
||||
{"rank": 1, "key": "p-1", "label": "规格一", "rateDrop": 0.02},
|
||||
{"rank": 2, "key": "p-2", "label": "规格二", "rateDrop": -0.01},
|
||||
])
|
||||
chart = build_visuals([evidence(result)], "画产品综合合格率变化", "turn-05b")["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "rate_drop_pp")
|
||||
self.assertEqual(chart["requiredFields"], [
|
||||
"black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"
|
||||
])
|
||||
self.assertEqual(chart["series"][0]["values"], [2.0, -1.0])
|
||||
|
||||
def test_quality_top_uses_ranked_weights_and_never_share(self):
|
||||
result = {
|
||||
"kind": "quality_top", "releaseId": "r-1", "stage": "black", "dimension": "product",
|
||||
"period": {"year": 2026, "month": 9},
|
||||
"filters": {"department": "电机部", "extraFilters": {}},
|
||||
"rankingCoverage": {"groups": 12, "complete": False},
|
||||
"rows": [
|
||||
{"rank": index + 1, "key": f"p-{index}", "label": f"产品{index}",
|
||||
"weight": index + 0.5, "share": 0.99 - index * 0.01}
|
||||
for index in range(12)
|
||||
],
|
||||
}
|
||||
chart = build_visuals([evidence(result)], "哪些产品报废最多", "turn-top") ["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "current_scrap_weight")
|
||||
self.assertEqual(chart["requiredFields"], ["black_scrap_weight"])
|
||||
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(10)])
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
|
||||
self.assertNotIn("share", json.dumps(chart, ensure_ascii=False))
|
||||
|
||||
def test_quality_top_categories_require_stage_canonical_source_fields(self):
|
||||
result = {
|
||||
"kind": "quality_top", "releaseId": "r-1", "stage": "bright", "dimension": "category",
|
||||
"period": {"year": 2026, "month": 9}, "filters": {"extraFilters": {}},
|
||||
"rankingCoverage": {"groups": 3, "complete": True},
|
||||
"cumulativeThresholdApplied": False,
|
||||
"rows": [
|
||||
{"rank": 1, "key": "加工废", "label": "加工废", "weight": 3.0},
|
||||
{"rank": 2, "key": "材料废", "label": "材料废", "weight": 2.0},
|
||||
{"rank": 3, "key": "扩散废", "label": "扩散废", "weight": 1.0},
|
||||
],
|
||||
}
|
||||
chart = build_visuals([evidence(result)], "分析亮片报废类别排名", "turn-cat")["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "source_defect_weight")
|
||||
self.assertEqual([item["key"] for item in chart["categories"]], [
|
||||
"bright_processing_scrap", "bright_material_scrap", "bright_diffusion_scrap"
|
||||
])
|
||||
self.assertEqual(chart["requiredFields"], [
|
||||
"bright_diffusion_scrap", "bright_material_scrap", "bright_processing_scrap"
|
||||
])
|
||||
result["rows"][0]["label"] = "VCM总计"
|
||||
self.assertEqual(build_visuals([evidence(result)], "分析亮片报废类别排名", "turn-cat")["visualState"], "not_requested")
|
||||
|
||||
def test_real_category_top_shape_without_ranking_coverage_requires_endpoint_denominator_proof(self):
|
||||
denominator = 7118.885021
|
||||
weights = [3421.516018, 2812.668772, 868.241231, 16.459]
|
||||
labels = ["加工废", "镀废", "材料废", "扩散废"]
|
||||
result = {
|
||||
"kind": "quality_top", "releaseId": "r-1", "stage": "bright", "dimension": "category",
|
||||
"period": {"year": 2026, "month": 9},
|
||||
"filters": {"department": "工业电机产业部", "includeUnassigned": False},
|
||||
"rankingCoverage": None,
|
||||
"denominator": denominator,
|
||||
"shareBasis": "所选筛选条件下选中原报废分类的重量总和",
|
||||
"cumulativeThresholdApplied": False,
|
||||
"rows": [
|
||||
{"rank": index + 1, "key": label, "label": label, "weight": weight,
|
||||
"denominator": denominator, "share": weight / denominator}
|
||||
for index, (label, weight) in enumerate(zip(labels, weights))
|
||||
],
|
||||
}
|
||||
doc = evidence(result)
|
||||
doc["apiSource"] = {"calls": [
|
||||
{"method": "GET", "path": "/api/releases", "params": {}},
|
||||
{"method": "GET", "path": "/api/analytics/top", "params": {
|
||||
"releaseId": "r-1", "stage": "bright", "dimension": "category", "limit": 10,
|
||||
"year": 2026, "month": 9, "department": "工业电机产业部", "includeUnassigned": False,
|
||||
}},
|
||||
]}
|
||||
|
||||
donut = build_visuals([doc], "亮片已列报废类别用环状图展示构成", "turn-real-category-donut")["visuals"][0]
|
||||
self.assertEqual(donut["type"], "donut")
|
||||
self.assertEqual(donut["series"][0]["values"], weights)
|
||||
self.assertEqual(donut["coverage"], {
|
||||
"totalRows": 4, "returnedRows": 4, "truncated": False, "shareBasis": "displayed_categories"
|
||||
})
|
||||
|
||||
pareto = build_visuals([doc], "亮片报废类别画柏拉图", "turn-real-category-pareto")["visuals"][0]
|
||||
self.assertEqual(pareto["type"], "pareto")
|
||||
self.assertEqual(pareto["series"][0]["values"], weights)
|
||||
|
||||
subset = json.loads(json.dumps(result))
|
||||
subset["rows"] = result["rows"][:3]
|
||||
subset["denominator"] = sum(weights[:3])
|
||||
for row in subset["rows"]:
|
||||
row["denominator"] = subset["denominator"]
|
||||
row["share"] = row["weight"] / subset["denominator"]
|
||||
subset_doc = evidence(subset)
|
||||
subset_doc["apiSource"] = doc["apiSource"]
|
||||
self.assertEqual(build_visuals([subset_doc], "亮片类别环状构成", "turn-authorized-subset")["visualState"], "ready")
|
||||
|
||||
invalid_cases = []
|
||||
low_limit = json.loads(json.dumps(doc))
|
||||
low_limit["apiSource"]["calls"][1]["params"]["limit"] = 3
|
||||
invalid_cases.append(low_limit)
|
||||
wrong_denominator = json.loads(json.dumps(doc))
|
||||
wrong_denominator["result"]["denominator"] += 1
|
||||
invalid_cases.append(wrong_denominator)
|
||||
threshold = json.loads(json.dumps(doc))
|
||||
threshold["result"]["cumulativeThresholdApplied"] = True
|
||||
invalid_cases.append(threshold)
|
||||
narrowed = json.loads(json.dumps(doc))
|
||||
narrowed["apiSource"]["calls"][1]["params"]["category"] = "加工废"
|
||||
invalid_cases.append(narrowed)
|
||||
mismatched_filter = json.loads(json.dumps(doc))
|
||||
mismatched_filter["apiSource"]["calls"][1]["params"]["department"] = "其他部门"
|
||||
invalid_cases.append(mismatched_filter)
|
||||
for index, invalid_doc in enumerate(invalid_cases):
|
||||
self.assertEqual(build_visuals([invalid_doc], "亮片类别环状构成", f"turn-category-invalid-{index}")["visualState"], "unavailable")
|
||||
|
||||
def test_candidate_priority_keeps_comparison_product_and_fine_defect_charts_separate(self):
|
||||
comparison = evidence(comparison_result(), ref="Q00000011")
|
||||
changes = evidence(changes_result(stage=None), ref="Q00000012")
|
||||
fine_result = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "bright", "filters": {"department": "其他部门"},
|
||||
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
|
||||
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
|
||||
"sourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废"), fine_field("材料隐裂重量", "材料废")],
|
||||
"comparisonSourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废"), fine_field("材料隐裂重量", "材料废")],
|
||||
"pairedRows": [
|
||||
{"key": "晶界扩散其他缺陷重量", "current": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 3},
|
||||
"comparison": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 2}, "deltaWeight": 1, "deltaAvailable": True},
|
||||
{"key": "材料隐裂重量", "current": {"label": "材料隐裂重量", "category": "材料废", "weight": 2},
|
||||
"comparison": {"label": "材料隐裂重量", "category": "材料废", "weight": 1}, "deltaWeight": 1, "deltaAvailable": True},
|
||||
],
|
||||
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
|
||||
"pairedRankingComplete": True,
|
||||
}
|
||||
fine = evidence(fine_result, ref="Q00000013")
|
||||
output = build_visuals([fine, changes, comparison], "分析本期变化并绘制综合对比、产品报废增量和来源缺陷图", "turn-priority")
|
||||
self.assertEqual([chart["metric"] for chart in output["visuals"]], [
|
||||
"composite_rate", "scrap_delta", "source_defect_delta_weight"
|
||||
])
|
||||
self.assertEqual(output["visuals"][2]["scope"]["department"], "其他部门")
|
||||
self.assertEqual(output["visuals"][2]["requiredFields"], ["晶界扩散其他缺陷重量", "材料隐裂重量"])
|
||||
scoped = build_visuals([fine, changes, comparison],
|
||||
"展示正式综合合格率、重点产品报废增量和亮片具体缺陷增量", "turn-priority-stage")
|
||||
self.assertEqual([chart["metric"] for chart in scoped["visuals"]], [
|
||||
"composite_rate", "scrap_delta", "source_defect_delta_weight"
|
||||
])
|
||||
|
||||
def test_fine_defects_use_paired_rows_and_do_not_turn_totals_only_into_zero(self):
|
||||
result = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "bright",
|
||||
"filters": {"department": "电机部", "extraFilters": {}},
|
||||
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
|
||||
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
|
||||
"sourceRows": 4,
|
||||
"sourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废")],
|
||||
"comparisonSourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废")],
|
||||
"pairedRows": [
|
||||
{"key": "晶界扩散其他缺陷重量", "current": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 0.0},
|
||||
"comparison": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 1.0}, "deltaWeight": -1.0, "deltaAvailable": True},
|
||||
],
|
||||
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
|
||||
"pairedRankingComplete": True,
|
||||
"categoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
|
||||
"expectedTotalsOnly": {"rows": 1}}],
|
||||
"comparisonCategoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
|
||||
"expectedTotalsOnly": {"rows": 0}}],
|
||||
}
|
||||
output = build_visuals([evidence(result)], "画图比较亮片来源具体缺陷增量", "turn-06")
|
||||
chart = output["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "source_defect_delta_weight")
|
||||
self.assertEqual(chart["stage"], "bright")
|
||||
self.assertEqual(chart["categories"], [{"key": "晶界扩散其他缺陷重量", "label": "晶界扩散其他缺陷重量"}])
|
||||
self.assertEqual(chart["series"][0]["values"], [-1.0])
|
||||
self.assertNotIn("VCM", chart["note"])
|
||||
self.assertEqual(chart["requiredFields"], ["晶界扩散其他缺陷重量"])
|
||||
|
||||
def test_fine_defect_missing_delta_stays_null_and_unavailable_coverage_skips(self):
|
||||
result = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "black", "filters": {},
|
||||
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
|
||||
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
|
||||
"sourceFields": [fine_field("责废尺寸重量", "责废"), fine_field("责废其他重量", "责废")],
|
||||
"comparisonSourceFields": [fine_field("责废尺寸重量", "责废"), fine_field("责废其他重量", "责废")],
|
||||
"pairedRows": [
|
||||
{"key": "责废尺寸重量", "current": {"label": "责废尺寸重量", "category": "责废", "weight": 2},
|
||||
"comparison": None, "deltaWeight": None, "deltaAvailable": False},
|
||||
{"key": "责废其他重量", "current": {"label": "责废其他重量", "category": "责废", "weight": 3},
|
||||
"comparison": {"label": "责废其他重量", "category": "责废", "weight": 1}, "deltaWeight": 2, "deltaAvailable": True},
|
||||
],
|
||||
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
|
||||
"pairedRankingComplete": False,
|
||||
}
|
||||
output = build_visuals([evidence(result)], "画图比较黑片来源缺陷增量", "turn-07")
|
||||
self.assertEqual(output["visuals"][0]["series"][0]["values"], [None, 2.0])
|
||||
result["coverage"] = {"available": False}
|
||||
self.assertEqual(build_visuals([evidence(result)], "画图比较黑片来源缺陷增量", "turn-07")["visualState"], "unavailable")
|
||||
|
||||
def test_fine_defects_authorize_only_real_source_fields_and_bind_category_scope(self):
|
||||
result = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "bright",
|
||||
"filters": {"department": "电机部", "extraFilters": {"项目类型": ["VCM"]}},
|
||||
"category": "加工废",
|
||||
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 3},
|
||||
"sourceRows": 3, "sourceFields": [fine_field("加工尺寸超差重量", "加工废")],
|
||||
"rows": [{"key": "加工尺寸超差重量", "label": "加工尺寸超差重量", "category": "加工废", "weight": 0}],
|
||||
"totalRows": 1, "truncated": False, "rankingComplete": True,
|
||||
}
|
||||
chart = build_visuals([evidence(result)], "画图看亮片来源具体缺陷重量", "turn-real-shape")["visuals"][0]
|
||||
self.assertEqual(chart["categories"], [{"key": "加工尺寸超差重量", "label": "加工尺寸超差重量"}])
|
||||
self.assertEqual(chart["requiredFields"], ["加工尺寸超差重量"])
|
||||
self.assertEqual(chart["series"][0]["values"], [0.0])
|
||||
self.assertEqual(chart["scope"]["category"], "加工废")
|
||||
self.assertEqual(chart["scope"]["extraFilters"], {"项目类型": ["VCM"]})
|
||||
|
||||
result["rows"][0]["key"] = "未授权缺陷重量"
|
||||
result["rows"][0]["label"] = "未授权缺陷重量"
|
||||
self.assertEqual(build_visuals([evidence(result)], "画图看亮片来源具体缺陷重量", "turn-real-shape")["visualState"], "unavailable")
|
||||
|
||||
def test_fine_defect_partial_totals_only_keeps_valid_mixed_rows_and_full_totals_only_is_not_zero(self):
|
||||
mixed = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "bright", "filters": {},
|
||||
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 4},
|
||||
"sourceRows": 4, "sourceFields": [fine_field("晶界扩散外观异常重量", "扩散废")],
|
||||
"rows": [{"key": "晶界扩散外观异常重量", "label": "晶界扩散外观异常重量",
|
||||
"category": "扩散废", "weight": 2.5}],
|
||||
"categoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
|
||||
"expectedTotalsOnly": {"rows": 1},
|
||||
"reconciliationStatus": "DIFF"}],
|
||||
"totalRows": 1, "truncated": False, "rankingComplete": True,
|
||||
}
|
||||
chart = build_visuals([evidence(mixed)], "画图看亮片来源具体缺陷排名", "turn-mixed")["visuals"][0]
|
||||
self.assertEqual(chart["series"][0]["values"], [2.5])
|
||||
self.assertNotIn("VCM", chart.get("note", ""))
|
||||
|
||||
all_totals = dict(mixed)
|
||||
all_totals["rows"] = [{"key": "晶界扩散外观异常重量", "label": "晶界扩散外观异常重量",
|
||||
"category": "扩散废", "weight": None}]
|
||||
all_totals["categoryReconciliation"] = [{"category": "扩散废", "sourceRows": 4,
|
||||
"expectedTotalsOnly": {"rows": 4},
|
||||
"reconciliationStatus": "EXPECTED_TOTALS_ONLY"}]
|
||||
self.assertEqual(build_visuals([evidence(all_totals)], "画图看亮片来源具体缺陷排名", "turn-mixed")["visualState"], "unavailable")
|
||||
|
||||
paired_totals = dict(all_totals)
|
||||
paired_totals["comparisonPeriod"] = {"year": 2026, "month": 8}
|
||||
paired_totals["comparisonCoverage"] = {"available": True, "sourceRows": 4}
|
||||
paired_totals["comparisonSourceFields"] = [fine_field("晶界扩散外观异常重量", "扩散废")]
|
||||
paired_totals["comparisonCategoryReconciliation"] = list(all_totals["categoryReconciliation"])
|
||||
paired_totals["sort"] = "scrap_delta"
|
||||
paired_totals["pairedRows"] = [{
|
||||
"key": "晶界扩散外观异常重量",
|
||||
"current": {"label": "晶界扩散外观异常重量", "category": "扩散废", "weight": None},
|
||||
"comparison": {"label": "晶界扩散外观异常重量", "category": "扩散废", "weight": None},
|
||||
"deltaWeight": None, "deltaAvailable": False,
|
||||
}]
|
||||
paired_totals["pairedTotalRows"] = 1
|
||||
paired_totals["pairedTruncated"] = False
|
||||
paired_totals["pairedRankingComplete"] = True
|
||||
self.assertEqual(build_visuals([evidence(paired_totals)], "画图比较亮片来源具体缺陷增量", "turn-mixed")["visualState"], "unavailable")
|
||||
|
||||
def test_fine_defect_chart_limits_to_ten_and_authorizes_only_displayed_fields(self):
|
||||
keys = ["责废尺寸重量", "责废缺角重量", "责废垂直度重量", "责废刀丝重量", "责废烂毛重量",
|
||||
"责废开裂重量", "责废其他重量", "料废尺寸重量", "料废缺角重量", "料废开裂重量",
|
||||
"料废烂毛重量", "料废其他重量"]
|
||||
result = {
|
||||
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
|
||||
"grain": "source_defect_weight", "stage": "black", "filters": {},
|
||||
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 2},
|
||||
"sourceRows": 2, "sourceFields": [fine_field(key, "责废" if key.startswith("责废") else "料废") for key in keys],
|
||||
"rows": [{"key": key, "label": key, "category": "责废" if key.startswith("责废") else "料废",
|
||||
"weight": float(index + 1)} for index, key in enumerate(keys)],
|
||||
"totalRows": len(keys), "truncated": False, "rankingComplete": True,
|
||||
}
|
||||
chart = build_visuals([evidence(result)], "画图看黑片来源具体缺陷排名", "turn-ten")["visuals"][0]
|
||||
self.assertEqual(len(chart["categories"]), 10)
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
|
||||
self.assertEqual(chart["requiredFields"], [item["key"] for item in chart["categories"]])
|
||||
|
||||
def test_multi_query_trend_requires_same_release_and_filters_and_keeps_gaps_null(self):
|
||||
docs = [
|
||||
evidence(metric_result({"year": 2026, "month": 1}, 0.91), ref="Q00000001"),
|
||||
evidence(metric_result({"year": 2026, "month": 3}, 0.93), ref="Q00000002"),
|
||||
evidence(metric_result({"year": 2026, "month": 4}, 0.94), ref="Q00000003"),
|
||||
]
|
||||
charts = build_visuals(docs, "画综合合格率趋势图", "turn-08")["visuals"]
|
||||
chart = next(item for item in charts if item["metric"] == "composite_rate")
|
||||
self.assertEqual([item["key"] for item in chart["categories"]], ["2026-01", "2026-02", "2026-03", "2026-04"])
|
||||
self.assertEqual(chart["series"][0]["values"], [0.91, None, 0.93, 0.94])
|
||||
annual = {'year': 2026, 'department': '电机部', 'trendGranularity': 'month'}
|
||||
bounded = build_visuals(docs, '画综合合格率趋势图', 'turn-annual', context=annual)['visuals']
|
||||
self.assertEqual(bounded[0]['series'][0]['values'], [0.91, None, 0.93, 0.94])
|
||||
unrelated = [evidence(metric_result({'year':2025,'month':12},0.50),ref='Q00000009'),
|
||||
evidence(metric_result({'year':2026,'month':2},0.10,filters={'department':'其他'}),ref='Q00000010')]
|
||||
filtered = build_visuals(docs+unrelated, '画综合合格率趋势图', 'turn-filtered-year', context=annual)['visuals']
|
||||
self.assertEqual(filtered[0]['series'][0]['values'], [0.91, None, 0.93, 0.94])
|
||||
quarter_docs = docs + [evidence(metric_result({'year':2026,'month':2},0.92),ref='Q00000011')]
|
||||
quarter = build_visuals(quarter_docs, '画综合合格率趋势图', 'turn-quarter', context={**annual,'quarter':1})['visuals']
|
||||
self.assertEqual(quarter[0]['series'][0]['values'], [0.91, 0.92, 0.93])
|
||||
single_month = build_visuals(docs, '画综合合格率趋势图', 'turn-month', context={**annual,'month':3})
|
||||
self.assertEqual(single_month['visualState'], 'unavailable')
|
||||
mismatch = [docs[0], docs[1], evidence(metric_result({"year": 2026, "month": 4}, 0.94, filters={"department": "其他"}), ref="Q00000004")]
|
||||
self.assertEqual(build_visuals(mismatch, "画综合合格率趋势图", "turn-08")["visualState"], "unavailable")
|
||||
mixed_release = [docs[0], docs[1], evidence(metric_result({"year": 2026, "month": 4}, 0.94, release_id="r-2"), ref="Q00000005", release_id="r-2")]
|
||||
self.assertEqual(build_visuals(mixed_release, "画综合合格率趋势图", "turn-08")["visualState"], "unavailable")
|
||||
|
||||
def test_trend_uses_the_same_unique_comparison_scope_as_the_text(self):
|
||||
scoped = comparison_result(filterBinding={'group':'A'},
|
||||
current={'period':{'year':2026},'compositeRate':.98,'black':{'rate':.99},'bright':{'rate':.99}},
|
||||
comparison={'period':{'year':2025},'compositeRate':.97,'black':{'rate':.98},'bright':{'rate':.99}})
|
||||
global_docs = [evidence(metric_result({'year':2026,'month':m},.90+m/100,filters={}),ref=f'Q0000000{m}') for m in (1,3,4)]
|
||||
charts = build_visuals([evidence(scoped)]+global_docs,'比较今年变化并画趋势图','turn-root-trend',context={'year':2026})['visuals']
|
||||
self.assertFalse(any(chart.get('sectionKey')=='trend' for chart in charts))
|
||||
local_docs = [evidence(metric_result({'year':2026,'month':m},.95+m/100,filters={'group':'A'}),ref=f'Q0000001{m}') for m in (1,3,4)]
|
||||
charts = build_visuals([evidence(scoped)]+global_docs+local_docs,'比较今年变化并画趋势图','turn-root-trend-ok',context={'year':2026})['visuals']
|
||||
trend = next(chart for chart in charts if chart.get('sectionKey')=='trend')
|
||||
self.assertEqual(trend['scope']['group'], 'A')
|
||||
self.assertEqual(trend['series'][0]['values'], [.96, None, .98, .99])
|
||||
|
||||
def test_native_trend_is_sorted_and_bounded(self):
|
||||
result = metric_result({"year": 2026, "month": 9}, 0.93)
|
||||
result.update({
|
||||
"trend": [
|
||||
{"period": "2026-03", "compositeRate": 0.93, "black": {"rate": 0.92}, "bright": {"rate": 0.94}},
|
||||
{"period": "2026-01", "compositeRate": 0.91, "black": {"rate": 0.90}, "bright": {"rate": 0.92}},
|
||||
{"period": "2026-04", "compositeRate": 0.94, "black": {"rate": 0.93}, "bright": {"rate": 0.95}},
|
||||
],
|
||||
"trendCount": 3,
|
||||
"trendTruncated": False,
|
||||
})
|
||||
chart = build_visuals([evidence(result)], "画合格率趋势图", "turn-09")["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "composite_rate")
|
||||
self.assertEqual([item["key"] for item in chart["categories"]], ["2026-01", "2026-02", "2026-03", "2026-04"])
|
||||
self.assertEqual(chart["series"][0]["values"], [0.91, None, 0.93, 0.94])
|
||||
result["trendTruncated"] = True
|
||||
self.assertEqual(build_visuals([evidence(result)], "画合格率趋势图", "turn-09")["visualState"], "unavailable")
|
||||
|
||||
def test_explicit_heatmap_and_area_are_real_period_trends_only(self):
|
||||
result = metric_result({"year": 2026, "month": 12}, 0.93)
|
||||
result.update({
|
||||
"trend": [
|
||||
{"period": f"2026-{month:02d}", "compositeRate": 0.8 + month / 100,
|
||||
"black": {"rate": 0.79 + month / 100, "scrapWeight": month},
|
||||
"bright": {"rate": 0.81 + month / 100, "scrapWeight": month / 2}}
|
||||
for month in range(1, 13)
|
||||
],
|
||||
"trendCount": 12,
|
||||
"trendTruncated": False,
|
||||
})
|
||||
ordinary = build_visuals([evidence(result)], "查看综合合格率月度趋势", "turn-usual-trend")["visuals"][0]
|
||||
self.assertEqual(ordinary["type"], "line")
|
||||
heatmap = build_visuals([evidence(result)], "用热力图展示综合合格率趋势", "turn-heatmap")["visuals"][0]
|
||||
self.assertEqual(heatmap["type"], "heatmap")
|
||||
self.assertEqual(len(heatmap["categories"]), 12)
|
||||
self.assertEqual(len(heatmap["series"]), 1)
|
||||
|
||||
result["trend"] = result["trend"][:4]
|
||||
result["trendCount"] = 4
|
||||
area = build_visuals([evidence(result)], "画亮片报废重量面积趋势图", "turn-area")["visuals"][0]
|
||||
self.assertEqual(area["type"], "area")
|
||||
self.assertEqual(area["stage"], "bright")
|
||||
self.assertEqual(area["metric"], "current_scrap_weight")
|
||||
self.assertTrue(all(value is None or value >= 0 for value in area["series"][0]["values"]))
|
||||
|
||||
result["trend"] = result["trend"][:2]
|
||||
result["trendCount"] = 2
|
||||
self.assertEqual(build_visuals([evidence(result)], "画面积图", "turn-area-invalid")["visualState"], "unavailable")
|
||||
|
||||
def test_formal_summary_produces_guarded_donut_and_pareto(self):
|
||||
defects = [
|
||||
{"name": "镀废", "weight": 3.0},
|
||||
{"name": "材料废", "weight": 5.0},
|
||||
{"name": "加工废", "weight": 0.0},
|
||||
{"name": "扩散废", "weight": 2.0},
|
||||
]
|
||||
result = metric_result({"year": 2026, "month": 9}, 0.93)
|
||||
result["summary"]["bright"] = {"defects": defects}
|
||||
doc = evidence(result)
|
||||
donut = build_visuals([doc], "用环状图展示亮片已列正式报废类别构成", "turn-donut")["visuals"][0]
|
||||
self.assertEqual(donut["type"], "donut")
|
||||
self.assertEqual(donut["metric"], "source_defect_weight")
|
||||
self.assertEqual(donut["series"][0]["values"], [3.0, 5.0, 0.0, 2.0])
|
||||
self.assertEqual(donut["coverage"], {
|
||||
"totalRows": 4, "returnedRows": 4, "truncated": False, "shareBasis": "displayed_categories"
|
||||
})
|
||||
self.assertEqual(donut["requiredFields"], [
|
||||
"bright_plating_scrap", "bright_material_scrap", "bright_processing_scrap", "bright_diffusion_scrap"
|
||||
])
|
||||
self.assertIn("占比按本图已列类别重量合计", donut["note"])
|
||||
|
||||
pareto = build_visuals([doc], "绘制亮片柏拉图并标注报废类别", "turn-pareto")["visuals"][0]
|
||||
self.assertEqual(pareto["type"], "pareto")
|
||||
self.assertEqual(pareto["series"][0]["values"], [5.0, 3.0, 2.0, 0.0])
|
||||
self.assertEqual([category["key"] for category in pareto["categories"]], [
|
||||
"bright_material_scrap", "bright_plating_scrap", "bright_diffusion_scrap", "bright_processing_scrap"
|
||||
])
|
||||
self.assertNotEqual(donut["id"], pareto["id"])
|
||||
|
||||
comparison = comparison_result()
|
||||
comparison["current"]["bright"] = {"defects": defects, "rate": 0.99, "scrapWeight": 5.0}
|
||||
current_donut = build_visuals([evidence(comparison)], "亮片当前已列类别构成环状图", "turn-comparison-donut")["visuals"][0]
|
||||
self.assertEqual(current_donut["type"], "donut")
|
||||
self.assertEqual(current_donut["period"], {"year": 2026, "month": 9})
|
||||
self.assertNotIn("comparisonPeriod", current_donut)
|
||||
|
||||
def test_summary_and_complete_category_top_only_make_share_charts_when_complete(self):
|
||||
result = metric_result({"year": 2026, "month": 9}, 0.93)
|
||||
result["summary"]["black"] = {"defects": [
|
||||
{"name": "责废", "weight": 1.0}, {"name": "料废", "weight": 2.0}, {"name": "外废", "weight": 3.0}
|
||||
]}
|
||||
result["summary"]["black"]["defects"][1]["weight"] = -2.0
|
||||
self.assertEqual(build_visuals([evidence(result)], "黑片环状图展示构成", "turn-negative")["visualState"], "unavailable")
|
||||
result["summary"]["black"]["defects"][1]["weight"] = 0.0
|
||||
result["summary"]["black"]["defects"][2]["weight"] = 0.0
|
||||
result["summary"]["black"]["defects"][0]["weight"] = 0.0
|
||||
self.assertEqual(build_visuals([evidence(result)], "黑片环状图展示构成", "turn-zero")["visualState"], "unavailable")
|
||||
result["summary"]["black"]["defects"][0]["weight"] = 1.0
|
||||
filtered = dict(result)
|
||||
filtered["filters"] = {"category": "责废", "extraFilters": {}}
|
||||
self.assertEqual(build_visuals([evidence(filtered)], "黑片环状图展示构成", "turn-filtered-category")["visualState"], "unavailable")
|
||||
|
||||
top = {
|
||||
"kind": "quality_top", "releaseId": "r-1", "stage": "black", "dimension": "category",
|
||||
"period": {"year": 2026, "month": 9}, "filters": {"extraFilters": {}},
|
||||
"rankingCoverage": {"groups": 3, "complete": True},
|
||||
"cumulativeThresholdApplied": False,
|
||||
"rows": [
|
||||
{"rank": 1, "key": "责废", "label": "责废", "weight": 8.0},
|
||||
{"rank": 2, "key": "料废", "label": "料废", "weight": 4.0},
|
||||
{"rank": 3, "key": "外废", "label": "外废", "weight": 1.0},
|
||||
],
|
||||
}
|
||||
chart = build_visuals([evidence(top)], "黑片柏拉图按正式类别", "turn-pareto-top")["visuals"][0]
|
||||
self.assertEqual(chart["type"], "pareto")
|
||||
self.assertEqual(chart["series"][0]["values"], [8.0, 4.0, 1.0])
|
||||
top["rankingCoverage"]["complete"] = False
|
||||
self.assertEqual(build_visuals([evidence(top)], "黑片柏拉图按正式类别", "turn-pareto-incomplete")["visualState"], "unavailable")
|
||||
|
||||
def test_colloquial_quality_analysis_requests_generate_bound_charts(self):
|
||||
questions = [
|
||||
'查一下消费电子美洲近期质量下滑严重的产品',
|
||||
'看看电机部合格率下跌的规格', '电机部质量走低的产品',
|
||||
'电机部合格率回落的产品', '电机部合格率降低的产品',
|
||||
'查查电机部质量变差的产品', '查查电机部质量变坏的产品',
|
||||
'查查电机部质量变糟的产品', '找出电机部降幅大的规格',
|
||||
'查电机部跌幅大的产品', '电机部质量最差的产品',
|
||||
'电机部质量较差的产品', '电机部合格率偏低的规格',
|
||||
'查查电机部质量不理想的产品', '电机部表现不佳的产品',
|
||||
'电机部质量不稳定的产品', '电机部质量波动大的产品',
|
||||
'电机部报废突然变多的产品', '电机部报废突增的规格',
|
||||
'电机部报废激增的产品', '近期值得关注的规格',
|
||||
'近期重点产品', '电机部质量好转的规格',
|
||||
'电机部合格率提升的产品', '电机部合格率回升的产品',
|
||||
'电机部报废减少的产品', '电机部质量环比怎么样',
|
||||
'电机部与上月差异', '电机部质量异常的产品',
|
||||
'电机部质量不好的产品', '电机部质量不佳的产品',
|
||||
'电机部质量差的产品', '电机部质量糟糕的产品',
|
||||
'电机部合格率变低的规格', '电机部质量退步的产品',
|
||||
'电机部质量不达标的产品', '电机部合格率低于目标的规格',
|
||||
'电机部质量降得厉害的产品', '电机部合格率掉得多的规格',
|
||||
]
|
||||
doc = evidence(comparison_result())
|
||||
context = {'releaseId': 'r-1', 'filters': {'department': '电机部'}}
|
||||
for question in questions:
|
||||
with self.subTest(question=question):
|
||||
output = build_visuals([doc], question, 'turn-colloquial', context=context)
|
||||
self.assertEqual(output['visualState'], 'ready')
|
||||
self.assertTrue(output['visuals'])
|
||||
self.assertTrue(all(v['scope'].get('department') == '电机部' for v in output['visuals']))
|
||||
self.assertTrue(all(v['evidenceRefs'] == ['Q1234ABCD'] for v in output['visuals']))
|
||||
|
||||
def test_colloquial_requests_keep_text_only_and_no_evidence_guards(self):
|
||||
question = '查一下近期质量下滑严重的产品'
|
||||
doc = evidence(comparison_result())
|
||||
for suffix in ['只要文字', '仅需文本', '仅用文字', '只返回数值', '不用画图', '不需要配图', '不要图表', '只需要文字', '仅输出数字', '仅文字', '无需图', '不必画图']:
|
||||
with self.subTest(suffix=suffix):
|
||||
self.assertEqual(build_visuals([doc], question + ',' + suffix, 'turn-text-only')['visuals'], [])
|
||||
for simple in ['综合合格率是多少', '黑片合格率多少', '产品名称是什么', '查一下数据源文件名']:
|
||||
with self.subTest(simple=simple):
|
||||
self.assertEqual(build_visuals([doc], simple, 'turn-simple')['visuals'], [])
|
||||
self.assertEqual(build_visuals([], question, 'turn-no-evidence')['visuals'], [])
|
||||
wrong_scope = {'releaseId': 'r-1', 'filters': {'department': '无关部门'}}
|
||||
labelled = build_visuals([doc], question, 'turn-wrong-scope', context=wrong_scope)['visuals']
|
||||
self.assertTrue(all(v['scope'].get('department') == '电机部' for v in labelled))
|
||||
|
||||
def test_simple_text_and_negative_visual_request_override(self):
|
||||
doc = evidence(comparison_result())
|
||||
self.assertEqual(build_visuals([doc], "综合合格率是多少", "turn-10")["visualState"], "not_requested")
|
||||
self.assertEqual(build_visuals([doc], "不要图,只要文字,比较两个期间", "turn-10")["visualState"], "not_requested")
|
||||
self.assertEqual(build_visuals([doc], "不要绘图,只要文本,比较两个期间", "turn-10")["visualState"], "not_requested")
|
||||
self.assertEqual(build_visuals([], "重新分析重点产品并写出降幅", "turn-followup", context={})["visualState"], "not_requested")
|
||||
self.assertEqual(build_visuals([], "画图比较重点产品降幅", "turn-followup", context={})["visualState"], "unavailable")
|
||||
column = build_visuals([doc], "不要环状图,用柱状图比较两期综合合格率", "turn-negative-style-column")["visuals"][0]
|
||||
self.assertEqual(column["type"], "column")
|
||||
self.assertEqual(build_visuals([doc], "不用柏拉图,显示条形图比较两期综合合格率", "turn-negative-style-bar")["visuals"][0]["type"], "bar")
|
||||
|
||||
def test_explicit_composite_trend_does_not_add_unrequested_stages_or_plot_missing_as_perfect(self):
|
||||
result = metric_result({"year": 2026}, 0.93)
|
||||
result.update({"trend": [
|
||||
{"period": "2026-01", "compositeRate": 0.91},
|
||||
{"period": "2026-02", "compositeRate": 1.0, "coverage": {"blackRows": 0, "brightRows": 0}},
|
||||
{"period": "2026-03", "compositeRate": 0.93},
|
||||
{"period": "2026-04", "compositeRate": 0.94},
|
||||
{"period": "2026-05", "compositeRate": 1.0, "monthStatus": "future"},
|
||||
], "trendCount": 5, "trendTruncated": False})
|
||||
charts = build_visuals([evidence(result)], "只展示综合合格率趋势图", "turn-nodata")["visuals"]
|
||||
self.assertEqual([chart["metric"] for chart in charts], ["composite_rate"])
|
||||
self.assertEqual(charts[0]["series"][0]["values"], [0.91, None, 0.93, 0.94, None])
|
||||
|
||||
def test_ids_are_stable_per_semantics_and_output_is_isolated(self):
|
||||
doc = evidence(comparison_result())
|
||||
doc["absoluteEvidencePath"] = "/Users/alice/private/source.json"
|
||||
first = build_visuals([doc], "比较两个期间变化", "turn-11")
|
||||
second = build_visuals([doc], "比较两个期间变化", "turn-11")
|
||||
self.assertEqual(first["visuals"][0]["id"], second["visuals"][0]["id"])
|
||||
self.assertNotIn("absoluteEvidencePath", json.dumps(first, ensure_ascii=False))
|
||||
self.assertNotIn("/Users/alice", json.dumps(first, ensure_ascii=False))
|
||||
self.assertLess(len(json.dumps(first, ensure_ascii=False).encode("utf-8")), MAX_BODY_BYTES)
|
||||
changed_turn = build_visuals([doc], "比较两个期间变化", "turn-12")
|
||||
self.assertNotEqual(first["visuals"][0]["id"], changed_turn["visuals"][0]["id"])
|
||||
|
||||
def test_malformed_evidence_and_large_rankings_fail_softly(self):
|
||||
malformed = evidence(changes_result(rows=[{"rank": True, "key": "p-1", "label": "x", "scrapDelta": 1}]))
|
||||
self.assertEqual(build_visuals([malformed], "画产品报废增量排名", "turn-13")["visualState"], "unavailable")
|
||||
result = changes_result(rows=[
|
||||
{"rank": index + 1, "key": f"p-{index}", "label": f"规格{index}", "scrapDelta": index}
|
||||
for index in range(15)
|
||||
], totalRows=15, returnedRows=15, truncated=False)
|
||||
chart = build_visuals([evidence(result)], "画产品报废增量排名", "turn-13")["visuals"][0]
|
||||
self.assertEqual(len(chart["categories"]), 10)
|
||||
self.assertTrue(chart["coverage"]["truncated"])
|
||||
self.assertEqual(chart["series"][0]["values"], list(map(float, range(10))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from analysis_visuals import _scope, build_visuals
|
||||
from test_month_range_evidence import QUESTION, context, metric_doc
|
||||
|
||||
|
||||
class ExclusionVisualScopeTests(unittest.TestCase):
|
||||
def setup_frame(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
|
||||
frame = context(period)
|
||||
frame['filters'] = {**frame['filters'], 'exclusions': {'product': ['SPEC-A']}}
|
||||
frame['_agentTask']['filters'] = copy.deepcopy(frame['filters'])
|
||||
doc = metric_doc(period, range(1, 10), filters=frame['filters'])
|
||||
return frame, doc
|
||||
|
||||
def test_excluded_product_is_preserved_in_the_complete_monthly_chart(self):
|
||||
frame, doc = self.setup_frame()
|
||||
result = build_visuals([doc], QUESTION, 'turn-test', context=frame)
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
chart = result['visuals'][0]
|
||||
self.assertEqual(chart['scope']['exclusions'], {'product': ['SPEC-A']})
|
||||
self.assertEqual(len(chart['series'][0]['values']), 9)
|
||||
self.assertEqual(chart['scope']['monthEnd'], 9)
|
||||
|
||||
def test_scope_mismatch_never_uses_a_chart_from_other_exclusions(self):
|
||||
for wrong in ({}, {'product': ['SPEC-B']}, {'product': ['SPEC-A'], 'group': ['OTHER']}):
|
||||
frame, doc = self.setup_frame()
|
||||
doc['result']['filters']['exclusions'] = wrong
|
||||
# Separate the source receipt from the immutable expected frame.
|
||||
frame['filters']['exclusions'] = {'product': ['SPEC-A']}
|
||||
frame['_agentTask']['filters'] = copy.deepcopy(frame['filters'])
|
||||
doc['result']['filters'] = {**doc['result']['filters'], 'exclusions': wrong}
|
||||
result = build_visuals([doc], QUESTION, 'turn-test', context=frame)
|
||||
self.assertEqual(result['visuals'], [])
|
||||
|
||||
def test_malformed_or_unsupported_exclusions_are_rejected(self):
|
||||
for wrong in ({'product': []}, {'product': [None]}, {'customerCode': ['001']},
|
||||
{'product': 'SPEC-A'}, {'product': [' ']}):
|
||||
self.assertEqual(_scope({'exclusions': wrong}), (None, None))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Regression: a grouped comparison must not expand into per-object reads."""
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
from nao_agent_task import normalize_task
|
||||
from nao_protocol_execution import _scoped_arguments, _validate_protocol_scope, execute
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
|
||||
|
||||
|
||||
def changes_node(**updates):
|
||||
return {'nodeId': 'groups', 'purpose': 'changes', 'dimension': 'group',
|
||||
'fields': ['rate'], 'stages': [], 'sort': 'rate_drop', 'limit': 1,
|
||||
'selector': {'kind': 'base'}, 'comparisonRequired': True,
|
||||
'visualPreference': 'none', 'visualType': None, **updates}
|
||||
|
||||
|
||||
def normalized(nodes):
|
||||
dimensions = list(dict.fromkeys(n.get('dimension') for n in nodes if n.get('dimension')))
|
||||
stages = list(dict.fromkeys(s for n in nodes for s in n['stages']))
|
||||
fields = list(dict.fromkeys(f for n in nodes for f in n['fields']))
|
||||
return normalize_task(task_value(kind='analysis', queryPlan=nodes,
|
||||
dimensions=dimensions, dimensionStages={}, fields=fields, stages=stages,
|
||||
trendRequired=False, comparisonRequired=True,
|
||||
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8},
|
||||
visualPreference='none'), '9月份哪个业务分组综合合格率下滑最多',
|
||||
{'_protocol': {'version': 1}})
|
||||
|
||||
|
||||
class MinimalQueryTests(unittest.TestCase):
|
||||
def test_auto_batch_style_hint_does_not_report_missing_chart(self):
|
||||
from analysis_visual_evidence import protocol_visual_result
|
||||
task=normalized([changes_node(visualPreference='auto',visualType='bar')])
|
||||
task.pop('queryPlan');task['visualPreference']='auto';task['visualType']='bar'
|
||||
scope={'nodeId':'groups','scopeId':'batch','purpose':'changes','task':task}
|
||||
with patch('analysis_visuals.build_visuals',return_value={'visuals':[]}):
|
||||
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
||||
self.assertEqual(result['visualState'],'not_requested',result)
|
||||
self.assertEqual(result['missingEvidence'],[])
|
||||
task['visualPreference']='required'
|
||||
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
||||
self.assertEqual(result['visualState'],'unavailable',result)
|
||||
self.assertTrue(result['missingEvidence'])
|
||||
task['visualPreference']='auto'
|
||||
with patch('analysis_visuals.build_visuals',return_value={'visuals':[{'metric':'rate_drop_pp'}]}):
|
||||
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
||||
self.assertEqual(result['visualState'],'ready',result)
|
||||
|
||||
def test_requested_weights_cannot_be_completed_when_hidden(self):
|
||||
from nao_changes import validate_changes
|
||||
task=normalized([changes_node(fields=['rate','totalWeight','scrapWeight'])])
|
||||
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
||||
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
||||
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':1,
|
||||
'rows':[{'rank':1,'key':'甲','current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
||||
with self.assertRaisesRegex(ValueError,'正式重量'):
|
||||
validate_changes(result,task,RELEASE)
|
||||
|
||||
def test_group_rate_answer_does_not_force_weight_details(self):
|
||||
from nao_changes import changes_report
|
||||
task=normalized([changes_node()]);task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
||||
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
||||
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':10,
|
||||
'rows':[{'rank':1,'key':'甲','rateDrop':0.2,
|
||||
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
||||
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
|
||||
self.assertEqual(len(report['requiredFactIds']),1)
|
||||
self.assertNotIn('投入',report['facts'][0]['text'])
|
||||
self.assertIn('下降20',report['facts'][0]['text'])
|
||||
self.assertNotIn('报废 kg','\n'.join(report['sections']))
|
||||
self.assertNotIn('投入 kg','\n'.join(report['sections']))
|
||||
|
||||
def test_batch_comparison_chart_keeps_both_periods_without_extra_metric_read(self):
|
||||
from analysis_visuals import build_visuals
|
||||
from nao_agent_task import bind_task
|
||||
task=normalized([changes_node(visualPreference='required')])
|
||||
task.pop('queryPlan');task['visualPreference']='required'
|
||||
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
||||
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
||||
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,
|
||||
'totalRows':10,'returnedRows':1,'truncated':True,
|
||||
'rows':[{'rank':1,'key':'甲','label':'甲','rateDrop':0.2,
|
||||
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
||||
doc={'evidenceRef':'Q12345678','releaseId':RELEASE,'mode':'changes','result':result}
|
||||
visual=build_visuals([doc],task['question'],TURN,
|
||||
context=bind_task({'releaseId':RELEASE},task))
|
||||
self.assertEqual(visual['visualState'],'ready',visual)
|
||||
self.assertEqual(visual['visuals'][0]['metric'],'rate_drop_pp')
|
||||
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[20.0])
|
||||
result['sort']='current_rate';task['changeQuery']['sort']='current_rate'
|
||||
visual=build_visuals([doc],task['question'],TURN,
|
||||
context=bind_task({'releaseId':RELEASE},task))
|
||||
self.assertEqual(visual['visualState'],'ready',visual)
|
||||
self.assertEqual(visual['visuals'][0]['metric'],'composite_rate')
|
||||
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[0.7])
|
||||
|
||||
def test_whole_group_comparison_is_one_batch_node(self):
|
||||
task = normalized([changes_node()])
|
||||
child = copy.deepcopy(task); child.pop('queryPlan')
|
||||
child['changeQuery'] = {'dimension': 'group', 'sort': 'rate_drop', 'limit': 1}
|
||||
scope = {'nodeId': 'groups', 'scopeId': 'batch', 'purpose': 'changes',
|
||||
'label': '业务分组批量对比', 'task': child, 'limit': 1}
|
||||
_validate_protocol_scope(scope, task['queryPlan'][0], task, 'release')
|
||||
args = _scoped_arguments({'releaseId': 'release'}, scope, task['queryPlan'][0])
|
||||
self.assertEqual(len(args), 1)
|
||||
self.assertEqual(args[0]['mode'], 'changes')
|
||||
self.assertEqual(args[0]['limit'], 1)
|
||||
self.assertNotIn('group', args[0]['filters'])
|
||||
|
||||
def test_optional_drilldown_depends_on_actual_leading_result(self):
|
||||
task = normalized([changes_node(limit=3), changes_node(nodeId='products', dimension='product',
|
||||
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
|
||||
'count':1,'stage':'dominant'})])
|
||||
self.assertEqual(task['queryPlan'][1]['selector']['sourceNodeId'], 'groups')
|
||||
self.assertTrue(task['queryPlan'][1]['optional'])
|
||||
|
||||
def test_cannot_select_unqueried_or_future_result(self):
|
||||
with self.assertRaises(ValueError):
|
||||
normalized([changes_node(selector={'kind':'changes_top','sourceNodeId':'future',
|
||||
'count':1,'stage':'overall'})])
|
||||
|
||||
def test_overall_and_stage_rate_nodes_cannot_reread_the_same_baseline(self):
|
||||
rate={'nodeId':'overall','purpose':'metrics','fields':['rate'],'stages':[],
|
||||
'selector':{'kind':'base'},'comparisonRequired':True,'visualPreference':'auto','visualType':None}
|
||||
with self.assertRaisesRegex(ValueError,'无需重复读取'):
|
||||
normalized([rate,{**rate,'nodeId':'stage_rate','stages':['black','bright']}])
|
||||
|
||||
def test_cause_does_not_rank_tiny_products_by_rate_alone(self):
|
||||
task=normalized([changes_node(dimension='product')])
|
||||
from nao_query_plan import normalize_query_plan
|
||||
task['question']='9月份合格率下降原因'
|
||||
with self.assertRaisesRegex(ValueError,'小批量'):
|
||||
normalize_query_plan(task['queryPlan'],root_task=task)
|
||||
|
||||
def test_performance_core_keeps_rate_ranking_and_scale_is_only_supporting(self):
|
||||
from nao_changes import changes_report
|
||||
task=normalized([changes_node(dimension='product',sort='current_rate',fields=['rate','totalWeight','scrapWeight'])])
|
||||
task['question']='3C哪些产品近期表现最差';task['changeQuery']={'dimension':'product','sort':'current_rate','limit':1}
|
||||
side=lambda rate,weight:{'rate':rate,'scrapWeight':weight,
|
||||
'black':{'rate':rate,'totalWeight':1000,'scrapWeight':weight},
|
||||
'bright':{'rate':1,'totalWeight':1000,'scrapWeight':0},
|
||||
'blackInputWeight':1000,'brightInputWeight':1000}
|
||||
tiny={'rank':1,'key':'小样','label':'小样','rateDrop':1,'current':side(0,1),'comparison':side(1,0)}
|
||||
# A large scrap volume can coexist with an improving rate. It cannot
|
||||
# supplant the actual lowest-rate product in the requested ranking.
|
||||
large={'key':'批量产品','label':'批量产品','scrapDelta':200,'current':side(.9,300),'comparison':side(.8,100)}
|
||||
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
||||
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'dimension':'product','stage':None,'sort':'current_rate','completeRanking':True,'totalRows':10,
|
||||
'rows':[tiny],'prioritySignals':{'currentScrap':[large],'scrapIncrease':[large]}}
|
||||
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
|
||||
core=[f['text'] for f in report['facts'] if f['id'] in report['requiredFactIds']]
|
||||
self.assertTrue(core)
|
||||
self.assertIn('小样',core[0]);self.assertIn('0.00%',core[0])
|
||||
self.assertFalse(any('批量产品' in text for text in core))
|
||||
signals=[f for f in report['facts'] if '批量产品' in f['text']]
|
||||
self.assertTrue(signals)
|
||||
self.assertTrue(all(f['kind']=='supporting' for f in signals))
|
||||
self.assertIn('小样','\n'.join(report['sections']))
|
||||
|
||||
def test_real_executor_reads_only_batch_and_honors_conditional_stop(self):
|
||||
task = normalized([changes_node(), changes_node(nodeId='products', dimension='product',
|
||||
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
|
||||
'count':1,'stage':'overall'})])
|
||||
child = copy.deepcopy(task);child.pop('queryPlan')
|
||||
child.update(dimensions=['group'],dimensionStages={'group':[]},
|
||||
changeQuery={'dimension':'group','sort':'rate_drop','limit':1})
|
||||
scope = {'nodeId':'groups','scopeId':'batch','purpose':'changes','label':'业务分组',
|
||||
'task':child,'limit':1}
|
||||
|
||||
class Api(BoundApi):
|
||||
def __init__(self):
|
||||
super().__init__();self.reads=[]
|
||||
def call(self,path,payload=None,method=None,*,node_id=None,scope_id=None):
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'version':1,'scopes':[scope], 'pending':[], 'gaps':[],
|
||||
'skipped':[{'nodeId':'products','reason':'no_adverse_change'}]}
|
||||
if path == '/api/analysis/changes':
|
||||
self.reads.append((path,copy.deepcopy(payload)))
|
||||
return {'kind':'quality_changes','releaseId':RELEASE,
|
||||
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'filters':task['filters'],'dimension':'group','stage':None,'sort':'rate_drop',
|
||||
'totalRows':10,'returnedRows':1,'truncated':True,'completeRanking':True,
|
||||
'coverage':{'rateComparable':True},
|
||||
'rows':[{'rank':1,'key':'组甲','label':'组甲','rateDrop':-0.01,'scrapDelta':-10,
|
||||
'current':{'rate':0.97,'scrapWeight':20},
|
||||
'comparison':{'rate':0.96,'scrapWeight':30}}],
|
||||
'protocolEvidence':{**identity(),'queryId':'batch-query','responseHash':'a'*64,
|
||||
'nodeId':node_id,'scopeId':scope_id}}
|
||||
if path.startswith('/api/metrics') or path.endswith('/defects'):
|
||||
raise AssertionError('排名问题不能附加逐对象指标或缺陷读取')
|
||||
return super().call(path,payload,method)
|
||||
|
||||
api=Api();api.task=task
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
||||
with patch('nao_evidence_finalizer._open_model', side_effect=OSError('offline synthesis')):
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':task['question']},
|
||||
task=task,authority={**identity(),'task':task},actor_root=root,delegated=api,
|
||||
connection=('http://offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertEqual(result['status'],'COMPLETED',result.get('evidenceCompletion'))
|
||||
self.assertEqual(len(api.reads),1)
|
||||
self.assertEqual(len(result['protocolEvidence']),1)
|
||||
self.assertNotIn('没有取得', result['answer'])
|
||||
self.assertIn('## 统计数据附录',result['answer'])
|
||||
self.assertIn('| 序号 | 对象 |',result['answer'])
|
||||
|
||||
|
||||
class ConfirmedScalarSelectionTests(unittest.TestCase):
|
||||
def scalar(self):
|
||||
original='2026年9月18X15X4规格的综合合格率是多少?不要包含18X15X4.1。'
|
||||
task=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9},comparisonRequired=False,
|
||||
trendRequired=False,visualPreference='none',filterUpdates={
|
||||
'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}}),original,{})
|
||||
task['rootGoal']=original
|
||||
task['productResolution']={'originalQuestion':original,'selectedName':'18X15X4+T',
|
||||
'productMatch':{'status':'ambiguous','candidates':[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]}}
|
||||
return task
|
||||
|
||||
def test_only_signed_scalar_choice_dispatches_same_metrics_and_keeps_goal_review(self):
|
||||
from nao_protocol_execution import _confirmed_scalar_selection
|
||||
task=self.scalar();before=copy.deepcopy(task);api=BoundApi()
|
||||
request={'releaseId':RELEASE,'turnId':TURN,'question':'选择18X15X4+T。','_autonomousAnswer':True}
|
||||
self.assertTrue(_confirmed_scalar_selection(request,task))
|
||||
for sufficient in (True,False):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize',return_value={
|
||||
'answer':'已核实本期正式率','evidenceSufficient':sufficient,
|
||||
'missingEvidence':[] if sufficient else ['原目标仍有未回答内容']}) as synth:
|
||||
result=execute(request=copy.deepcopy(request),task=task,authority=identity(),actor_root=root,
|
||||
delegated=api,connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
context=synth.call_args.kwargs['context']
|
||||
self.assertTrue(context['_autonomousAnswer']);self.assertTrue(context['_confirmedScalarSelection'])
|
||||
self.assertEqual(context['_agentTask']['rootGoal'],before['rootGoal'])
|
||||
self.assertEqual(result['status'],'COMPLETED' if sufficient else 'PARTIAL')
|
||||
self.assertEqual(result['agentTask'],before)
|
||||
self.assertEqual(len(result['protocolEvidence']),1)
|
||||
self.assertEqual(task,before)
|
||||
self.assertTrue(all('exclusions=' in path and 'product=18X15X4%2BT' in path
|
||||
for _,path,_ in api.calls if path.startswith('/api/metrics')))
|
||||
|
||||
def test_old_unselected_investigative_and_non_scalar_tasks_stay_rejected(self):
|
||||
from nao_protocol_execution import _confirmed_scalar_selection
|
||||
request={'_autonomousAnswer':True};task=self.scalar()
|
||||
self.assertFalse(_confirmed_scalar_selection({},task))
|
||||
for changes in ({'stages':['black']},{'trendRequired':True},{'comparisonRequired':True},
|
||||
{'verifySource':True},{'fullDetails':True},{'executionMode':'native_agent'},
|
||||
{'dimensions':['product']},{'fields':['rate','totalWeight']},
|
||||
{'period':{'year':2026}},{'productResolution':{}},{'rootGoal':'找出重点规格'}):
|
||||
with self.subTest(changes=changes):
|
||||
self.assertFalse(_confirmed_scalar_selection(request,{**task,**changes}))
|
||||
for original in ('找重点规格的综合合格率是多少?','合格率为什么下降?','最低合格率前五项是多少?'):
|
||||
altered=copy.deepcopy(task);altered['rootGoal']=original;altered['productResolution']['originalQuestion']=original
|
||||
self.assertFalse(_confirmed_scalar_selection(request,altered))
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Month-range evidence stays bound to its exact selected calendar window."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
|
||||
from analysis_visuals import build_visuals
|
||||
from nao_agent_report import _build_trend_report
|
||||
from nao_evidence_finalizer import _coverage_period, coverage_sufficient
|
||||
from nao_evidence_report import _frame_key, _same_periods, period, select_comparison_frames
|
||||
from nao_period import expected_months, normalize_period
|
||||
|
||||
|
||||
QUESTION = '查看客户甲2026年3至5月的综合合格率趋势图'
|
||||
RELEASE = 'release-range'
|
||||
FILTERS = {'extraFilters': {'customerCode': ['客户甲']}}
|
||||
|
||||
|
||||
def selected_task(period_value, *, period_coverage='available'):
|
||||
return {
|
||||
'version': 1, 'kind': 'metrics', 'question': QUESTION, 'goal': QUESTION,
|
||||
'period': period_value, 'comparisonPeriod': None, 'comparisonRequired': False,
|
||||
'filters': FILTERS, 'freshQuery': True, 'dimensions': [], 'dimensionStages': {},
|
||||
'stages': [], 'fields': ['rate'], 'historyMode': 'none', 'trendRequired': True,
|
||||
'periodCoverage': period_coverage, 'visualPreference': 'auto', 'visualType': None,
|
||||
'responseStyle': 'brief', 'verifySource': False, 'fullDetails': False,
|
||||
}
|
||||
|
||||
|
||||
def context(period_value, *, period_coverage='available'):
|
||||
task = selected_task(period_value, period_coverage=period_coverage)
|
||||
return {'releaseId': RELEASE, 'period': period_value, 'filters': FILTERS, '_agentTask': task}
|
||||
|
||||
|
||||
def metric_doc(selected_period, months, *, release=RELEASE, filters=None, ref='Q1234ABCD'):
|
||||
rates = {1: .71, 2: .72, 3: .73, 4: .74, 5: .75, 6: .76, 7: .77, 8: .78, 9: .79, 10: .80}
|
||||
trend = [
|
||||
{'period': f"2026-{month:02d}", 'compositeRate': rates[month],
|
||||
'black': {'rate': rates[month] - .01}, 'bright': {'rate': rates[month] + .01},
|
||||
'coverage': {'blackRows': 5, 'brightRows': 5}}
|
||||
for month in months
|
||||
]
|
||||
return {
|
||||
'evidenceRef': ref, 'mode': 'query', 'releaseId': release,
|
||||
'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': release, 'view': 'overall',
|
||||
'period': selected_period, 'filters': filters if filters is not None else FILTERS,
|
||||
'summary': {'compositeRate': .74}, 'trend': trend, 'trendCount': len(trend),
|
||||
'trendTruncated': False,
|
||||
'coverage': {'sourceMonths': [
|
||||
{'year': 2026, 'month': month, 'blackRows': 5, 'brightRows': 5} for month in months
|
||||
]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class MonthRangeEvidenceTests(unittest.TestCase):
|
||||
def test_period_helper_supports_ranges_and_preserves_single_month_quarter_year(self):
|
||||
self.assertIsNone(normalize_period(None))
|
||||
self.assertEqual(expected_months({'year': 2026, 'monthStart': 1, 'monthEnd': 9}), list(range(1, 10)))
|
||||
self.assertEqual(expected_months({'year': 2026, 'monthStart': 3, 'monthEnd': 5}), [3, 4, 5])
|
||||
self.assertEqual(expected_months({'year': 2026}), list(range(1, 13)))
|
||||
self.assertEqual(expected_months({'year': 2026, 'quarter': 2}), [4, 5, 6])
|
||||
self.assertEqual(expected_months({'year': 2026, 'month': 9}), [9])
|
||||
|
||||
def test_range_requires_paired_valid_ordered_endpoints(self):
|
||||
invalid = (
|
||||
{'year': 2026, 'monthStart': 3},
|
||||
{'year': 2026, 'monthEnd': 5},
|
||||
{'year': 2026, 'monthStart': 5, 'monthEnd': 3},
|
||||
{'year': 2026, 'monthStart': 0, 'monthEnd': 5},
|
||||
{'year': 2026, 'monthStart': 3, 'monthEnd': 13},
|
||||
{'year': 2026, 'monthStart': 3, 'monthEnd': 5, 'month': 4},
|
||||
{'year': 2026, 'monthStart': 3, 'monthEnd': 5, 'quarter': 2},
|
||||
)
|
||||
for value in invalid:
|
||||
with self.subTest(period=value), self.assertRaises(ValueError):
|
||||
normalize_period(value)
|
||||
|
||||
def test_report_renders_range_and_does_not_fill_an_absent_month_with_zero(self):
|
||||
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
|
||||
item = metric_doc(selected, [1, 2, 3, 5, 6, 7, 8, 9])
|
||||
report = _build_trend_report([item], context(selected), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
text = report['facts'][0]['text']
|
||||
self.assertIn('2026年1–9月', text)
|
||||
self.assertIn('1月71.00%', text)
|
||||
self.assertIn('4月未提供', text)
|
||||
self.assertIn('9月79.00%', text)
|
||||
self.assertNotIn('4月0.00%', text)
|
||||
|
||||
def test_short_range_includes_only_selected_months_and_rejects_outer_data(self):
|
||||
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
|
||||
item = metric_doc(selected, [3, 4, 5])
|
||||
report = _build_trend_report([item], context(selected), QUESTION)
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('3月73.00%', report['facts'][0]['text'])
|
||||
self.assertIn('5月75.00%', report['facts'][0]['text'])
|
||||
self.assertNotIn('2月', report['facts'][0]['text'])
|
||||
self.assertNotIn('6月', report['facts'][0]['text'])
|
||||
|
||||
out_of_range = metric_doc(selected, [2, 3, 4, 5])
|
||||
rejected = _build_trend_report([out_of_range], context(selected), QUESTION)
|
||||
self.assertFalse(rejected['coverageSufficient'])
|
||||
self.assertFalse(rejected['facts'])
|
||||
self.assertIn('期间以外', rejected['missingEvidence'][0])
|
||||
|
||||
def test_unknown_stage_rows_do_not_become_known_zero_in_a_composite_trend_point(self):
|
||||
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
item = metric_doc(selected, [1, 2, 3, 4, 5])
|
||||
february = next(row for row in item['result']['trend'] if row['period'] == '2026-02')
|
||||
february['coverage']['brightRows'] = None
|
||||
parent_february = next(row for row in item['result']['coverage']['sourceMonths']
|
||||
if row['month'] == 2)
|
||||
parent_february['brightRows'] = None
|
||||
report = _build_trend_report([item], context(selected), QUESTION)
|
||||
text = report['facts'][0]['text']
|
||||
self.assertIn('2月未提供', text)
|
||||
self.assertNotIn('2月72.00%', text)
|
||||
|
||||
def test_period_identity_and_release_scope_reject_other_range_evidence(self):
|
||||
large = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
|
||||
short = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
|
||||
self.assertNotEqual(_frame_key({'current': {'period': large}, 'comparison': {'period': {'year': 2025}},
|
||||
'filterBinding': FILTERS}),
|
||||
_frame_key({'current': {'period': short}, 'comparison': {'period': {'year': 2025}},
|
||||
'filterBinding': FILTERS}))
|
||||
self.assertEqual(period(short), '2026年3–5月')
|
||||
|
||||
def comparison(ref, current_period):
|
||||
return {
|
||||
'evidenceRef': ref, 'mode': 'compare',
|
||||
'result': {'kind': 'quality_comparison', 'releaseId': RELEASE,
|
||||
'filterBinding': FILTERS,
|
||||
'current': {'period': current_period},
|
||||
'comparison': {'period': {'year': 2025}},
|
||||
'delta': {'available': True, 'percentagePoints': 1.0}},
|
||||
}
|
||||
|
||||
roots, limitations = select_comparison_frames(
|
||||
[comparison('Q11111111', large), comparison('Q22222222', short)], context(short))
|
||||
self.assertEqual([root['evidenceRef'] for root in roots], ['Q22222222'])
|
||||
self.assertEqual(limitations, [])
|
||||
roots, limitations = select_comparison_frames([comparison('Q11111111', large)], context(short))
|
||||
self.assertTrue(limitations)
|
||||
|
||||
wrong = metric_doc(large, [3, 4, 5])
|
||||
report = _build_trend_report([wrong], context(short), QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertFalse(report['facts'])
|
||||
self.assertFalse(coverage_sufficient([wrong], QUESTION, context(short)))
|
||||
|
||||
expected = {'kind': 'quality_changes', 'currentPeriod': short,
|
||||
'comparisonPeriod': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}}
|
||||
wrong_frame = {'kind': 'quality_changes', 'currentPeriod': large,
|
||||
'comparisonPeriod': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}}
|
||||
self.assertTrue(_same_periods(expected, {
|
||||
'kind': 'quality_comparison',
|
||||
'current': {'period': short},
|
||||
'comparison': {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}},
|
||||
'filterBinding': FILTERS,
|
||||
}))
|
||||
self.assertFalse(_same_periods(wrong_frame, {
|
||||
'kind': 'quality_comparison',
|
||||
'current': {'period': short},
|
||||
'comparison': {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}},
|
||||
'filterBinding': FILTERS,
|
||||
}))
|
||||
|
||||
def test_complete_range_requires_every_selected_month_and_source_coverage(self):
|
||||
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
|
||||
item = metric_doc(selected, [3, 4, 5])
|
||||
item['result']['coverage']['sourceMonths'].pop(1)
|
||||
report = _build_trend_report([item], context(selected, period_coverage='complete'), QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn('全部月份', ' '.join(report['missingEvidence']))
|
||||
self.assertFalse(coverage_sufficient([item], QUESTION,
|
||||
context(selected, period_coverage='complete')))
|
||||
|
||||
missing_month = metric_doc(selected, [3, 5])
|
||||
report = _build_trend_report([missing_month], context(selected, period_coverage='complete'), QUESTION)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn('4月未提供', report['facts'][0]['text'])
|
||||
|
||||
def test_visual_scope_and_points_stay_inside_selected_range(self):
|
||||
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
|
||||
item = metric_doc(selected, [1, 2, 3, 5, 6, 7, 8, 9])
|
||||
visual_context = {
|
||||
**context(selected),
|
||||
'_agentTask': {
|
||||
'version': 1, 'kind': 'metrics', 'question': QUESTION,
|
||||
'visualPreference': 'required', 'visualType': 'line',
|
||||
'dimensions': [], 'stages': [], 'fields': ['rate'],
|
||||
},
|
||||
}
|
||||
result = build_visuals([item], QUESTION, 'range-visual', context=visual_context)
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
chart = next(chart for chart in result['visuals'] if chart.get('metric') == 'composite_rate')
|
||||
self.assertEqual(chart['scope']['year'], 2026)
|
||||
self.assertEqual(chart['scope']['monthStart'], 1)
|
||||
self.assertEqual(chart['scope']['monthEnd'], 9)
|
||||
self.assertEqual(chart['series'][0]['values'], [.71, .72, .73, None, .75, .76, .77, .78, .79])
|
||||
|
||||
outside = metric_doc(selected, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
|
||||
rejected = build_visuals([outside], QUESTION, 'range-visual-outside', context=visual_context)
|
||||
self.assertEqual(rejected['visuals'], [])
|
||||
|
||||
def test_finalizer_period_identity_accepts_range_exactly(self):
|
||||
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
|
||||
self.assertEqual(_coverage_period(selected), selected)
|
||||
self.assertIsNone(_coverage_period({'year': 2026, 'monthStart': 3}))
|
||||
self.assertIsNone(_coverage_period({'year': 2026, 'monthStart': 6, 'monthEnd': 5}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,76 @@
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
|
||||
from nao_metric_request import frozen_metric_frame
|
||||
from nao_quality_tools import QualityToolError, QualityTools, normalize_period
|
||||
from nao_query_contract import bind_query
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
|
||||
class MonthRangeTaskTests(unittest.TestCase):
|
||||
def test_range_survives_condition_changes_and_exact_query_binding(self):
|
||||
period = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
|
||||
context = {'_protocol': {'version': 1}, 'releaseId': 'fixed'}
|
||||
original = normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
|
||||
period=period, trendRequired=True, filterUpdates={'extraFilters': {'customerCode': ['800001']}}),
|
||||
'客户三至五月趋势', context)
|
||||
inherited = bind_task(context, original)
|
||||
updated = normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
|
||||
trendRequired=True, filterUpdates={'extraFilters': None}), '取消客户,期间不变', inherited)
|
||||
self.assertEqual(updated['period'], period)
|
||||
self.assertNotIn('extraFilters', updated['filters'])
|
||||
self.assertEqual(frozen_metric_frame(bind_task(context, updated))['period'], period)
|
||||
contract = {'task': updated, 'releaseId': 'fixed'}
|
||||
self.assertEqual(bind_query({'mode': 'query'}, contract)['period'], period)
|
||||
for wrong in ({'year': 2026}, {'year': 2026, 'month': 5},
|
||||
{'year': 2026, 'monthStart': 1, 'monthEnd': 5}):
|
||||
with self.assertRaises(ValueError):
|
||||
bind_query({'mode': 'query', 'period': wrong}, contract)
|
||||
|
||||
def test_invalid_ranges_are_rejected_by_planner_and_tools(self):
|
||||
for period in ({'year': 2026, 'monthStart': 1}, {'year': 2026, 'monthEnd': 9},
|
||||
{'year': 2026, 'monthStart': 9, 'monthEnd': 1},
|
||||
{'year': 2026, 'monthStart': True, 'monthEnd': 9},
|
||||
{'year': 2026, 'monthStart': 1, 'monthEnd': 13},
|
||||
{'year': 2026, 'monthStart': 1, 'monthEnd': 9, 'month': 9},
|
||||
{'year': 2026, 'monthStart': 1, 'monthEnd': 9, 'quarter': 3}):
|
||||
with self.subTest(period=period):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task(task_value(period=period), '月度趋势', {})
|
||||
with self.assertRaises(QualityToolError):
|
||||
normalize_period(period)
|
||||
|
||||
def test_metric_receipt_cannot_omit_or_expand_month_range(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
|
||||
response = {'summary': {}, 'coverage': {'rawRows': 1, 'includedRows': 1}, 'release': {'id': 'fixed'}, 'context': {**period, 'view': 'overall', 'includeUnassigned': False}}
|
||||
QualityTools._validate_source_snapshot(response, 'fixed', period, {}, 'overall')
|
||||
for changed in ({'year': 2026, 'view': 'overall'},
|
||||
{**period, 'monthEnd': 12, 'view': 'overall'}):
|
||||
with self.assertRaises(QualityToolError):
|
||||
QualityTools._validate_source_snapshot({**response, 'context': changed}, 'fixed', period, {}, 'overall')
|
||||
|
||||
def test_new_task_resets_root_goal_and_prompt_describes_ranges(self):
|
||||
previous = normalize_task(task_value(period={'year': 2025}), '旧目标', {})
|
||||
previous['rootGoal'] = '旧根目标'
|
||||
context = bind_task({'_protocol': {'version': 1}}, previous)
|
||||
proposal = task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
|
||||
goal='客户月度新目标', newTask=True, trendRequired=True,
|
||||
period={'year': 2026, 'monthStart': 1, 'monthEnd': 9})
|
||||
semantic = {'aligned': True, 'requiresEvidence': True, 'requiresTrend': True,
|
||||
'requiresComparison': False, 'requiresSourceVerification': False, 'requiresAllSourceRows': False}
|
||||
with patch('nao_agent_task._model_json', side_effect=[proposal, semantic]) as model:
|
||||
planned = plan_task(base_url='https://offline.invalid', api_key='synthetic', model='m',
|
||||
question='新任务:客户1至9月综合合格率趋势', context=context, cancel_event=threading.Event())
|
||||
self.assertEqual(planned['rootGoal'], '客户月度新目标')
|
||||
prompt = model.call_args_list[0].kwargs['system']
|
||||
self.assertIn('monthStart', prompt)
|
||||
self.assertIn('单独month只代表该月', prompt)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,919 @@
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_task import _explicit_read_safeguards, normalize_task, bind_task, plan_task, plan_repairs, TaskUnavailable
|
||||
from nao_analysis_coverage import coverage_gaps, plan_missing_queries, formal_metric_coverage, semantic_metric_gaps
|
||||
from nao_metric_request import metric_request
|
||||
from nao_evidence_finalizer import coverage_sufficient
|
||||
from test_nao_analysis_coverage import evidence
|
||||
from test_nao_metric_requests import metric_evidence
|
||||
|
||||
|
||||
def task_value(**changes):
|
||||
return {'goal': '完整理解本轮质量问题', 'kind': 'analysis', 'freshQuery': True,
|
||||
'dimensions': ['department', 'group', 'product', 'defect'], 'stages': ['black', 'bright'],
|
||||
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'verifySource': False,
|
||||
'fullDetails': False, 'visualPreference': 'auto', 'visualType': None,
|
||||
'responseStyle': 'standard', 'needsClarification': False, 'clarification': '',
|
||||
'filterUpdates': {}, **changes}
|
||||
|
||||
|
||||
def semantic_ok():
|
||||
return {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}
|
||||
|
||||
|
||||
class AgentTaskTest(unittest.TestCase):
|
||||
def context(self, question, **changes):
|
||||
return bind_task({}, normalize_task(task_value(**changes), question, {}))
|
||||
|
||||
def test_negated_lists_do_not_request_raw_rows_but_positive_siblings_still_do(self):
|
||||
for question in (
|
||||
'新任务:查询2026年9月客户800001在黑片和亮片中的全部成品规格清单,去重并显示各自来源阶段。这里只要成员清单,不要正式指标、排名或逐行原始记录。',
|
||||
'只要清单,不需要指标及全部原始记录',
|
||||
'不要图表或核验原件',
|
||||
):
|
||||
with self.subTest(question=question):
|
||||
self.assertEqual(_explicit_read_safeguards(question),{'requiresSourceVerification':False,'requiresAllSourceRows':False})
|
||||
for question in ('不要排名,但需要逐行原始记录','不要排名,逐行读取源记录','不要去重且逐行读取全部原始记录','不要图表或排名,请逐行读取原始记录'):
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
|
||||
|
||||
def test_current_read_negations_do_not_force_or_cancel_a_different_purpose(self):
|
||||
for text in ('不读逐行原始记录', '不读取全部原始记录', '不做逐条原始记录读取', '原始记录不需要逐行读取'):
|
||||
with self.subTest(text=text):
|
||||
self.assertFalse(_explicit_read_safeguards(text)['requiresAllSourceRows'])
|
||||
self.assertEqual(_explicit_read_safeguards('不要核验原件,但请逐行读取原始记录'),
|
||||
{'requiresSourceVerification': False, 'requiresAllSourceRows': True})
|
||||
self.assertEqual(_explicit_read_safeguards('不读逐行原始记录,但请核验归档原件'),
|
||||
{'requiresSourceVerification': True, 'requiresAllSourceRows': False})
|
||||
self.assertTrue(_explicit_read_safeguards('不要去重且逐行读取全部原始记录')['requiresAllSourceRows'])
|
||||
|
||||
def test_defect_only_metrics_plan_is_repaired_instead_of_answering_parent_input(self):
|
||||
replies=[task_value(kind='metrics',dimensions=['defect'],stages=['black'],fields=['totalWeight'],dimensionStages={'defect':['black']}),
|
||||
task_value(kind='analysis',dimensions=['defect'],stages=['black'],fields=['scrapWeight'],dimensionStages={'defect':['black']}),semantic_ok()]
|
||||
payloads=[]
|
||||
def opener(request,timeout):
|
||||
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
|
||||
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='查看具体缺陷排名',context={},cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(planned['kind'],'analysis')
|
||||
self.assertEqual(planned['fields'],['scrapWeight'])
|
||||
self.assertEqual(payloads[1]['validationDetails']['field'],'kind')
|
||||
|
||||
def test_semantic_coverage_is_independent_of_user_wording(self):
|
||||
for question in ['分析整体质量变化', '帮我全面看看最近的质量表现', '最近生产表现还好吗']:
|
||||
context = self.context(question)
|
||||
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 5)
|
||||
self.assertEqual(len(coverage_gaps([evidence('quality_comparison')], question, context)), 5)
|
||||
question = '哪些产品恶化最明显'
|
||||
context = self.context(question, dimensions=['product'])
|
||||
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 1)
|
||||
|
||||
def test_all_source_defect_fields_are_not_misread_as_all_raw_records(self):
|
||||
question = ('改为只看黑片,客户800001不变,期间仍是2026年8至9月与7月比较。'
|
||||
'新增加工单位等于机加工部,并排除规格18X15X4+Z;'
|
||||
'按缺陷维度列出全部来源细项重量和变化,保留柱状图。')
|
||||
requirements = _explicit_read_safeguards(question)
|
||||
self.assertFalse(requirements['requiresAllSourceRows'])
|
||||
self.assertFalse(requirements['requiresSourceVerification'])
|
||||
for aggregate_request in ('汇总全部来源细项重量', '列出所有源字段汇总',
|
||||
'比较全部缺陷类别重量', '逐条列出全部来源缺陷字段汇总'):
|
||||
with self.subTest(aggregate_request=aggregate_request):
|
||||
self.assertFalse(_explicit_read_safeguards(aggregate_request)['requiresAllSourceRows'])
|
||||
|
||||
def test_explicit_row_level_source_requests_still_require_full_details(self):
|
||||
for question in ('逐行读取原始记录', '读取全部源记录', '列出所有原始数据行',
|
||||
'输出全部原始行字段', '每条记录列出所有字段'):
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
|
||||
self.assertFalse(_explicit_read_safeguards('不需要逐行读取全部源记录')['requiresAllSourceRows'])
|
||||
|
||||
def test_semantic_metrics_accept_aliases_and_date_notation(self):
|
||||
question = '只看SYNTH-16X15+T,2026-09对2026-08,黑片亮片良率和投入、报废'
|
||||
value = task_value(kind='metrics', dimensions=[], filterUpdates={'product': 'SYNTH-16X15+T'},
|
||||
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8})
|
||||
context = bind_task({}, normalize_task(value, question))
|
||||
request = metric_request(question, context)
|
||||
self.assertIsNotNone(request)
|
||||
self.assertEqual(request['fields'], ['rate', 'totalWeight', 'scrapWeight'])
|
||||
contract = formal_metric_coverage(metric_evidence(), question, context)
|
||||
self.assertEqual(contract['missingEvidence'], [])
|
||||
self.assertEqual(len(contract['metrics']), 12)
|
||||
|
||||
def test_semantic_full_records_and_source_verification_cannot_pass_on_query_only(self):
|
||||
docs = [{'mode': 'query', 'result': {'kind': 'quality_query'}}]
|
||||
for question, updates in [('把每一条记录都列出来', {'kind': 'details', 'fullDetails': True}),
|
||||
('对照原始Excel核实数值', {'kind': 'source_verify', 'verifySource': True})]:
|
||||
self.assertFalse(coverage_sufficient(docs, question, self.context(question, dimensions=[], **updates)))
|
||||
|
||||
def test_scope_updates_keep_other_filters_and_release(self):
|
||||
context = {'releaseId': 'fixed-release', 'period': {'year': 2026, 'month': 9},
|
||||
'group': '汽车交通-海外', 'filters': {'group': '汽车交通-海外', 'product': 'A'}}
|
||||
task = normalize_task(task_value(filterUpdates={'group': '汽车交通-国内', 'product': None}), '那国内呢', context)
|
||||
bound = bind_task(context, task)
|
||||
self.assertEqual(bound['filters'], {'group': '汽车交通-国内'})
|
||||
self.assertNotIn('group', bound)
|
||||
self.assertEqual(bound['releaseId'], 'fixed-release')
|
||||
self.assertEqual(bound['period'], context['period'])
|
||||
for bad in ({'filterUpdates': {'releaseId': 'evil'}}, {'freshQuery': 'true'}, {'period': {'year': 2026, 'month': 13}}):
|
||||
with self.assertRaises(TaskUnavailable): normalize_task(task_value(**bad), 'q', context)
|
||||
|
||||
def test_recoverable_filter_shapes_preserve_existing_scope(self):
|
||||
context={'filters':{'department':'消费电子','product':'OLD'}}
|
||||
value=task_value(filterUpdates={'product':[' 18X15X4+T '], 'department':' '})
|
||||
self.assertEqual(normalize_task(value,'q',context)['filters'],{'department':'消费电子','product':'18X15X4+T'})
|
||||
value['filterUpdates']={'product':[]}
|
||||
self.assertEqual(normalize_task(value,'q',context)['filters'],context['filters'])
|
||||
value['filterUpdates']={'product':None}
|
||||
self.assertNotIn('product',normalize_task(value,'q',context)['filters'])
|
||||
value['filterUpdates']={'product':['A','B']}
|
||||
with self.assertRaises(TaskUnavailable) as err: normalize_task(value,'q',context)
|
||||
self.assertEqual(err.exception.field,'filterUpdates.product')
|
||||
|
||||
def test_model_plan_is_validated_and_retried_without_keyword_parser(self):
|
||||
replies = [{}, task_value(dimensions=['product'], freshQuery=True, dimensionStages={'product': []}),semantic_ok()]
|
||||
def opener(request, timeout):
|
||||
self.assertNotIn('apiKey', request.data.decode())
|
||||
value = replies.pop(0)
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(value)}}]}).encode())
|
||||
task = plan_task(base_url='https://example.test/v1', api_key='test-only', model='any-compatible-model',
|
||||
question='按最新数据再跑一遍', context={}, previous=None,
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertTrue(task['freshQuery'])
|
||||
self.assertEqual(task['dimensions'], ['product'])
|
||||
self.assertFalse(replies)
|
||||
|
||||
def test_retry_semantics_retain_prior_question_without_using_partial_answer(self):
|
||||
from unittest.mock import patch
|
||||
context={'releaseId':'fixed-release',
|
||||
'_serverQuestions':[{'turnId':'previous','question':'9月汽车交通的合格率为什么下降?','status':'PARTIAL'}]}
|
||||
wrong=task_value(goal='比较9月报废构成',dimensions=['group'],stages=['black'],fields=['scrapWeight'],
|
||||
comparisonRequired=False,dimensionStages={'group':['black']},period={'year':2026,'month':9})
|
||||
correct=task_value(goal='比较国内与海外对9月合格率下降的变化线索',dimensions=['group'],
|
||||
fields=['rate','totalWeight','scrapWeight'],comparisonRequired=True,dimensionStages={'group':[]},
|
||||
period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8})
|
||||
rejection={'aligned':False,'reason':'前文问合格率变化,不是单期报废构成'}
|
||||
replies=[wrong,rejection,rejection,correct,semantic_ok()]
|
||||
payloads=[]
|
||||
def response(**kwargs):
|
||||
payloads.append(kwargs['payload']);return replies.pop(0)
|
||||
with patch('nao_agent_task._model_json',side_effect=response):
|
||||
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='主要是国内还是国外的',context=context,cancel_event=threading.Event(),semantic_review=True)
|
||||
self.assertTrue(planned['comparisonRequired'])
|
||||
self.assertEqual(planned['comparisonPeriod'],{'year':2026,'month':8})
|
||||
self.assertEqual(planned['dimensionStages'],{'group':[]})
|
||||
self.assertEqual(planned['rootGoal'],'9月汽车交通的合格率为什么下降?')
|
||||
self.assertEqual(payloads[0]['previousUserQuestions'],context['_serverQuestions'])
|
||||
self.assertNotIn('answer',json.dumps(payloads[0]['previousUserQuestions']))
|
||||
self.assertFalse(replies)
|
||||
|
||||
def test_last_valid_plan_survives_false_rejection_only_with_two_consistent_reviews(self):
|
||||
from unittest.mock import patch
|
||||
first=task_value(dimensions='invalid')
|
||||
second=task_value(fields=['invalid'])
|
||||
final=task_value(goal='查询今年产品合格率趋势',kind='metrics',dimensions=[],
|
||||
fields=['rate'],stages=[],dimensionStages={},filterUpdates={'product':['18X15X4+T']},
|
||||
period={'year':2026})
|
||||
reject={'aligned':False,'reason':'产品分析缺少group'}
|
||||
replies=[first,second,final,reject,semantic_ok(),semantic_ok()]
|
||||
with patch('nao_agent_task._model_json',side_effect=replies) as call:
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年18X15X4+T合格率变化趋势',context={},cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['product'],'18X15X4+T')
|
||||
self.assertEqual(task['period'],{'year':2026})
|
||||
self.assertEqual(task['fields'],['rate'])
|
||||
|
||||
def test_filter_only_followup_reuses_prior_query_plan_and_recomputes_child_scopes(self):
|
||||
from unittest.mock import patch
|
||||
plan = [
|
||||
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':5,
|
||||
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
|
||||
'visualType':'pareto','visualPreference':'auto'},
|
||||
{'nodeId':'trend','purpose':'trend','fields':['scrapRate'],'stages':['black'],
|
||||
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
|
||||
'metric':'totalWeight','count':2},
|
||||
'visualType':'line','visualPreference':'auto'},
|
||||
]
|
||||
previous_value = task_value(kind='analysis',queryPlan=plan,dimensions=['product'],
|
||||
dimensionStages={'product':['black']},stages=['black'],fields=['totalWeight','scrapRate'],
|
||||
trendRequired=True,comparisonRequired=False,period={'year':2026},periodCoverage='available')
|
||||
previous = normalize_task(previous_value,'按产品重量排名后查看前两项趋势',{'_protocol':{'version':1}})
|
||||
context = {'_protocol':{'version':1},'_agentTask':previous,'_serverQuestions':[]}
|
||||
proposal = {'goal':'保持其他分析不变,只改部门筛选','kind':'analysis','freshQuery':True,
|
||||
'verifySource':False,'fullDetails':False,'needsClarification':False,
|
||||
'filterUpdates':{'department':'部门甲'},'periodCoverage':'available'}
|
||||
reviewed = {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,
|
||||
'requiresAllSourceRows':False,'requiresTrend':True,'requiresComparison':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[proposal,reviewed]):
|
||||
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='其他不变,只把部门改为部门甲',context=context,
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(planned['filters']['department'],'部门甲')
|
||||
self.assertEqual(planned['queryPlan'],previous['queryPlan'])
|
||||
self.assertEqual(planned['fields'],['totalWeight','scrapRate'])
|
||||
self.assertTrue(planned['trendRequired'])
|
||||
|
||||
def test_query_plan_prompt_explains_selector_objects_and_root_stage_union(self):
|
||||
from unittest.mock import patch
|
||||
nodes = [
|
||||
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':10,
|
||||
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
|
||||
'visualType':'pareto','visualPreference':'auto','comparisonRequired':False},
|
||||
{'nodeId':'trend','purpose':'trend','fields':['rate'],'stages':[],
|
||||
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
|
||||
'metric':'totalWeight','count':2},
|
||||
'visualType':'line','visualPreference':'auto','comparisonRequired':False},
|
||||
]
|
||||
invalid = task_value(kind='analysis',queryPlan=nodes,dimensions=['product'],
|
||||
dimensionStages={'product':['black']},stages=['black','bright'],
|
||||
fields=['rate','totalWeight'],trendRequired=True,comparisonRequired=False,
|
||||
period={'year':2026,'monthStart':1,'monthEnd':9})
|
||||
valid = {**invalid,'stages':['black']}
|
||||
review = {**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[invalid,valid,review]) as call:
|
||||
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='按黑片投入重量排名,再看前两项综合合格率走势',
|
||||
context={'_protocol':{'version':1}},cancel_event=threading.Event())
|
||||
self.assertEqual(planned['stages'],['black'])
|
||||
prompt = call.call_args_list[0].kwargs['system']
|
||||
self.assertIn('selector始终是带kind的JSON对象',prompt)
|
||||
self.assertIn('{"kind":"base"}',prompt)
|
||||
self.assertIn('{"kind":"catalog_members","dimension":"group","names":["部门甲-国内","部门甲-海外"]}',prompt)
|
||||
self.assertIn('{"kind":"ranking_top","sourceNodeId":"rank","stage":"black","metric":"totalWeight","count":2}',prompt)
|
||||
self.assertIn('{"kind":"previous_ranking","queryId":"trusted-query-id","stage":"black","metric":"totalWeight","ranks":[1,2]}',prompt)
|
||||
self.assertIn('root stages必须是所有节点stages数组的字面去重并集',prompt)
|
||||
self.assertIn('"stages":[]',prompt)
|
||||
self.assertIn('nodeId必须符合[a-z][a-z0-9_-]{0,31}',prompt)
|
||||
self.assertIn('root dimensionStages记录ranking/changes节点实际排名的维度和阶段',prompt)
|
||||
self.assertIn('没有ranking节点时必须为{}',prompt)
|
||||
self.assertIn('root dimensions来自ranking节点的dimension及selector的dimension',prompt)
|
||||
self.assertIn('父filters中的department不会自动成为分析维度',prompt)
|
||||
self.assertIn('产业部甲内比较国内和海外组:root dimensions=[group]、stages=[black]、dimensionStages={}',prompt)
|
||||
self.assertIn('"needsClarification":false',prompt)
|
||||
self.assertIn('多指定目录对象分别查询指标、比较或趋势时必须用queryPlan和catalog_members',prompt)
|
||||
self.assertIn('上轮排名对象只能用previous_ranking选择',prompt)
|
||||
self.assertIn('同比追问在root保留comparisonMode="yoy"',prompt)
|
||||
self.assertIn('若去年来源缺失,保留同比期间并如实返回部分结果',prompt)
|
||||
shapes = call.call_args_list[1].kwargs['payload']['requiredShapes']
|
||||
self.assertEqual(shapes['queryPlanSelectorExamples'][0],{'kind':'base'})
|
||||
self.assertEqual(shapes['queryPlanSelectorExamples'][2]['kind'],'ranking_top')
|
||||
self.assertIn('root stages=[black]',shapes['queryPlanRootExample'])
|
||||
self.assertIn('trend node fields=[rate], stages=[]',shapes['queryPlanRootExample'])
|
||||
self.assertIn('expands to two product scopes',shapes['queryPlanRootExample'])
|
||||
self.assertEqual(shapes['queryPlanNodeIdPattern'],
|
||||
'[a-z][a-z0-9_-]{0,31}; lowercase ASCII initial, 1..32 chars')
|
||||
self.assertIn('with no ranking nodes use {}',shapes['queryPlanDimensionStages'])
|
||||
self.assertIn('comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan',
|
||||
shapes['queryPlanRequiredWhenGroupedMetrics'])
|
||||
group_example = shapes['queryPlanGroupComparisonExample']
|
||||
self.assertFalse(group_example['needsClarification'])
|
||||
self.assertEqual(group_example['dimensions'],['group'])
|
||||
self.assertEqual(group_example['stages'],['black'])
|
||||
self.assertEqual(group_example['dimensionStages'],{})
|
||||
self.assertEqual(group_example['queryPlan'][0]['stages'],[])
|
||||
self.assertTrue(all(node['comparisonRequired'] for node in group_example['queryPlan']))
|
||||
self.assertEqual(call.call_count,3)
|
||||
|
||||
def test_protocol_v1_grouped_metrics_without_query_plan_enters_planner_repair(self):
|
||||
from unittest.mock import patch
|
||||
groups=['部门甲-国内','部门甲-海外']
|
||||
context={'_protocol':{'version':1},'_businessCatalog':{
|
||||
'departments':['部门甲'],'groups':groups,
|
||||
'groupDepartments':{name:'部门甲' for name in groups}}}
|
||||
question='比较部门甲国内和海外的综合合格率及黑片投入重量同比'
|
||||
legacy=task_value(goal=question,kind='metrics',dimensions=['group'],
|
||||
dimensionStages={'group':['black']},fields=['rate','totalWeight'],stages=['black'],
|
||||
trendRequired=False,comparisonRequired=True,comparisonMode='yoy',
|
||||
period={'year':2026,'month':9},filterUpdates={'department':'部门甲'},
|
||||
targetEntities={'group':groups})
|
||||
with self.assertRaises(TaskUnavailable) as err:
|
||||
normalize_task(legacy,question,context)
|
||||
self.assertEqual(err.exception.field,'queryPlan')
|
||||
self.assertEqual(err.exception.expected,'bounded node plan')
|
||||
# Non-protocol callers retain the legacy normalization behavior.
|
||||
self.assertTrue(normalize_task(legacy,question)['comparisonRequired'])
|
||||
|
||||
selector={'kind':'catalog_members','dimension':'group','names':groups}
|
||||
valid={**legacy,'dimensionStages':{},'queryPlan':[
|
||||
{'nodeId':'group_rate','purpose':'metrics','fields':['rate'],'stages':[],
|
||||
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
|
||||
{'nodeId':'group_weight','purpose':'metrics','fields':['totalWeight'],'stages':['black'],
|
||||
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
|
||||
]}
|
||||
review={**semantic_ok(),'requiresTrend':False,'requiresComparison':True,
|
||||
'targetEntities':{'group':groups}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[legacy,valid,review]) as call:
|
||||
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=question,context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(len(planned['queryPlan']),2)
|
||||
self.assertEqual(planned['comparisonMode'],'yoy')
|
||||
self.assertEqual(planned['comparisonPeriod'],{'year':2025,'month':9})
|
||||
self.assertEqual(call.call_args_list[1].kwargs['payload']['requiredShapes']['queryPlanRequiredWhenGroupedMetrics'],
|
||||
'protocol v1: nonempty dimensions plus rate/scrapRate fields, comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan; keep the full user requirement')
|
||||
self.assertEqual(call.call_count,3)
|
||||
|
||||
def test_last_valid_plan_survives_transient_review_protocol_failure(self):
|
||||
from unittest.mock import patch
|
||||
bad_shape=task_value(dimensions='invalid')
|
||||
bad_fields=task_value(fields=['invalid'])
|
||||
valid=task_value(goal='查看今年合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
|
||||
good={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
|
||||
responses=[bad_shape,bad_fields,valid,{'aligned':'true'},good]
|
||||
events=[]
|
||||
with patch('nao_agent_task._model_json',side_effect=responses) as call:
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年合格率走势',context={},cancel_event=threading.Event(),on_validation_error=events.append)
|
||||
self.assertTrue(task['trendRequired']);self.assertEqual(task['period'],{'year':2026})
|
||||
self.assertEqual(call.call_count,5)
|
||||
self.assertEqual(call.call_args_list[3].kwargs['payload']['plannedTask'],call.call_args_list[4].kwargs['payload']['plannedTask'])
|
||||
self.assertEqual([(e['phase'],e['reviewAttempt'],e['field'],e['receivedType']) for e in events if e.get('phase')=='task_review'],
|
||||
[('task_review',1,'aligned','str')])
|
||||
|
||||
def test_review_protocol_retry_is_bounded_and_keeps_valid_plan(self):
|
||||
from unittest.mock import patch
|
||||
plan=task_value(kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[])
|
||||
malformed={'aligned':True,'requiresEvidence':'true','requiresSourceVerification':False,'requiresAllSourceRows':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[plan,malformed,semantic_ok()]) as call:
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='查9月合格率',context={},cancel_event=threading.Event())
|
||||
self.assertEqual(result['kind'],'metrics');self.assertEqual(call.call_count,3)
|
||||
with patch('nao_agent_task._model_json',side_effect=[plan,TaskUnavailable('模型超时'),malformed]) as call:
|
||||
with self.assertRaises(TaskUnavailable) as err:
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='查9月合格率',context={},cancel_event=threading.Event())
|
||||
self.assertEqual(err.exception.field,'review.protocol');self.assertEqual(call.call_count,3)
|
||||
|
||||
def test_review_disagreement_never_bypasses_evidence_or_scope(self):
|
||||
from unittest.mock import patch
|
||||
value=task_value(dimensions=['group'],dimensionStages={'group':[]})
|
||||
rejection={'aligned':False,'reason':'范围错误'}
|
||||
contradictory={**semantic_ok(),'requiresSourceVerification':True}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,rejection,semantic_ok(),contradictory]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='比较各业务分组',context={},cancel_event=threading.Event())
|
||||
|
||||
def test_monthly_trend_is_structured_and_semantic_review_rejects_annual_total(self):
|
||||
from unittest.mock import patch
|
||||
annual=task_value(goal='查询年度合格率',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026})
|
||||
trend={**annual,'goal':'查询今年逐月合格率走势','trendRequired':True}
|
||||
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[annual,review,trend,review]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='看看今年良率怎么变化',context={},cancel_event=threading.Event())
|
||||
self.assertTrue(task['trendRequired']);self.assertEqual(task['periodCoverage'],'available')
|
||||
with self.assertRaises(TaskUnavailable):normalize_task({**annual,'trendRequired':'true'},'q')
|
||||
self.assertFalse(normalize_task(annual,'q')['trendRequired'])
|
||||
|
||||
def test_missing_trend_review_gets_bounded_independent_check(self):
|
||||
from unittest.mock import patch
|
||||
plan=task_value(goal='查询月度合格率走势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
|
||||
incomplete={**{k:v for k,v in semantic_ok().items() if k!='requiresTrend'},'requiresComparison':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{}, {'requiresTrend':True}]) as call:
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年的变化趋势',context={},cancel_event=threading.Event())
|
||||
self.assertTrue(task['trendRequired']);self.assertEqual(call.call_count,4)
|
||||
self.assertEqual(call.call_args_list[2].kwargs['payload']['plannedTask']['goal'],plan['goal'])
|
||||
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{},{}]) as call:
|
||||
with self.assertRaises(TaskUnavailable) as err:
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年的变化趋势',context={},cancel_event=threading.Event())
|
||||
self.assertEqual(err.exception.field,'requiresTrend.review');self.assertEqual(call.call_count,4)
|
||||
current={**plan,'goal':'查询9月合格率','period':{'year':2026,'month':9},'trendRequired':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[current,dict(incomplete),{'requiresTrend':False}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='9月合格率',context={},cancel_event=threading.Event())
|
||||
self.assertFalse(task['trendRequired'])
|
||||
|
||||
def test_pure_trend_does_not_add_a_comparison_period_but_yoy_keeps_it(self):
|
||||
from unittest.mock import patch
|
||||
plan=task_value(goal='今年月度合格率趋势',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],
|
||||
stages=[],period={'year':2026},trendRequired=True,comparisonRequired=True)
|
||||
pure={**plan,'comparisonRequired':False}
|
||||
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
|
||||
with patch('nao_agent_task._model_json',side_effect=[plan,verdict,pure,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年合格率每月走势',context={},cancel_event=threading.Event())
|
||||
self.assertTrue(task['trendRequired']);self.assertFalse(task['comparisonRequired'])
|
||||
self.assertIsNone(task['comparisonPeriod'])
|
||||
yoy={**plan,'goal':'今年月度合格率同比趋势','comparisonPeriod':{'year':2025}}
|
||||
yoy_review={**semantic_ok(),'requiresTrend':True,'requiresComparison':True}
|
||||
with patch('nao_agent_task._model_json',side_effect=[yoy,yoy_review]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年合格率趋势和去年同比',context={},cancel_event=threading.Event())
|
||||
self.assertTrue(task['comparisonRequired']);self.assertEqual(task['comparisonPeriod'],{'year':2025})
|
||||
|
||||
def test_ambiguous_catalogue_requires_choice_and_keeps_original_trend_for_followup(self):
|
||||
from unittest.mock import patch
|
||||
candidates=[{'name':'18X15X4+T','departments':[],'groups':[]},
|
||||
{'name':'18X15X4+Z','departments':[],'groups':[]}]
|
||||
product_match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
|
||||
context={'_businessCatalog':{'productMatch':product_match},
|
||||
'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
|
||||
value=task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
|
||||
filterUpdates={'product':'18X15X4+T'})
|
||||
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'single'}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,review]):
|
||||
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
|
||||
self.assertTrue(pending['needsClarification']);self.assertNotIn('product',pending['filters'])
|
||||
self.assertIn('18X15X4+T',pending['clarification']);self.assertIn('18X15X4+Z',pending['clarification'])
|
||||
self.assertEqual(pending['productResolution']['originalQuestion'],'今年18x15x4合格率变化趋势')
|
||||
selected={'status':'exact','candidates':[candidates[0]],'candidateCount':1,'truncated':False,
|
||||
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
|
||||
context['_businessCatalog']['productMatch']=selected
|
||||
# Simulate a planner forgetting the prior year and trend and the only
|
||||
# server history being an unrelated greeting.
|
||||
short=task_value(goal='确认选项',kind='metrics',dimensions=['product'],dimensionStages={'product':[]},
|
||||
fields=[],stages=[],filterUpdates={})
|
||||
with patch('nao_agent_task._model_json',side_effect=[short,review]) as model:
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
|
||||
self.assertFalse(task['needsClarification']);self.assertEqual(task['filters']['product'],'18X15X4+T')
|
||||
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
|
||||
self.assertTrue(task['trendRequired']);self.assertEqual(task['dimensions'],[])
|
||||
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
|
||||
self.assertEqual(task['productResolution']['selectedName'],'18X15X4+T')
|
||||
self.assertEqual(model.call_args_list[1].kwargs['payload']['pendingProductQuestion'],task['rootGoal'])
|
||||
selected['candidates']=[{'name':'18X15X4+OTHER'}]
|
||||
with patch('nao_agent_task._model_json',side_effect=[short]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
|
||||
|
||||
def test_selected_product_is_bound_even_with_static_base_query_plan(self):
|
||||
from unittest.mock import patch
|
||||
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
|
||||
previous=normalize_task(task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',
|
||||
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
|
||||
trendRequired=True,needsClarification=True,clarification='请选择规格'), '今年18x15x4合格率变化趋势', {})
|
||||
previous['filters']={'exclusions':{'product':['18X15X4.1+Z']}}
|
||||
previous['productResolution']={'originalQuestion':'今年18x15x4合格率变化趋势',
|
||||
'productMatch':{'status':'ambiguous','candidates':candidates}}
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
|
||||
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
|
||||
'previousQuestion':'今年18x15x4合格率变化趋势'}}}
|
||||
value=task_value(goal='查询已选规格的综合合格率',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9},queryPlan=[{
|
||||
'nodeId':'value','purpose':'metrics','fields':['rate'],'stages':[],
|
||||
'selector':{'kind':'base'},'visualPreference':'none','visualType':None}])
|
||||
repaired={**value,'trendRequired':True,'period':{'year':2026},'queryPlan':[{
|
||||
**value['queryPlan'][0],'purpose':'trend','visualType':'line'}]}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,repaired,{**semantic_ok(),'requiresTrend':True}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['product'],'18X15X4+T')
|
||||
self.assertEqual(task['period'],{'year':2026})
|
||||
self.assertTrue(task['trendRequired'])
|
||||
self.assertEqual(task['queryPlan'][0]['selector'],{'kind':'base'})
|
||||
self.assertEqual(task['queryPlan'][0]['purpose'],'trend')
|
||||
self.assertEqual(task['filters']['exclusions']['product'],['18X15X4.1+Z'])
|
||||
|
||||
def test_new_analysis_without_supported_execution_route_is_repaired_before_confirmation(self):
|
||||
from unittest.mock import patch
|
||||
value=task_value(goal='查询9月综合合格率',kind='analysis',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'kind':'metrics'}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='查2026年9月综合合格率',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['kind'],'metrics');self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_explicit_history_only_report_cannot_restart_prior_investigation(self):
|
||||
from unittest.mock import patch
|
||||
from nao_native_planning import history_only_requested
|
||||
question='请用本会话第一轮已核实证据生成分析报告,不重新查询数据。'
|
||||
self.assertTrue(history_only_requested(question))
|
||||
self.assertFalse(history_only_requested('不用报告,请重新查询2026年9月数据'))
|
||||
self.assertFalse(history_only_requested('不重新查询历史数据,但请重新查询最新数据'))
|
||||
bad=task_value(goal='重新分析9月下降原因',kind='analysis',executionMode='native_agent',
|
||||
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
good=task_value(goal='复用第一轮证据整理报告',kind='report',historyMode='first',freshQuery=False,
|
||||
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False,trendRequired=False)
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=question,context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['kind'],'report');self.assertFalse(task['freshQuery'])
|
||||
self.assertEqual(task['historyMode'],'first');self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_first_history_mode_requires_new_autonomous_report(self):
|
||||
value=task_value(kind='report',historyMode='first',freshQuery=False,
|
||||
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
|
||||
for context in ({}, {'_protocol':{'version':1,'autonomousEnabled':False}}):
|
||||
with self.subTest(context=context), self.assertRaises(TaskUnavailable):
|
||||
normalize_task(value,'整理首轮结果',context)
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task({**value,'kind':'restate'},'整理首轮结果',
|
||||
{'_protocol':{'version':1,'autonomousEnabled':True}})
|
||||
|
||||
def test_history_no_read_phrasings_repair_before_any_fresh_query(self):
|
||||
from unittest.mock import patch
|
||||
from nao_native_planning import history_only_requested
|
||||
questions=('仅用本会话第一轮已核实结果生成报告,不要开展新查询',
|
||||
'基于第一轮已核实结果,不做新的查询,生成报告',
|
||||
'只整理第一轮已核实结果,不需要重新读取数据')
|
||||
bad=task_value(goal='重新查数',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
good=task_value(goal='整理第一轮已核实结果',kind='report',historyMode='first',freshQuery=False,
|
||||
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
|
||||
for question in questions:
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(history_only_requested(question))
|
||||
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=question,context={'_protocol':{'version':1,'autonomousEnabled':True}},
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(task['historyMode'],'first')
|
||||
self.assertFalse(task['freshQuery'])
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_first_history_request_cannot_silently_select_latest(self):
|
||||
from unittest.mock import patch
|
||||
value=task_value(goal='整理首轮已核实证据',kind='report',historyMode='last',freshQuery=False,
|
||||
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'historyMode':'first'}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='用本会话第一轮已核实证据整理报告,不重新查询数据。',
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True}},cancel_event=threading.Event())
|
||||
self.assertEqual(task['historyMode'],'first')
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_selection_completes_pending_empty_metric_intent_without_erasing_scope(self):
|
||||
from unittest.mock import patch
|
||||
original='2026年9月18X15X4合格率是多少,不含18X15X4.1'
|
||||
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
|
||||
previous=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
|
||||
fields=[],stages=[],period={'year':2026,'month':9},needsClarification=True,
|
||||
clarification='请选择规格',filterUpdates={'exclusions':{'product':['18X15X4.1+Z']}}),original,{})
|
||||
previous['productResolution']={'originalQuestion':original,'productMatch':{
|
||||
'status':'ambiguous','candidates':candidates}}
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
|
||||
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
|
||||
'previousQuestion':original}}}
|
||||
value=task_value(goal='查询选定规格本期综合率',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
with patch('nao_agent_task._model_json',side_effect=[{**value,'kind':'analysis'}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
|
||||
self.assertEqual(task['fields'],['rate']);self.assertEqual(task['kind'],'analysis')
|
||||
from nao_protocol_execution import _confirmed_scalar_selection
|
||||
self.assertTrue(_confirmed_scalar_selection({'_autonomousAnswer':True},task))
|
||||
self.assertEqual(task['filters'],{'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}})
|
||||
self.assertEqual(task['rootGoal'],original)
|
||||
|
||||
def test_ambiguous_candidates_remain_clarifiable_when_planner_fails(self):
|
||||
from unittest.mock import patch
|
||||
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
|
||||
match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
|
||||
context={'_businessCatalog':{'productMatch':match}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[{}, {}, {}]):
|
||||
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
|
||||
self.assertTrue(pending['needsClarification']);self.assertTrue(pending['productResolution']['plannerDeferred'])
|
||||
self.assertIn('18X15X4+T',pending['clarification']);self.assertNotIn('product',pending['filters'])
|
||||
context['_businessCatalog']['productMatch']={'status':'exact','candidates':[candidates[0]],
|
||||
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
|
||||
planned=task_value(goal='今年18X15X4+T合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[planned,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertTrue(task['trendRequired'])
|
||||
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
|
||||
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
|
||||
|
||||
def test_customer_code_trend_stays_authorized_and_bound_in_queries_and_followup(self):
|
||||
from unittest.mock import patch
|
||||
from nao_query_contract import bind_query
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
context={'_businessCatalog':catalog,'filters':{'department':'消费电子','extraFilters':{'blankProducer':['Y']}}}
|
||||
value=task_value(goal='查看客户代码 C-2026 的今年月度合格率走势',kind='metrics',
|
||||
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
|
||||
trendRequired=True,filterUpdates={'extraFilters':{'customerCode':'C-2026'}})
|
||||
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,review]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码 C-2026 今年的合格率趋势',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters'],{'department':'消费电子','extraFilters':{'blankProducer':['Y'],'customerCode':['C-2026']}})
|
||||
self.assertEqual(task['period'],{'year':2026});self.assertTrue(task['trendRequired'])
|
||||
query=bind_query({'mode':'query'}, {'task':task,'releaseId':'fixed'})
|
||||
self.assertEqual(query['filters']['extraFilters']['customerCode'],['C-2026'])
|
||||
with self.assertRaises(ValueError):
|
||||
bind_query({'mode':'query','filters':{'extraFilters':{'customerCode':['OTHER']}}},
|
||||
{'task':task,'releaseId':'fixed'})
|
||||
followup=task_value(goal='继续看客户 C-2026 的今年月度合格率趋势',kind='metrics',
|
||||
dimensions=[],dimensionStages={},fields=['rate'],stages=[],trendRequired=True,
|
||||
filterUpdates={})
|
||||
next_context=bind_task(context,task)
|
||||
with patch('nao_agent_task._model_json',side_effect=[followup,review]):
|
||||
next_task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='换成折线图',context=next_context,previous=task,cancel_event=threading.Event())
|
||||
self.assertEqual(next_task['filters']['extraFilters']['customerCode'],['C-2026'])
|
||||
self.assertEqual(next_task['period'],{'year':2026});self.assertTrue(next_task['trendRequired'])
|
||||
denied={'_businessCatalog':{'extraFields':[]}}
|
||||
with self.assertRaises(TaskUnavailable):normalize_task(value,'q',denied)
|
||||
|
||||
def test_customer_code_cannot_be_lost_when_planner_and_review_disagree(self):
|
||||
from unittest.mock import patch
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
value=task_value(goal='查看客户代码 C-2026 今年月度合格率趋势',kind='metrics',
|
||||
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
|
||||
trendRequired=True,filterUpdates={})
|
||||
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C-2026'])
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':{'extraFields':[]}},cancel_event=threading.Event())
|
||||
invented={**verdict,'customerCodes':['80099999']}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,invented]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
|
||||
def test_customer_filter_is_not_invented_or_retained_after_cancellation(self):
|
||||
from unittest.mock import patch
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C-2026']}}}
|
||||
keep=task_value(goal='今年整体合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
|
||||
period={'year':2026},filterUpdates={})
|
||||
clear={**keep,'filterUpdates':{'extraFilters':{'customerCode':None}}}
|
||||
verdict={**semantic_ok(),'customerCodes':[]}
|
||||
with patch('nao_agent_task._model_json',side_effect=[keep,verdict,clear,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='不按客户筛选,看今年整体合格率',context=context,cancel_event=threading.Event())
|
||||
self.assertNotIn('customerCode',task['filters'].get('extraFilters',{}))
|
||||
invented={**keep,'filterUpdates':{'extraFilters':{'customerCode':['INVENTED']}}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[invented,verdict]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年整体合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
|
||||
def test_customer_code_switch_preserves_exact_new_case(self):
|
||||
from unittest.mock import patch
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C1']}}}
|
||||
value=task_value(goal='客户代码 c1 今年合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
|
||||
period={'year':2026},filterUpdates={'extraFilters':{'customerCode':['c1']}})
|
||||
verdict={**semantic_ok(),'customerCodes':['c1']}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码 C1 换成 c1,看今年合格率',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['extraFilters']['customerCode'],['c1'])
|
||||
|
||||
def test_missing_customer_subject_is_rechecked_without_replanning(self):
|
||||
from unittest.mock import patch
|
||||
value=task_value(goal='客户代码C1合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],filterUpdates={})
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{}, {'customerCodes':['C1']}]) as call:
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C1']);self.assertEqual(call.call_count,4)
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{},{}]) as call:
|
||||
with self.assertRaises(TaskUnavailable) as err:
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertEqual(err.exception.field,'customerCodes.review');self.assertEqual(call.call_count,4)
|
||||
|
||||
def test_product_shaped_customer_code_is_not_automatically_a_product(self):
|
||||
from unittest.mock import patch
|
||||
from nao_product_identification import _normalized
|
||||
question='客户代码“18x15x4”今年合格率趋势'
|
||||
token='18x15x4';at=_normalized(question).index(token)
|
||||
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
|
||||
mention={'status':'ambiguous','query':token,'normalizedSpan':[at,at+len(token)],
|
||||
'candidates':candidates,'candidateCount':2,'truncated':False}
|
||||
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}],
|
||||
'productMatch':{**mention,'mentions':[mention]}}
|
||||
value=task_value(goal='客户代码18x15x4今年按月合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
|
||||
filterUpdates={'extraFilters':{'customerCode':[token]}})
|
||||
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':[token],'productScope':'single'}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=question,context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertFalse(task['needsClarification']);self.assertNotIn('product',task['filters'])
|
||||
self.assertEqual(task['filters']['extraFilters']['customerCode'],[token])
|
||||
double='客户代码18x15x4和产品18x15x4今年合格率趋势'
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=double,context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertTrue(task['needsClarification']);self.assertIn('18X15X4+T',task['clarification'])
|
||||
|
||||
def test_single_product_analysis_cannot_lose_product_filter(self):
|
||||
from unittest.mock import patch
|
||||
match={'status':'exact','query':'18x15x4','candidates':[{'name':'18X15X4+T'}],
|
||||
'candidateCount':1,'truncated':False}
|
||||
catalog={'productMatch':match}
|
||||
value=task_value(goal='为什么18X15X4+T合格率下降',kind='analysis',dimensions=['product'],
|
||||
dimensionStages={'product':[]},fields=['rate'],stages=[],filterUpdates={},comparisonRequired=True)
|
||||
verdict={**semantic_ok(),'productScope':'single'}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='为什么18X15X4+T合格率下降',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertNotIn('product',task['dimensions'])
|
||||
comparison={**verdict,'productScope':'compare'}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,comparison]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='把18X15X4+T与其他产品比较',context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertNotIn('product',task['filters']);self.assertIn('product',task['dimensions'])
|
||||
|
||||
def test_product_mentioned_as_background_is_not_automatically_applied(self):
|
||||
from unittest.mock import patch
|
||||
name='18X15X4+T';match={'status':'exact','query':name,'candidates':[{'name':name}],'candidateCount':1,'truncated':False}
|
||||
catalog={'productMatch':match,'mentionedProducts':[name]}
|
||||
value=task_value(goal='先不看18X15X4+T,查看今年总体合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,filterUpdates={})
|
||||
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'other'}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,review]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=value['goal'],context={'_businessCatalog':catalog},cancel_event=threading.Event())
|
||||
self.assertNotIn('product',task['filters'])
|
||||
|
||||
def test_greeting_does_not_become_the_business_root_question(self):
|
||||
from unittest.mock import patch
|
||||
value=task_value(goal='今年客户代码C1合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
|
||||
context={'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='今年客户代码C1合格率趋势',context=context,cancel_event=threading.Event())
|
||||
self.assertNotEqual(task['rootGoal'],'你好')
|
||||
|
||||
def test_retry_does_not_accept_an_unverified_semantic_review(self):
|
||||
from unittest.mock import patch
|
||||
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
|
||||
proposal=task_value(dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[proposal,{'aligned':'yes'},{'aligned':'yes'}]) as call:
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='主要是哪一组',context=context,cancel_event=threading.Event(),semantic_review=True)
|
||||
self.assertEqual(call.call_count,3)
|
||||
|
||||
def test_retry_comparison_replans_a_history_only_answer_as_a_fresh_analysis(self):
|
||||
from unittest.mock import patch
|
||||
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
|
||||
prior=task_value(kind='explain',freshQuery=False,historyMode='last',dimensions=[],fields=[],comparisonRequired=True)
|
||||
analysis=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[prior,analysis,semantic_ok()]) as call:
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='哪个分组',
|
||||
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
|
||||
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery']);self.assertEqual(call.call_count,3)
|
||||
|
||||
def test_unverified_business_claim_cannot_use_direct_conversation(self):
|
||||
from unittest.mock import patch
|
||||
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
|
||||
bad={'kind':'explain','goal':'判断合格率下降主要来自哪个分组',
|
||||
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
|
||||
'answer':'主要是国内分组所致;国内合格率降幅明显更大,海外基本持平。'}}
|
||||
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[bad,good,semantic_ok()]):
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
|
||||
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
|
||||
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
|
||||
|
||||
def test_direct_response_requiring_evidence_is_replanned(self):
|
||||
from unittest.mock import patch
|
||||
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
|
||||
bad={'kind':'explain','goal':'判断哪个分组需要关注',
|
||||
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
|
||||
'answer':'国内是重点关注对象,海外情况稳定。'}}
|
||||
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},
|
||||
good,semantic_ok()]):
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
|
||||
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
|
||||
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
|
||||
|
||||
def test_first_question_direct_response_is_reviewed_for_evidence(self):
|
||||
from unittest.mock import patch
|
||||
bad={'kind':'explain','goal':'判断哪个分组需要关注',
|
||||
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
|
||||
'answer':'国内是重点关注对象,海外情况稳定。'}}
|
||||
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},good,semantic_ok()]):
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='9月合格率下降主要是国内还是国外',context={},cancel_event=threading.Event())
|
||||
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
|
||||
|
||||
def test_direct_response_review_without_evidence_decision_fails_closed(self):
|
||||
from unittest.mock import patch
|
||||
direct={'kind':'explain','goal':'回应问候',
|
||||
'directResponse':{'purpose':'greeting','requiresMemory':False,'answer':'你好。'}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[direct,{'aligned':True}]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='你好',context={},cancel_event=threading.Event())
|
||||
|
||||
def test_group_targets_are_bound_from_semantic_plan_and_scoped_catalog(self):
|
||||
from unittest.mock import patch
|
||||
groups=['汽车交通-国内','汽车交通-海外','其他部门-国内']
|
||||
context={'_businessCatalog':{'groups':groups,'groupDepartments':{'汽车交通-国内':'新能源汽车磁钢',
|
||||
'汽车交通-海外':'新能源汽车磁钢','其他部门-国内':'其他部门'}}}
|
||||
value=task_value(goal='比较国内与海外的合格率变化',dimensions=['group'],dimensionStages={'group':[]},
|
||||
filterUpdates={'department':'新能源汽车磁钢'})
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='主要是国内还是国外的',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(result['targetEntities']['group'],groups[:2])
|
||||
value.update(goal='比较这两个地区',targetEntities={'group':groups[:2]})
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='请比较这两个地区',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(result['targetEntities']['group'],groups[:2])
|
||||
|
||||
def test_planner_repairs_with_invalid_plan_and_field_contract_then_continues(self):
|
||||
replies=[task_value(dimensions='unknown'),task_value(fields=['unsupported']),task_value(dimensions=['product'],dimensionStages={'product': []}),semantic_ok()]
|
||||
payloads=[]
|
||||
def opener(request,timeout):
|
||||
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='绘制产品图',
|
||||
context={},cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(task['dimensions'],['product'])
|
||||
self.assertEqual(len(payloads),4)
|
||||
self.assertEqual(payloads[1]['invalidPlan']['dimensions'],'unknown')
|
||||
self.assertEqual(payloads[1]['requiredShapes']['dimensions']['type'],'array')
|
||||
self.assertIn('validationFeedback',payloads[2])
|
||||
self.assertFalse(replies)
|
||||
|
||||
def test_repair_plan_cannot_widen_pinned_scope_or_period(self):
|
||||
task = normalize_task(task_value(filterUpdates={'group': '消费电子-美洲'},
|
||||
period={'year': 2026, 'month': 9}), 'q')
|
||||
def run(query):
|
||||
def opener(request, timeout):
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({'queries': [query]})}}]}).encode())
|
||||
return plan_repairs(base_url='https://example.test/v1', api_key='test-only', model='m', task=task,
|
||||
evidence=[], gaps=['missing'], failures=[], release_id='fixed',
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertEqual(run({'mode': 'compare', 'filters': {}})[0]['filters']['group'],'消费电子-美洲')
|
||||
with self.assertRaises(TaskUnavailable): run({'mode':'compare','filters':{'group':'其他分组'}})
|
||||
with self.assertRaises(TaskUnavailable): run({'mode': 'execute_sql', 'sql': 'SELECT 1'})
|
||||
with self.assertRaises(TaskUnavailable): run({'mode': 'query', 'filters': task['filters'], 'period': {'year': 2026, 'month': 8}})
|
||||
self.assertEqual(run({'mode': 'query', 'filters': task['filters'], 'period': task['period'], 'releaseId': 'other'})[0]['releaseId'], 'fixed')
|
||||
self.assertEqual(run({'mode': 'query', 'filters': task['filters']})[0]['period'], task['period'])
|
||||
|
||||
def test_single_period_metrics_bind_scope_stage_fields_and_source_months(self):
|
||||
import copy
|
||||
question = '九月黑片良率告诉我一下'
|
||||
value = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 9}, filterUpdates={'product': 'SYNTH-16X15+T'})
|
||||
context = bind_task({'releaseId': 'release-a'}, normalize_task(value, question))
|
||||
side = metric_evidence()[0]['result']['current']
|
||||
docs = [{'mode': 'query', 'result': {'kind': 'quality_metrics', 'releaseId': 'release-a',
|
||||
'period': side['period'], 'filters': context['filters'], 'summary': side, 'coverage': side['coverage']}}]
|
||||
self.assertEqual(semantic_metric_gaps(docs, question, context), [])
|
||||
for mutation in ('month', 'product', 'rate', 'coverage'):
|
||||
bad = copy.deepcopy(docs)
|
||||
result = bad[0]['result']
|
||||
if mutation == 'month': result['period']['month'] = 8
|
||||
if mutation == 'product': result['filters']['product'] = 'OTHER'
|
||||
if mutation == 'rate': result['summary']['black']['rate'] = None
|
||||
if mutation == 'coverage': result['coverage']['sourceMonths'][0]['blackRows'] = 0
|
||||
with self.subTest(mutation=mutation):
|
||||
self.assertTrue(semantic_metric_gaps(bad, question, context))
|
||||
|
||||
def test_client_task_cannot_bypass_server_planner(self):
|
||||
from nao_quality_runtime import _normalize_request
|
||||
import uuid
|
||||
payload = {key: str(uuid.uuid4()) for key in ('turnId', 'sessionId', 'actorId', 'releaseId')}
|
||||
payload.update(question='q', accessToken='test-only', context={'_agentTask': {'freshQuery': False},
|
||||
'agentContext': {'filters': {}}, '_resolvedMetricRequest': {}})
|
||||
normalized = _normalize_request(payload)
|
||||
self.assertEqual(normalized['context'], {})
|
||||
|
||||
def test_overall_task_keeps_primary_frame_when_native_drills_into_a_group(self):
|
||||
from nao_evidence_report import select_comparison_frames
|
||||
question = '9月份合格率下降原因'
|
||||
context = self.context(question, dimensions=[], period={'year': 2026, 'month': 9}, comparisonRequired=True)
|
||||
overall = evidence('quality_comparison')
|
||||
local = evidence('quality_comparison', filters={'group': '汽车交通-国内'}, ref='Q00000002')
|
||||
roots, limitations = select_comparison_frames([overall, local], context)
|
||||
self.assertEqual(roots, [overall])
|
||||
self.assertEqual(limitations, [])
|
||||
|
||||
|
||||
class WeightRankingDefaultCoverageTests(unittest.TestCase):
|
||||
def test_weight_ranking_without_explicit_chart_type_defaults_to_published_months(self):
|
||||
value=task_value(kind='analysis', dimensions=['product'], stages=['bright'], fields=['totalWeight'],
|
||||
trendRequired=False, comparisonRequired=False, visualType=None)
|
||||
self.assertEqual(normalize_task(value,'今年亮片投入重量按产品排名')['periodCoverage'],'available')
|
||||
value['periodCoverage']='complete'
|
||||
self.assertEqual(normalize_task(value,'要求完整全年齐全')['periodCoverage'],'complete')
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Regression cases for complete, release-bound broad analysis."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
|
||||
|
||||
|
||||
PERIOD = {'year': 2026, 'month': 9}
|
||||
PRIOR = {'year': 2026, 'month': 8}
|
||||
|
||||
|
||||
def evidence(kind, *, dimension=None, stage=None, filters=None, sort=None, ref='Q00000001', available=True):
|
||||
result = {'kind': kind, 'releaseId': 'release-a'}
|
||||
if kind == 'quality_comparison':
|
||||
result.update(filterBinding=filters or {}, current={'period': PERIOD, 'compositeRate': .95},
|
||||
comparison={'period': PRIOR, 'compositeRate': .96}, delta={'available': True, 'percentagePoints': -1})
|
||||
else:
|
||||
result.update(filters=filters or {}, currentPeriod=PERIOD, comparisonPeriod=PRIOR,
|
||||
dimension=dimension, stage=stage, sort=sort)
|
||||
if kind == 'quality_fine_defects':
|
||||
result['period'] = PERIOD
|
||||
result['coverage'] = {'available': available}
|
||||
result['comparisonCoverage'] = {'available': available}
|
||||
result['category'] = None
|
||||
result['defect'] = None
|
||||
return {'evidenceRef': ref, 'result': result}
|
||||
|
||||
|
||||
class AnalysisCoverageTest(unittest.TestCase):
|
||||
def test_broad_question_plans_only_missing_dimensions_in_same_frame(self):
|
||||
docs = [evidence('quality_comparison', filters={'group': '汽车交通-国内'}),
|
||||
evidence('quality_changes', dimension='department', filters={'group': '汽车交通-国内'}, sort='scrap_delta')]
|
||||
planned = plan_missing_queries(docs, '分析整体质量变化', {'year': 2026, 'month': 9, 'group': '汽车交通-国内'})
|
||||
self.assertEqual([(item['mode'], item.get('dimension'), item.get('stage')) for item in planned],
|
||||
[('changes', 'group', None), ('changes', 'product', None),
|
||||
('defects', 'defect', 'black'), ('defects', 'defect', 'bright')])
|
||||
self.assertTrue(all(item['filters'] == {'group': '汽车交通-国内'} and item['releaseId'] == 'release-a'
|
||||
and item['period'] == PERIOD and item['comparisonPeriod'] == PRIOR for item in planned))
|
||||
self.assertEqual(len(coverage_gaps(docs, '分析整体质量变化', {'year': 2026, 'month': 9, 'group': '汽车交通-国内'})), 4)
|
||||
|
||||
def test_complete_broad_question_and_narrow_number(self):
|
||||
docs = [evidence('quality_comparison')]
|
||||
docs += [evidence('quality_changes', dimension=dimension, sort='scrap_delta') for dimension in ('department', 'group', 'product')]
|
||||
docs += [evidence('quality_fine_defects', dimension='defect', stage=stage) for stage in ('black', 'bright')]
|
||||
self.assertEqual(coverage_gaps(docs, '整体质量变化如何'), [])
|
||||
self.assertEqual(plan_missing_queries(docs, '综合合格率是多少'), [])
|
||||
self.assertEqual(coverage_gaps([docs[0]], '综合合格率是多少'), [])
|
||||
|
||||
def test_rate_only_and_unavailable_defects_cannot_complete_broad_analysis(self):
|
||||
docs = [evidence('quality_comparison')]
|
||||
docs += [evidence('quality_changes', dimension=dimension, sort='scrap_delta') for dimension in ('department', 'group')]
|
||||
docs.append(evidence('quality_changes', dimension='product', sort='rate_drop'))
|
||||
docs.append(evidence('quality_fine_defects', dimension='defect', stage='black', available=False))
|
||||
docs.append(evidence('quality_fine_defects', dimension='defect', stage='bright'))
|
||||
gaps = coverage_gaps(docs, '整体质量变化如何')
|
||||
self.assertTrue(any('产品' in gap for gap in gaps))
|
||||
self.assertTrue(any('黑片' in gap for gap in gaps))
|
||||
self.assertEqual([(item['mode'], item.get('dimension'), item.get('stage')) for item in plan_missing_queries(docs, '整体质量变化如何')],
|
||||
[('changes', 'product', None)])
|
||||
|
||||
def test_product_stage_pair_is_required_only_for_actual_product_drilldown(self):
|
||||
docs = [evidence('quality_comparison')]
|
||||
docs.append(evidence('quality_fine_defects', dimension='defect', stage='black', filters={'product': '规格A'}, sort='current_scrap'))
|
||||
self.assertTrue(any('亮片' in item for item in coverage_gaps(docs, '分析重点产品')))
|
||||
self.assertEqual(coverage_gaps(docs, '只看黑片规格A的缺陷'), [])
|
||||
|
||||
def test_actual_focus_product_question_requires_product_ranking_evidence(self):
|
||||
docs = [evidence('quality_comparison')]
|
||||
question = '分析下近期要关注的重点产品'
|
||||
self.assertTrue(any('产品多维重点排序' in gap for gap in coverage_gaps(docs, question)))
|
||||
self.assertEqual([(item['mode'], item['dimension']) for item in plan_missing_queries(docs, question)],
|
||||
[('changes', 'product')])
|
||||
docs.append(evidence('quality_changes', dimension='product', sort='scrap_delta'))
|
||||
self.assertEqual(coverage_gaps(docs, question), [])
|
||||
|
||||
def test_stage_scoped_broad_question_uses_same_stage_for_changes(self):
|
||||
docs = [evidence('quality_comparison')]
|
||||
context = {'stage': 'black'}
|
||||
plans = plan_missing_queries(docs, '分析整体质量变化', context)
|
||||
self.assertEqual([(item['mode'], item.get('stage')) for item in plans],
|
||||
[('changes', 'black')] * 3 + [('defects', 'black')])
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Stage rate support must not hide official zero-weight comparison values."""
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_changes import validate_changes, changes_report
|
||||
|
||||
|
||||
def fixture():
|
||||
period, prior = {'year': 2026, 'month': 9}, {'year': 2026, 'month': 8}
|
||||
task = {'period': period, 'comparisonPeriod': prior, 'filters': {'department': '测试部'}, 'stages': [],
|
||||
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'question': '本期合格率最低的产品',
|
||||
'changeQuery': {'dimension': 'product', 'sort': 'current_rate', 'limit': 1}}
|
||||
absent = {'rate': None, 'qualifiedWeight': None, 'totalWeight': None, 'scrapWeight': None}
|
||||
result = {'kind': 'quality_changes', 'releaseId': 'fixture-release', 'currentPeriod': period,
|
||||
'comparisonPeriod': prior, 'filters': task['filters'], 'dimension': 'product', 'sort': 'current_rate',
|
||||
'stage': None, 'completeRanking': True, 'totalRows': 898, 'rows': [{
|
||||
'rank': 1, 'key': '规格甲', 'label': '规格甲', 'changeType': 'continuing',
|
||||
'present': {'current': True, 'comparison': True}, 'rateDrop': None, 'scrapDelta': None,
|
||||
'current': {'rate': 0, 'rateSupport': {'blackMeasured': True, 'brightMeasured': False,
|
||||
'compositeReferenceAvailable': True, 'missingStages': ['bright']},
|
||||
'black': {'rate': 0, 'qualifiedWeight': None, 'totalWeight': 10.438, 'scrapWeight': 10.438},
|
||||
'bright': copy.deepcopy(absent), 'blackInputWeight': 10.438, 'brightInputWeight': None, 'scrapWeight': None},
|
||||
'comparison': {'rate': None, 'rateSupport': {'blackMeasured': False, 'brightMeasured': False,
|
||||
'compositeReferenceAvailable': False, 'missingStages': ['black', 'bright']},
|
||||
'black': {'rate': None, 'qualifiedWeight': None, 'totalWeight': 0, 'scrapWeight': 0},
|
||||
'bright': copy.deepcopy(absent), 'blackInputWeight': 0, 'brightInputWeight': None, 'scrapWeight': None}}]}
|
||||
return result, task
|
||||
|
||||
|
||||
class ChangesWeightReferenceTests(unittest.TestCase):
|
||||
def test_current_rate_accepts_official_zero_baseline_with_unavailable_rate(self):
|
||||
result, task = fixture()
|
||||
self.assertIs(validate_changes(result, task, 'fixture-release'), result)
|
||||
report = changes_report([{'result': result, 'evidenceRef': 'Q00000001'}], task, 'fixture-release')
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('两期合格率不可比', report['facts'][0]['text'])
|
||||
self.assertIn('0.00%', report['facts'][0]['text'])
|
||||
self.assertNotIn('下降100', report['facts'][0]['text'])
|
||||
self.assertIn('本期亮片阶段率未实测', report['facts'][0]['text'])
|
||||
self.assertIn('基期黑片、亮片阶段率未实测', report['facts'][0]['text'])
|
||||
self.assertIsNone(result['rows'][0]['comparison']['rate'])
|
||||
self.assertIsNone(result['rows'][0]['current']['bright']['totalWeight'])
|
||||
self.assertIn('黑片报废重量 0.00→10.44 kg', report['facts'][1]['text'])
|
||||
|
||||
def test_new_product_absent_baseline_does_not_require_or_invent_weights(self):
|
||||
result, task = fixture()
|
||||
result['rows'][0]['present']['comparison'] = False
|
||||
result['rows'][0]['comparison']['black'] = None
|
||||
result['rows'][0]['changeType'] = 'new'
|
||||
self.assertIs(validate_changes(result, task, 'fixture-release'), result)
|
||||
|
||||
def test_rate_missing_stage_cannot_hide_missing_requested_weight(self):
|
||||
result, task = fixture()
|
||||
result['rows'][0]['comparison']['black']['totalWeight'] = None
|
||||
with self.assertRaisesRegex(ValueError, r'black.totalWeight'):
|
||||
validate_changes(result, task, 'fixture-release')
|
||||
|
||||
def test_null_stages_and_hidden_measured_fields_remain_incomplete(self):
|
||||
for measured in (False, True):
|
||||
result, task = fixture()
|
||||
side = result['rows'][0]['comparison']
|
||||
side['black']['totalWeight'] = side['black']['scrapWeight'] = None
|
||||
side['rateSupport']['blackMeasured'] = measured
|
||||
with self.subTest(measured=measured), self.assertRaises(ValueError):
|
||||
validate_changes(result, task, 'fixture-release')
|
||||
|
||||
def test_invalid_weight_is_rejected_even_when_stage_rate_unavailable(self):
|
||||
for value in (-1, float('nan'), True):
|
||||
result, task = fixture()
|
||||
result['rows'][0]['comparison']['black']['scrapWeight'] = value
|
||||
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||
validate_changes(result, task, 'fixture-release')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Offline isolation checks for concurrent NAO actor sessions; never calls a model."""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
import nao_quality_runtime as runtime
|
||||
import nao_quality_runtime_mcp as runtime_mcp
|
||||
|
||||
|
||||
class WaitingResponse:
|
||||
def __init__(self):
|
||||
self.closed = threading.Event()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.close()
|
||||
|
||||
def __iter__(self):
|
||||
self.closed.wait(0.5)
|
||||
return iter(())
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
|
||||
|
||||
class WaitingClient:
|
||||
def open(self, *_args, **_kwargs):
|
||||
return WaitingResponse()
|
||||
|
||||
|
||||
class SessionIsolationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-")
|
||||
self.users = Path(self.temp.name).resolve()
|
||||
self.manager = runtime.RuntimeManager(users_root=self.users)
|
||||
self.actor_id = str(uuid.uuid4())
|
||||
self.actor_b = str(uuid.uuid4())
|
||||
self.session_a = str(uuid.uuid4())
|
||||
self.session_b = str(uuid.uuid4())
|
||||
self.release_a = str(uuid.uuid4())
|
||||
self.release_b = str(uuid.uuid4())
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def test_same_actor_sessions_have_independent_roots_locks_credentials_and_mcp_paths(self):
|
||||
root_a, lock_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, lock_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
self.assertEqual(root_a, self.users / self.actor_id / "sessions" / self.session_a)
|
||||
self.assertEqual(root_b, self.users / self.actor_id / "sessions" / self.session_b)
|
||||
self.assertNotEqual(root_a, root_b)
|
||||
self.assertNotEqual(root_a / "active.lock", root_b / "active.lock")
|
||||
|
||||
access_a = runtime.write_access_file(root_a, "token-session-A", self.release_a,
|
||||
expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_a)
|
||||
access_b = runtime.write_access_file(root_b, "token-session-B", self.release_b,
|
||||
expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_b)
|
||||
snapshot_a = runtime.load_access_snapshot(
|
||||
access_a, runtime_root=self.users, expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_a,
|
||||
)
|
||||
snapshot_b = runtime.load_access_snapshot(
|
||||
access_b, runtime_root=self.users, expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_b,
|
||||
)
|
||||
self.assertEqual((snapshot_a.actor_id, snapshot_a.token, snapshot_a.release_id),
|
||||
(self.actor_id, "token-session-A", self.release_a))
|
||||
self.assertEqual((snapshot_b.actor_id, snapshot_b.token, snapshot_b.release_id),
|
||||
(self.actor_id, "token-session-B", self.release_b))
|
||||
self.assertNotEqual(access_a, access_b)
|
||||
|
||||
runtime._write_private_state(root_a)
|
||||
runtime._write_private_state(root_b)
|
||||
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=True,
|
||||
refresh_mcp=True, backup_schema=False,
|
||||
expected_actor_id=self.actor_id)
|
||||
runtime._write_project(root_b, self.session_b, "test-model", refresh_templates=True,
|
||||
refresh_mcp=True, backup_schema=False,
|
||||
expected_actor_id=self.actor_id)
|
||||
mcp_a = json.loads((root_a / "project/agent/mcps/mcp.json").read_text())
|
||||
mcp_b = json.loads((root_b / "project/agent/mcps/mcp.json").read_text())
|
||||
env_a = mcp_a["mcpServers"]["quality"]["env"]
|
||||
env_b = mcp_b["mcpServers"]["quality"]["env"]
|
||||
self.assertEqual(env_a["QUALITY_AGENT_ACTOR_ID"], self.actor_id)
|
||||
self.assertEqual(env_b["QUALITY_AGENT_ACTOR_ID"], self.actor_id)
|
||||
self.assertEqual(env_a["QUALITY_AGENT_SESSION_ID"], self.session_a)
|
||||
self.assertEqual(env_b["QUALITY_AGENT_SESSION_ID"], self.session_b)
|
||||
self.assertEqual(env_a["QUALITY_AGENT_ACCESS_FILE"], str(access_a))
|
||||
self.assertEqual(env_b["QUALITY_AGENT_ACCESS_FILE"], str(access_b))
|
||||
self.assertNotEqual(env_a["QUALITY_AGENT_SESSION_STATE_FILE"],
|
||||
env_b["QUALITY_AGENT_SESSION_STATE_FILE"])
|
||||
self.assertNotEqual(runtime._write_private_state(root_a), runtime._write_private_state(root_b))
|
||||
finally:
|
||||
self.manager.release_actor(lock_a)
|
||||
self.manager.release_actor(lock_b)
|
||||
# flock files are stable coordination objects and are never unlinked.
|
||||
self.assertTrue((root_a / "active.lock").is_file())
|
||||
self.assertTrue((root_b / "active.lock").is_file())
|
||||
|
||||
def test_same_session_is_mutually_exclusive_but_other_session_and_actor_can_run(self):
|
||||
_root_a, lock_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, lock_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
root_c, lock_c = self.manager.acquire_actor(self.actor_b, self.session_a)
|
||||
try:
|
||||
self.assertNotEqual(root_b, root_c)
|
||||
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "另一轮分析"):
|
||||
self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
finally:
|
||||
self.manager.release_actor(lock_c)
|
||||
self.manager.release_actor(lock_b)
|
||||
self.manager.release_actor(lock_a)
|
||||
_again, lock_again = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
self.manager.release_actor(lock_again)
|
||||
|
||||
def test_access_snapshot_rejects_actor_session_mismatch_and_symlink(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
try:
|
||||
access = runtime.write_access_file(root_a, "strict-token", self.release_a,
|
||||
expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_a)
|
||||
with self.assertRaisesRegex(ValueError, "当前会话"):
|
||||
runtime.load_access_snapshot(access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor_b,
|
||||
expected_session_id=self.session_a)
|
||||
with self.assertRaisesRegex(ValueError, "路径与当前会话"):
|
||||
runtime.load_access_snapshot(access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_b)
|
||||
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "actor/session"):
|
||||
runtime.write_access_file(root_a, "wrong-owner", self.release_a,
|
||||
expected_actor_id=self.actor_b,
|
||||
expected_session_id=self.session_a)
|
||||
link = root_a / "linked-access.json"
|
||||
link.symlink_to(access)
|
||||
with self.assertRaisesRegex(ValueError, "路径无效"):
|
||||
runtime.load_access_snapshot(link, runtime_root=self.users,
|
||||
expected_actor_id=self.actor_id,
|
||||
expected_session_id=self.session_a)
|
||||
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "actor/session"):
|
||||
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=False,
|
||||
refresh_mcp=False, backup_schema=False,
|
||||
expected_actor_id=self.actor_b)
|
||||
finally:
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_mcp_shim_accepts_only_fixed_actor_session_paths(self):
|
||||
fake_root = self.users / "fake-project"
|
||||
users_root = fake_root / ".runtime" / "nao-users"
|
||||
session_root = users_root / self.actor_id / "sessions" / self.session_a
|
||||
session_root.mkdir(parents=True)
|
||||
environment = {
|
||||
"QUALITY_AGENT_USERS_ROOT": str(users_root),
|
||||
"QUALITY_AGENT_ACCESS_FILE": str(session_root / "access.json"),
|
||||
"QUALITY_AGENT_EVIDENCE_DIR": str(session_root / "evidence"),
|
||||
"QUALITY_AGENT_SESSION_STATE_FILE": str(session_root / "private" / "current-session.json"),
|
||||
"QUALITY_AGENT_ACTOR_ID": self.actor_id,
|
||||
"QUALITY_AGENT_SESSION_ID": self.session_a,
|
||||
}
|
||||
with patch.object(runtime_mcp, "ROOT", fake_root), patch.dict(os.environ, environment, clear=False):
|
||||
access, evidence, state, actor, session, root = runtime_mcp._paths()
|
||||
self.assertEqual(access, session_root / "access.json")
|
||||
self.assertEqual(evidence, session_root / "evidence")
|
||||
self.assertEqual(state, session_root / "private" / "current-session.json")
|
||||
self.assertEqual((actor, session), (self.actor_id, self.session_a))
|
||||
self.assertEqual(root, users_root.resolve())
|
||||
os.environ["QUALITY_AGENT_ACCESS_FILE"] = str(users_root / self.actor_id / "access.json")
|
||||
with self.assertRaisesRegex(ValueError, "路径无效"):
|
||||
runtime_mcp._paths()
|
||||
|
||||
def test_cancel_stops_only_the_current_session_chat(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
state_a = runtime.SessionState(root_a, self.actor_id, self.session_a)
|
||||
state_b = runtime.SessionState(root_b, self.actor_id, self.session_b)
|
||||
state_a.set_chat_id(self.session_a, "native-chat-A")
|
||||
state_b.set_chat_id(self.session_b, "native-chat-B")
|
||||
cancel = threading.Event()
|
||||
cancel.set()
|
||||
stopped = []
|
||||
|
||||
def stop_native(_client, _base, chat_id):
|
||||
stopped.append(chat_id)
|
||||
return True
|
||||
|
||||
with patch.object(runtime, "_stop_native",
|
||||
side_effect=lambda _client, _base, chat_id: stopped.append(chat_id) or True):
|
||||
result = runtime._sse_request(
|
||||
client=WaitingClient(), base_url="http://127.0.0.1:10000", model="offline-test",
|
||||
question="q", release_id=self.release_a, context=None,
|
||||
chat_id=state_a.get_chat_id(self.session_a), session_state=state_a,
|
||||
session_id=self.session_a, actor_root=root_a, emit=lambda _event: None,
|
||||
cancel_event=cancel, timeout=5,
|
||||
)
|
||||
self.assertEqual(result[1], "CANCELLED")
|
||||
self.assertEqual(stopped, ["native-chat-A"])
|
||||
self.assertEqual(state_b.get_chat_id(self.session_b), "native-chat-B")
|
||||
self.assertFalse((root_b / "instance.json").exists())
|
||||
finally:
|
||||
self.manager.release_actor(fd_b)
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_stop_instance_only_targets_processes_recorded_in_that_session(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
for root, session_id, pids, port in (
|
||||
(root_a, self.session_a, [10101, 10102], 41001),
|
||||
(root_b, self.session_b, [20201, 20202], 42001),
|
||||
):
|
||||
runtime._atomic_json(root / "instance.json", {
|
||||
"actorId": self.actor_id, "sessionId": session_id,
|
||||
"processes": [{"pid": pid, "pgid": pid, "lstart": "simulated",
|
||||
"kind": "server", "port": port + index}
|
||||
for index, pid in enumerate(pids)],
|
||||
})
|
||||
killed = []
|
||||
|
||||
def matches(record, runtime_root):
|
||||
return runtime_root == root_a and record.get("pid") in (10101, 10102)
|
||||
|
||||
with patch.object(runtime, "_process_matches", side_effect=matches), \
|
||||
patch.object(runtime, "_ps_value", return_value="simulated process"), \
|
||||
patch.object(runtime.os, "killpg", side_effect=lambda pgid, _sig: killed.append(pgid)), \
|
||||
patch.object(runtime.time, "sleep"):
|
||||
runtime._stop_instance(root_a, grace_seconds=0)
|
||||
self.assertEqual(set(killed), {10101, 10102})
|
||||
self.assertTrue((root_b / "instance.json").is_file())
|
||||
finally:
|
||||
self.manager.release_actor(fd_b)
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_stop_all_skips_active_session_and_scans_legacy_and_idle_sessions(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
legacy = self.users / self.actor_b
|
||||
legacy.mkdir(parents=True)
|
||||
runtime._atomic_json(legacy / "instance.json", {"actorId": self.actor_b, "processes": []})
|
||||
for root in (root_a, root_b, legacy):
|
||||
runtime._atomic_json(root / "access.json", {"token": "private"})
|
||||
self.manager.release_actor(fd_b)
|
||||
try:
|
||||
stopped = []
|
||||
manager = self.manager
|
||||
with patch.object(runtime, "_stop_instance", side_effect=lambda root: stopped.append(root) or True):
|
||||
count = manager.stop_idle_all()
|
||||
self.assertEqual(count, 2)
|
||||
self.assertEqual(set(stopped), {root_b, legacy})
|
||||
self.assertTrue((root_a / "access.json").exists())
|
||||
self.assertFalse((root_b / "access.json").exists())
|
||||
self.assertFalse((legacy / "access.json").exists())
|
||||
self.assertIn(legacy, runtime._runtime_roots(self.users))
|
||||
self.assertIn(root_a, runtime._runtime_roots(self.users))
|
||||
finally:
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_project_template_symlink_cannot_write_into_another_session(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
project_a = root_a / "project"
|
||||
project_a.mkdir()
|
||||
project_b = root_b / "project"
|
||||
project_b.mkdir()
|
||||
protected_config = project_b / "nao_config.yaml"
|
||||
protected_config.write_text("session B config", encoding="utf-8")
|
||||
(project_a / "nao_config.yaml").symlink_to(protected_config)
|
||||
with self.assertRaisesRegex(ValueError, "运行路径无效"):
|
||||
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=True,
|
||||
refresh_mcp=True, backup_schema=False,
|
||||
expected_actor_id=self.actor_id)
|
||||
self.assertEqual(protected_config.read_text(encoding="utf-8"), "session B config")
|
||||
finally:
|
||||
self.manager.release_actor(fd_b)
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_server_log_open_rejects_symlink_and_non_regular_files(self):
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
logs_a = root_a / "logs"
|
||||
logs_b = root_b / "logs"
|
||||
logs_a.mkdir()
|
||||
logs_b.mkdir()
|
||||
protected_log = logs_b / "server.log"
|
||||
protected_log.write_text("session B log", encoding="utf-8")
|
||||
(logs_a / "server.log").symlink_to(protected_log)
|
||||
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "日志文件路径无效"):
|
||||
runtime._open_server_log(logs_a / "server.log", root_a)
|
||||
self.assertEqual(protected_log.read_text(encoding="utf-8"), "session B log")
|
||||
|
||||
(logs_a / "server.log").unlink()
|
||||
fifo = logs_a / "server.log"
|
||||
os.mkfifo(fifo)
|
||||
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "日志文件路径无效"):
|
||||
runtime._open_server_log(fifo, root_a)
|
||||
finally:
|
||||
self.manager.release_actor(fd_b)
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
def test_legacy_session_history_is_copied_to_private_root_without_changing_source(self):
|
||||
actor_root = self.users / self.actor_id
|
||||
legacy_private = actor_root / "private"
|
||||
legacy_private.mkdir(parents=True)
|
||||
mapping = legacy_private / "sessions.json"
|
||||
runtime._atomic_json(mapping, {
|
||||
"actorId": self.actor_id,
|
||||
"sessions": {self.session_a: "old-native-chat-A", self.session_b: "old-native-chat-B"},
|
||||
})
|
||||
mapping_before = mapping.read_bytes()
|
||||
runtime._atomic_json(legacy_private / "credentials.json", {
|
||||
"authSecret": "legacy-auth", "email": "legacy@example.invalid", "password": "legacy-password",
|
||||
})
|
||||
legacy_project = actor_root / "project"
|
||||
legacy_project.mkdir(parents=True)
|
||||
source_db = legacy_private / "nao.sqlite"
|
||||
with contextlib.closing(sqlite3.connect(source_db)) as database:
|
||||
with database:
|
||||
# Mirrors NAO's actual SQLite project.path and chat.project_id
|
||||
# relationship used by agent routes and MCP project lookup.
|
||||
database.executescript("""
|
||||
CREATE TABLE project (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
org_id TEXT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
path TEXT,
|
||||
agent_settings TEXT,
|
||||
enabled_tools TEXT NOT NULL DEFAULT '[]',
|
||||
known_mcp_servers TEXT NOT NULL DEFAULT '[]',
|
||||
disabled_mcp_servers TEXT NOT NULL DEFAULT '[]',
|
||||
disabled_mcp_tools TEXT NOT NULL DEFAULT '[]',
|
||||
env_vars TEXT NOT NULL DEFAULT '{}',
|
||||
CONSTRAINT local_project_path_required
|
||||
CHECK(CASE WHEN type = 'local' THEN path IS NOT NULL ELSE TRUE END)
|
||||
);
|
||||
CREATE TABLE chat (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
project_id TEXT NOT NULL REFERENCES project(id),
|
||||
title TEXT DEFAULT 'New Conversation' NOT NULL,
|
||||
is_starred INTEGER DEFAULT false NOT NULL,
|
||||
deleted_at INTEGER,
|
||||
slack_thread_id TEXT,
|
||||
teams_thread_id TEXT,
|
||||
telegram_thread_id TEXT,
|
||||
mattermost_thread_id TEXT,
|
||||
discord_thread_id TEXT,
|
||||
whatsapp_thread_id TEXT,
|
||||
fork_metadata TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
""")
|
||||
database.execute(
|
||||
"INSERT INTO project(id,name,type,path) VALUES (?,?,?,?)",
|
||||
("legacy-project-id", "Quality", "local", str(legacy_project)),
|
||||
)
|
||||
database.executemany(
|
||||
"INSERT INTO chat(id,user_id,project_id,created_at,updated_at) VALUES (?,?,?,?,?)",
|
||||
(("old-native-chat-A", "legacy-user", "legacy-project-id", 1, 1),
|
||||
("old-native-chat-B", "legacy-user", "legacy-project-id", 2, 2)),
|
||||
)
|
||||
source_db.chmod(0o600)
|
||||
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
|
||||
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
|
||||
try:
|
||||
self.assertTrue(self.manager._migrate_legacy_session(self.actor_id, self.session_a, root_a))
|
||||
self.assertTrue(self.manager._migrate_legacy_session(self.actor_id, self.session_b, root_b))
|
||||
state = runtime.SessionState(root_a, self.actor_id, self.session_a)
|
||||
other_state = runtime.SessionState(root_b, self.actor_id, self.session_b)
|
||||
self.assertEqual(state.get_chat_id(self.session_a), "old-native-chat-A")
|
||||
self.assertEqual(other_state.get_chat_id(self.session_b), "old-native-chat-B")
|
||||
self.assertNotEqual(root_a / "private" / "nao.sqlite", source_db)
|
||||
with contextlib.closing(sqlite3.connect(root_a / "private" / "nao.sqlite")) as database:
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,path FROM project WHERE id='legacy-project-id'"
|
||||
).fetchone(), ("legacy-project-id", str(root_a / "project")))
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,project_id FROM chat WHERE id='old-native-chat-A'"
|
||||
).fetchone(), ("old-native-chat-A", "legacy-project-id"))
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,project_id FROM chat WHERE id='old-native-chat-B'"
|
||||
).fetchone(), ("old-native-chat-B", "legacy-project-id"))
|
||||
with contextlib.closing(sqlite3.connect(root_b / "private" / "nao.sqlite")) as database:
|
||||
with database:
|
||||
database.execute(
|
||||
"INSERT INTO chat(id,user_id,project_id,created_at,updated_at) VALUES (?,?,?,?,?)",
|
||||
("session-B-only", "legacy-user", "legacy-project-id", 3, 3),
|
||||
)
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,path FROM project WHERE id='legacy-project-id'"
|
||||
).fetchone(), ("legacy-project-id", str(root_b / "project")))
|
||||
with contextlib.closing(sqlite3.connect(root_a / "private" / "nao.sqlite")) as database:
|
||||
rows = database.execute("SELECT id FROM chat ORDER BY id").fetchall()
|
||||
self.assertEqual(rows, [("old-native-chat-A",), ("old-native-chat-B",)])
|
||||
with contextlib.closing(sqlite3.connect(source_db)) as database:
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,path FROM project WHERE id='legacy-project-id'"
|
||||
).fetchone(), ("legacy-project-id", str(legacy_project)))
|
||||
self.assertEqual(database.execute(
|
||||
"SELECT id,project_id FROM chat WHERE id='old-native-chat-A'"
|
||||
).fetchone(), ("old-native-chat-A", "legacy-project-id"))
|
||||
self.assertEqual(mapping.read_bytes(), mapping_before)
|
||||
self.assertTrue(source_db.is_file())
|
||||
self.assertEqual(runtime._read_json(root_a / "private" / "credentials.json")["authSecret"],
|
||||
"legacy-auth")
|
||||
finally:
|
||||
self.manager.release_actor(fd_b)
|
||||
self.manager.release_actor(fd_a)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Semantic conversational replies must avoid agent overhead without bypassing evidence work."""
|
||||
import io,json,sys,threading,unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_agent_task import normalize_task,plan_task,TaskUnavailable
|
||||
|
||||
|
||||
def direct(**changes):
|
||||
return {'kind':'explain','goal':'回应开场问候并介绍质量分析能力',
|
||||
'directResponse':{'purpose':'capabilities','answer':'你好,我可以帮你查询质量指标、比较期间变化并分析重点规格。','requiresMemory':False},**changes}
|
||||
|
||||
class ConversationTests(unittest.TestCase):
|
||||
def test_minimal_semantic_reply_needs_no_full_business_plan(self):
|
||||
task=normalize_task(direct(),'请介绍一下你能做哪些事')
|
||||
self.assertEqual(task['directResponse']['answer'],direct()['directResponse']['answer'])
|
||||
self.assertFalse(task['freshQuery']);self.assertEqual(task['visualPreference'],'none')
|
||||
self.assertEqual(task['dimensions'],[]);self.assertEqual(task['fields'],[])
|
||||
def test_direct_reply_cannot_request_business_actions_or_hide_memory_writes(self):
|
||||
cases=[{'kind':'metrics'},{'freshQuery':True},{'comparisonRequired':True},
|
||||
{'verifySource':True},{'fullDetails':True},{'needsClarification':True},
|
||||
{'fields':['rate']},{'stages':['black']},{'dimensions':['product']},
|
||||
{'filterUpdates':{'group':'OTHER'}},{'period':{'year':2026,'month':9}},
|
||||
{'historyMode':'all'},{'visualPreference':'required'}]
|
||||
for changes in cases:
|
||||
with self.subTest(changes=changes),self.assertRaises(TaskUnavailable):normalize_task(direct(**changes),'q')
|
||||
for changes in ({'requiresMemory':True},{'purpose':'business_analysis'},
|
||||
{'answer':'本月合格率为95.5%'},{'answer':'已完成统计 [Q12345678]'},
|
||||
{'answer':'https://example.test'},{'answer':'Bearer secretvalue'}):
|
||||
value=direct();value['directResponse'].update(changes)
|
||||
with self.subTest(changes=changes),self.assertRaises(TaskUnavailable):normalize_task(value,'q')
|
||||
def test_plain_chat_preserves_active_business_frame_without_declaring_new_analysis(self):
|
||||
task=normalize_task(direct(),'谢谢你',{'releaseId':'r','period':{'year':2026,'month':9},'filters':{'group':'G'}})
|
||||
self.assertEqual(task['period'],{'year':2026,'month':9});self.assertEqual(task['filters'],{'group':'G'})
|
||||
self.assertFalse(task['freshQuery'])
|
||||
def test_rephrased_capability_reply_is_still_plain_conversation(self):
|
||||
value=direct(kind='restate')
|
||||
result=normalize_task(value,'换一种说法介绍一下你能做什么')
|
||||
self.assertEqual(result['kind'],'explain');self.assertIn('directResponse',result)
|
||||
def test_optional_null_direct_reply_does_not_block_real_business_planning(self):
|
||||
from test_nao_agent_task import task_value
|
||||
value=task_value(dimensions=['product'],dimensionStages={'product':['black']},directResponse=None)
|
||||
result=normalize_task(value,'查黑片重点规格')
|
||||
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery']);self.assertNotIn('directResponse',result)
|
||||
def test_numbered_capability_lists_do_not_trigger_replanning_or_allow_metric_values(self):
|
||||
value=direct();value['directResponse']['answer']='我能帮助你:\n1. 查询质量指标\n2、比较期间变化\n3) 分析重点规格'
|
||||
task=normalize_task(value,'介绍一下能力')
|
||||
self.assertNotIn('1.',task['directResponse']['answer']);self.assertEqual(task['directResponse']['answer'].count('- '),3)
|
||||
value['directResponse']['answer']='1. 合格率为95.5%'
|
||||
with self.assertRaises(TaskUnavailable):normalize_task(value,'q')
|
||||
value['directResponse']['answer']='我可以帮你筛选TOP5重点规格,也可比较两期指标。'
|
||||
self.assertIn('TOP5',normalize_task(value,'介绍能力')['directResponse']['answer'])
|
||||
for text in ('合格率百分之九十五','报废重量17513','我已记住你的偏好','已经保存了你的设置'):
|
||||
value['directResponse']['answer']=text
|
||||
with self.subTest(text=text),self.assertRaises(TaskUnavailable):normalize_task(value,'q')
|
||||
def test_capability_units_are_not_claimed_measurements(self):
|
||||
value=direct();value['directResponse']['answer']='可查询投入重量(kg)、合格率(%),并按个百分点比较变化。'
|
||||
self.assertIn('kg',normalize_task(value,'介绍能力')['directResponse']['answer'])
|
||||
|
||||
def test_capabilities_can_describe_query_results_without_claiming_work_completed(self):
|
||||
value=direct();value['directResponse']['answer']='我可以解释查询结果,并把当前数据整理成图表和报告。'
|
||||
self.assertIn('查询结果',normalize_task(value,'能做什么')['directResponse']['answer'])
|
||||
value['directResponse']['answer']='我已经查到最新数据,合格率为95%'
|
||||
with self.assertRaises(TaskUnavailable):normalize_task(value,'q')
|
||||
|
||||
def test_direct_repair_keeps_minimal_reply_and_points_to_the_actual_field(self):
|
||||
bad=direct();bad['directResponse']['answer']='合格率95%'
|
||||
replies=[bad,direct(),{'aligned':True,'requiresEvidence':False}];payloads=[];errors=[]
|
||||
def opener(req,timeout):
|
||||
payloads.append(json.loads(json.loads(req.data)['messages'][1]['content']))
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
|
||||
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='你好',context={},cancel_event=threading.Event(),opener=opener,on_validation_error=errors.append)
|
||||
self.assertIn('directResponse',result);self.assertEqual(errors[0]['field'],'directResponse.answer')
|
||||
self.assertEqual(errors[0]['receivedType'],'dict')
|
||||
self.assertIn('directResponse',payloads[1]['invalidPlan']);self.assertIn('无需补全',payloads[1]['repairInstructions'])
|
||||
|
||||
def test_planner_reviews_model_authored_reply(self):
|
||||
bodies=[]
|
||||
def opener(request,timeout):
|
||||
bodies.append(json.loads(request.data));reply=direct() if len(bodies)==1 else {'aligned':True,'requiresEvidence':False}
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(reply)}}]}).encode())
|
||||
task=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='嗨,先认识一下',
|
||||
context={},cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(len(bodies),2);self.assertIn('directResponse',task)
|
||||
self.assertIn('directResponse',bodies[0]['messages'][0]['content'])
|
||||
|
||||
def test_exact_json_code_fence_does_not_require_replanning_or_skip_validation(self):
|
||||
calls=[]
|
||||
def opener(request,timeout):
|
||||
calls.append(request)
|
||||
reply=direct() if len(calls)==1 else {'aligned':True,'requiresEvidence':False}
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':'```json\n'+json.dumps(reply)+'\n```'}}]}).encode())
|
||||
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='你好',context={},cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(len(calls),2);self.assertFalse(result['freshQuery'])
|
||||
|
||||
def test_internal_turn_correlation_marker_is_not_user_visible(self):
|
||||
from nao_quality_runtime import public_progress_text
|
||||
from nao_thinking_stream import public_thinking_text
|
||||
text='原文 quality-turn:12345678-1234-1234-1234-123456789abc 继续答复'
|
||||
for sanitizer in (public_progress_text,public_thinking_text):
|
||||
output=sanitizer(text);self.assertNotIn('quality-turn:',output);self.assertIn('继续答复',output)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,56 @@
|
||||
import io,json,sys,tempfile,unittest,uuid
|
||||
from pathlib import Path
|
||||
from contextlib import ExitStack
|
||||
from unittest.mock import MagicMock,patch
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_agent_task import normalize_task
|
||||
from test_nao_conversation import direct
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
class ConversationRuntimeTests(unittest.TestCase):
|
||||
def run_case(self,case):
|
||||
with tempfile.TemporaryDirectory() as folder,ExitStack() as stack:
|
||||
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
||||
req={key:str(uuid.uuid4()) for key in ('turnId','actorId','sessionId','releaseId')};req.update(question='谢谢你',context={},accessToken='synthetic')
|
||||
prior=root/'private/last-agent-task.json';business={'releaseId':req['releaseId'],'sessionId':req['sessionId'],
|
||||
'baseMetricFrame':{},'task':normalize_task(task_value(dimensions=[],dimensionStages={}), '分析质量变化')}
|
||||
prior.write_text(json.dumps(business));prior.chmod(0o600);saved=prior.read_bytes()
|
||||
controls=root/'private/last-model-controls.json';controls.write_text('{}');controls.chmod(0o600);control_before=controls.read_bytes()
|
||||
visible=root/'private/last-visible-answer.json';visible.write_text('{"answer":"原业务结果"}');visible.chmod(0o600)
|
||||
task=normalize_task(direct(),'谢谢你') if case!='memory' else normalize_task(task_value(kind='explain',freshQuery=False,dimensions=[],stages=[],fields=[]),'请记住我偏好简洁')
|
||||
manager=MagicMock();manager.acquire_actor.return_value=(root,None);manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','synthetic','m','http://127.0.0.1:5000')
|
||||
state=MagicMock();state.get_chat_id.return_value=None;gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1';gate.request_audit=[]
|
||||
def planned(**kw):
|
||||
if case=='cancel':kw['cancel_event'].set()
|
||||
return task
|
||||
conversation=runtime.ConversationAccumulator();conversation.blocks['a']='已记录你的表达偏好。';conversation.block_order=['a'];conversation.record_event('assistant',text=conversation.answer)
|
||||
native=MagicMock(return_value=(conversation,'COMPLETED','stop',None));auth=MagicMock();primary=MagicMock()
|
||||
mocks={'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
|
||||
'_model_settings':MagicMock(return_value=('https://offline.invalid','synthetic','m',{'enabled':True,'effort':'medium','protocol':'thinking'})),
|
||||
'_model_gate_settings':MagicMock(return_value=(49191,'synthetic')),'SessionState':MagicMock(return_value=state),
|
||||
'_input_cancel_listener':MagicMock(),'DelegatedApi':MagicMock(),'plan_agent_task':planned,
|
||||
'_sse_request':native,'_authenticate':auth,'read_primary_evidence':primary,'_story_files':MagicMock(return_value=(None,None))}
|
||||
for key,value in mocks.items():stack.enter_context(patch.object(runtime,key,value))
|
||||
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
|
||||
result=runtime.run_turn(req,stdout=io.StringIO(),stdin=io.StringIO())
|
||||
if case=='memory':
|
||||
self.assertEqual(result['status'],'COMPLETED');manager.ensure_instance.assert_called_once();native.assert_called_once();auth.assert_called_once()
|
||||
else:
|
||||
manager.ensure_instance.assert_not_called();native.assert_not_called();auth.assert_not_called();primary.assert_not_called()
|
||||
self.assertEqual(prior.read_bytes(),saved);self.assertEqual(controls.read_bytes(),control_before)
|
||||
self.assertEqual(visible.read_text(),'{"answer":"原业务结果"}')
|
||||
self.assertEqual(result['toolCalls'],[]);self.assertEqual(result['queryEvidenceRefs'],[])
|
||||
if case=='cancel':
|
||||
self.assertEqual(result['status'],'CANCELLED');self.assertEqual(result['answer'],'');self.assertFalse((root/'private/last-conversation-answer.json').exists())
|
||||
self.assertFalse((root/'private/non-query-turns'/ (req['turnId']+'.json')).exists())
|
||||
else:
|
||||
self.assertEqual(result['status'],'COMPLETED');self.assertEqual(result['answer'],task['directResponse']['answer'])
|
||||
self.assertEqual(result['timeline'][-1]['kind'],'assistant')
|
||||
self.assertTrue((root/'private/non-query-turns'/ (req['turnId']+'.json')).exists())
|
||||
gate.close.assert_called_once()
|
||||
def test_conversation_skips_native_title_and_memory_without_replacing_business_state(self):self.run_case('plain')
|
||||
def test_cancelled_conversation_does_not_emit_an_answer_or_change_business_state(self):self.run_case('cancel')
|
||||
def test_explicit_memory_request_still_starts_the_native_agent(self):self.run_case('memory')
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,29 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_analysis_coverage import coverage_gaps
|
||||
|
||||
|
||||
class DefectScopeTest(unittest.TestCase):
|
||||
def test_three_selected_products_do_not_require_parent_defects(self):
|
||||
task = {'version':1,'kind':'analysis','question':'fixture','dimensions':['defect'],
|
||||
'stages':['black','bright'],'defectScope':'products','defectProductCount':3}
|
||||
root = {'releaseId':'r','filterBinding':{'group':'g'},
|
||||
'current':{'period':{'year':2026,'month':9}},
|
||||
'comparison':{'period':{'year':2026,'month':8}}}
|
||||
docs = [{'result':{'kind':'quality_fine_defects','releaseId':'r','dimension':'defect','stage':stage,
|
||||
'filters':{'group':'g','product':product},'period':root['current']['period'],
|
||||
'comparisonPeriod':root['comparison']['period'],'sort':'scrap_delta',
|
||||
'coverage':{'available':True},'comparisonCoverage':{'available':True}}}
|
||||
for product in ('a','b','c') for stage in ('black','bright')]
|
||||
with patch('nao_analysis_coverage._root', return_value=root):
|
||||
self.assertEqual(coverage_gaps(docs,'fixture',{'_agentTask':task}), [])
|
||||
self.assertTrue(coverage_gaps(docs[:4],'fixture',{'_agentTask':task}))
|
||||
wrong = [{**doc,'result':{**doc['result'],'filters':{'group':'other','product':'c'}}} for doc in docs[4:]]
|
||||
self.assertTrue(coverage_gaps(docs[:4]+wrong,'fixture',{'_agentTask':task}))
|
||||
task['defectScope']='both'
|
||||
self.assertTrue(any('来源具体缺陷' in gap for gap in coverage_gaps(docs,'fixture',{'_agentTask':task})))
|
||||
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Mixed overall metrics and dimension-specific stages must not share a coverage scope."""
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'verification')]
|
||||
from nao_agent_task import normalize_task, bind_task, plan_task, plan_repairs, TaskUnavailable
|
||||
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
|
||||
from nao_product_defect_pairing import plan_missing_product_defect_stages
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_analysis_coverage import evidence, PERIOD, PRIOR
|
||||
from test_nao_product_defect_pairing import evidence as defect_evidence
|
||||
|
||||
QUESTION = '对比两期综合合格率,再查询黑片报废增量最大的3个规格'
|
||||
|
||||
|
||||
def mixed_task(**changes):
|
||||
return normalize_task(task_value(dimensions=['product'], stages=['black', 'bright'],
|
||||
dimensionStages={'product': ['black']}, comparisonRequired=True,
|
||||
period=PERIOD, comparisonPeriod=PRIOR, **changes), QUESTION)
|
||||
|
||||
|
||||
def documents(stage='black'):
|
||||
return [evidence('quality_comparison'),
|
||||
evidence('quality_changes', dimension='product', stage=stage, sort='scrap_delta')]
|
||||
|
||||
|
||||
class DimensionStageTests(unittest.TestCase):
|
||||
def test_overall_comparison_and_black_product_rank_keep_independent_stages(self):
|
||||
task = mixed_task()
|
||||
context = bind_task({'releaseId': 'release-a'}, task)
|
||||
self.assertEqual(task['stages'], ['black', 'bright'])
|
||||
self.assertNotIn('stage', context) # Do not turn composite rate into black-sheet rate.
|
||||
self.assertEqual(coverage_gaps(documents(), QUESTION, context), [])
|
||||
self.assertEqual(plan_missing_queries(documents(), QUESTION, context), [])
|
||||
self.assertTrue(coverage_gaps(documents('bright'), QUESTION, context))
|
||||
planned = plan_missing_queries([documents()[0]], QUESTION, context)
|
||||
self.assertEqual([(p['dimension'], p['stage']) for p in planned], [('product', 'black')])
|
||||
|
||||
def test_wrong_period_scope_release_and_stage_still_block_completion(self):
|
||||
context = bind_task({'releaseId': 'release-a'}, mixed_task())
|
||||
for field, value in [('stage', 'bright'), ('currentPeriod', {'year': 2026, 'month': 10}),
|
||||
('comparisonPeriod', {'year': 2026, 'month': 7}),
|
||||
('releaseId', 'other-release'), ('filters', {'group': 'other-scope'})]:
|
||||
docs = copy.deepcopy(documents()); docs[1]['result'][field] = value
|
||||
with self.subTest(field=field): self.assertTrue(coverage_gaps(docs, QUESTION, context))
|
||||
self.assertTrue(coverage_gaps(documents()[1:], QUESTION, context))
|
||||
|
||||
def test_separate_black_and_bright_ranks_require_both_and_never_accept_a_combined_rank(self):
|
||||
task = mixed_task(); task['dimensionStages']['product'] = ['black', 'bright']
|
||||
context = bind_task({}, task)
|
||||
self.assertEqual(len(coverage_gaps(documents(), QUESTION, context)), 1)
|
||||
self.assertEqual(plan_missing_queries(documents(), QUESTION, context)[0]['stage'], 'bright')
|
||||
both = documents() + [documents('bright')[1]]
|
||||
self.assertEqual(coverage_gaps(both, QUESTION, context), [])
|
||||
self.assertTrue(coverage_gaps(documents(None), QUESTION, context))
|
||||
|
||||
def test_combined_ranking_cannot_be_proven_by_a_single_stage(self):
|
||||
task = mixed_task(); task['dimensionStages']['product'] = []
|
||||
context = bind_task({}, task)
|
||||
self.assertTrue(coverage_gaps(documents(), QUESTION, context))
|
||||
self.assertEqual(coverage_gaps(documents(None), QUESTION, context), [])
|
||||
|
||||
def test_legacy_plans_are_not_silently_reinterpreted_from_the_evidence(self):
|
||||
task = mixed_task(); task.pop('dimensionStages')
|
||||
self.assertTrue(coverage_gaps(documents(), QUESTION, bind_task({}, task)))
|
||||
|
||||
def test_dimension_stage_schema_rejects_untrusted_or_ambiguous_values(self):
|
||||
for scopes in ({'product': 'black'}, {'product': ['unknown']}, {'product': [True]},
|
||||
{'secret': ['black']}, {'defect': []}):
|
||||
with self.subTest(scopes=scopes), self.assertRaises(TaskUnavailable):
|
||||
normalize_task(task_value(dimensionStages=scopes), 'q')
|
||||
|
||||
def test_missing_dimension_scope_is_returned_to_the_model_for_repair(self):
|
||||
replies = [task_value(dimensions=['product']),
|
||||
task_value(dimensions=['product'], dimensionStages={'product': ['black']}),
|
||||
{'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}]
|
||||
payloads = []
|
||||
def opener(request, timeout):
|
||||
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
|
||||
'content': json.dumps(replies.pop(0))}}]}).encode())
|
||||
task = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='fixture',
|
||||
question=QUESTION, context={}, cancel_event=threading.Event(), opener=opener)
|
||||
self.assertEqual(task['dimensionStages'], {'product': ['black']})
|
||||
self.assertIn('dimensionStages', payloads[1]['requiredShapes'])
|
||||
self.assertIn('dimensionStages', payloads[1]['validationFeedback'])
|
||||
|
||||
def test_repair_queries_cannot_switch_a_dimension_to_the_other_stage(self):
|
||||
def run(stage):
|
||||
query = {'mode': 'changes', 'dimension': 'product', 'filters': {}, 'stage': stage}
|
||||
def opener(request, timeout):
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
|
||||
'content': json.dumps({'queries': [query]})}}]}).encode())
|
||||
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='fixture',
|
||||
task=mixed_task(), evidence=[], gaps=['missing'], failures=[], release_id='release-a',
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertEqual(run('black')[0]['stage'], 'black')
|
||||
with self.assertRaises(TaskUnavailable): run('bright')
|
||||
self.assertEqual(run(None)[0]['stage'], 'black')
|
||||
|
||||
def test_defect_drilldown_keeps_its_own_stage_in_a_mixed_task(self):
|
||||
task = normalize_task(task_value(dimensions=['defect'], stages=['black', 'bright'],
|
||||
dimensionStages={'defect': ['black']}), QUESTION)
|
||||
context = bind_task({}, task)
|
||||
self.assertEqual(plan_missing_product_defect_stages([defect_evidence('A', 'black')], QUESTION, context), [])
|
||||
missing = plan_missing_product_defect_stages([defect_evidence('A', 'bright')], QUESTION, context)
|
||||
self.assertEqual([q['stage'] for q in missing], ['black'])
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,481 @@
|
||||
"""Evidence-size, completeness and bounded synthesis regressions; no model calls."""
|
||||
import io
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import ssl
|
||||
import unittest
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_evidence_finalizer as finalizer
|
||||
import nao_quality_runtime as runtime
|
||||
|
||||
|
||||
FORMAL_PRODUCT_QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
|
||||
FORMAL_PRODUCT_GAPS = ['缺少仅针对SYNTH-16X15+T规格的两期分规格投入重量正式数据',
|
||||
'缺少仅针对该规格的黑片、亮片报废重量分规格正式数据']
|
||||
|
||||
|
||||
def formal_product_evidence():
|
||||
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
|
||||
'filterBinding': {'product': 'SYNTH-16X15+T', 'includeUnassigned': False},
|
||||
'delta': {'available': True, 'percentagePoints': -2},
|
||||
'stageRateChangesPP': {'black': -1, 'bright': -1}}
|
||||
for side, month, inputs, scraps in [('current', 9, (1200.5, 2400.5), (36.1, 72.1)),
|
||||
('comparison', 8, (1000.5, 2000.5), (20.1, 40.1))]:
|
||||
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}}
|
||||
for index, stage in enumerate(('black', 'bright')):
|
||||
result[side][stage] = {'rate': .97, 'totalWeight': inputs[index], 'scrapWeight': scraps[index]}
|
||||
return [{'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}]
|
||||
|
||||
|
||||
def unpack(packed):
|
||||
def decode(value):
|
||||
if isinstance(value, list): return [decode(item) for item in value]
|
||||
if not isinstance(value, dict): return value
|
||||
if set(value) == {'$shared'}: return decode(packed['shared'][value['$shared']])
|
||||
if set(value) == {'$columns', '$values'}:
|
||||
return [dict(zip(decode(value['$columns']), decode(row))) for row in decode(value['$values'])]
|
||||
return {key: decode(item) for key,item in value.items()}
|
||||
return decode(packed['evidence'])
|
||||
|
||||
|
||||
class EvidenceFinalizerTest(unittest.TestCase):
|
||||
def test_real_connection_keeps_system_certificate_and_hostname_verification(self):
|
||||
with patch.object(finalizer,'urlopen',return_value=io.BytesIO(b'{}')) as request:
|
||||
finalizer._open_model('request',35)
|
||||
context=request.call_args.kwargs['context']
|
||||
self.assertEqual(context.verify_mode,ssl.CERT_REQUIRED)
|
||||
self.assertTrue(context.check_hostname)
|
||||
|
||||
def test_worker_finalizes_only_this_turn_and_preserves_partial_and_cancelled_states(self):
|
||||
for outcome in ('complete','retry','retry_no_queries','fresh_no_queries','restatement','native_complete','native_query','insufficient','contradictory','server_incomplete','finalizer_error','cancel','cancel_after_stop','stop_failed'):
|
||||
with self.subTest(outcome=outcome),tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir()
|
||||
old=root/'evidence'/('nao-quality-'+'b'*32+'.json')
|
||||
old.write_text('{}');old.chmod(0o600)
|
||||
payload={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')}
|
||||
payload.update(question='全部明细' if outcome=='server_incomplete' else '比较质量',context={},accessToken='test-delegation')
|
||||
if outcome == 'fresh_no_queries': payload['question'] = '请重新查询刚才的数据'
|
||||
if outcome == 'restatement': payload['question'] = '请复述上一轮结论'
|
||||
if outcome in ('retry','retry_no_queries'):payload['context']={'_restartConversation':True}
|
||||
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
|
||||
manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','test-gate-key','mimo-v2.6-flash','http://127.0.0.1:5000')
|
||||
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1'
|
||||
stream=runtime.ConversationAccumulator();stream.blocks['first']='查询已返回';stream.block_order.append('first')
|
||||
stream.record_event('assistant',text='查询已返回')
|
||||
stream.tool_calls=[{'tool':'mcp_call','status':'completed'}]
|
||||
cancellation=[]
|
||||
def sse(**kwargs):
|
||||
if outcome in ('retry','retry_no_queries'):
|
||||
self.assertIsNone(kwargs['chat_id'])
|
||||
self.assertEqual(kwargs['context'],{})
|
||||
if outcome in ('retry_no_queries', 'fresh_no_queries', 'restatement'):
|
||||
stream.tool_calls=[]
|
||||
return stream,'COMPLETED','stop',None
|
||||
cancellation.append(kwargs['cancel_event'])
|
||||
document={'tool':'quality_data','releaseId':payload['releaseId'],'evidenceRef':'Q00000001',
|
||||
'mode':'compare','result':{'kind':'quality_query' if outcome=='native_query' else 'quality_comparison','releaseId':payload['releaseId'],
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':.95},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':.96},
|
||||
'delta':{'available':True,'percentagePoints':-1}}}
|
||||
path=root/'evidence'/('nao-quality-'+'a'*32+'.json');path.write_text(json.dumps(document));path.chmod(0o600)
|
||||
if outcome == 'cancel_after_stop':
|
||||
kwargs['cancel_event'].set()
|
||||
return stream, 'PARTIAL', 'coverage_ready', True
|
||||
if outcome in ('native_complete','native_query','finalizer_error'):return stream,'COMPLETED','stop',None
|
||||
return stream,'PARTIAL','timeout',outcome!='stop_failed'
|
||||
def synthesize(**kwargs):
|
||||
self.assertEqual([d['evidenceRef'] for d in kwargs['evidence']],['Q00000001'])
|
||||
self.assertEqual(kwargs['base_url'],'http://127.0.0.1:49191/v1')
|
||||
self.assertEqual(kwargs['api_key'],'test-gate-key')
|
||||
self.assertEqual(kwargs['context'],{})
|
||||
if outcome=='finalizer_error':raise finalizer.FinalizationUnavailable('整理失败')
|
||||
if outcome=='cancel':cancellation[0].set()
|
||||
return {'answer':'最终统计 [Q00000001]','model':'mimo-v2.6-flash','usage':{},'finishReason':'stop',
|
||||
'evidenceSufficient':outcome!='insufficient','missingEvidence':['缺少用户要求的对照期'] if outcome=='insufficient' else [],
|
||||
'modelDeclaredEvidenceSufficient':outcome not in ('insufficient','contradictory'),
|
||||
'completionAdjusted':outcome=='contradictory'}
|
||||
with patch.object(runtime,'RuntimeManager',return_value=manager),patch.object(runtime,'write_access_file',return_value=root/'access.json'), \
|
||||
patch.object(runtime,'_authenticate'),patch.object(runtime,'_input_cancel_listener'),patch.object(runtime,'SessionState'), \
|
||||
patch.object(runtime,'_sse_request',side_effect=sse),patch.object(runtime,'_story_files',return_value=(None,None)), \
|
||||
patch.object(runtime,'_model_settings',return_value=('https://model.test/v1','test-key','mimo-v2.6-flash',{'enabled':False,'effort':'auto','protocol':'thinking'})) as selected, \
|
||||
patch.object(runtime,'plan_agent_task',return_value=None,create=True), \
|
||||
patch.object(runtime,'_model_gate_settings',return_value=(49191,'test-gate-key')),patch.object(runtime,'DelegatedApi'), \
|
||||
patch('nao_model_gate.ModelGate',return_value=gate) as gate_factory, \
|
||||
patch.object(finalizer,'synthesize',side_effect=synthesize) as request, \
|
||||
patch.object(finalizer,'coverage_sufficient',return_value=outcome not in ('finalizer_error','server_incomplete')):
|
||||
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
|
||||
selected.assert_called_once()
|
||||
self.assertEqual(gate_factory.call_args.kwargs['upstream_key'],'test-key')
|
||||
gate.close.assert_called_once()
|
||||
expected={'complete':'COMPLETED','retry':'COMPLETED','retry_no_queries':'PARTIAL','fresh_no_queries':'PARTIAL','restatement':'COMPLETED','native_complete':'COMPLETED','native_query':'COMPLETED','insufficient':'PARTIAL','contradictory':'COMPLETED','server_incomplete':'PARTIAL','finalizer_error':'PARTIAL','cancel':'CANCELLED','cancel_after_stop':'CANCELLED','stop_failed':'PARTIAL'}[outcome]
|
||||
self.assertEqual(result['status'],expected)
|
||||
self.assertEqual(request.call_count,0 if outcome in ('stop_failed','native_query','retry_no_queries','fresh_no_queries','restatement','cancel_after_stop') else 1)
|
||||
if outcome in ('retry_no_queries', 'fresh_no_queries'):
|
||||
self.assertEqual(result['stopReason'],'evidence_incomplete')
|
||||
self.assertIn('未取得新的查询证据',result['answer'])
|
||||
if outcome=='native_query':
|
||||
self.assertEqual(result['stopReason'],'stop')
|
||||
self.assertEqual(result['answer'],'查询已返回')
|
||||
self.assertFalse(any(call['tool']=='model.finalize' for call in result['toolCalls']))
|
||||
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],'COMPLETED')
|
||||
if outcome in ('complete','native_complete'):self.assertEqual(result['stopReason'],'evidence_finalized')
|
||||
if outcome in ('complete','native_complete'):
|
||||
self.assertEqual(len([entry for entry in result['timeline'] if entry['kind']=='assistant']),1)
|
||||
self.assertEqual(result['timeline'][-1]['text'],'最终统计 [Q00000001]')
|
||||
self.assertEqual(json.loads((root/'private/last-visible-answer.json').read_text())['answer'],'最终统计 [Q00000001]')
|
||||
if outcome in ('complete','native_complete','insufficient','contradictory','server_incomplete','finalizer_error'):
|
||||
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],expected)
|
||||
if outcome=='contradictory':
|
||||
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
|
||||
self.assertTrue(result['evidenceCompletion']['completionAdjusted'])
|
||||
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
if outcome=='server_incomplete':
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
|
||||
def test_columnar_and_shared_encoding_roundtrips_without_rounding_or_null_loss(self):
|
||||
evidence=[{'rows':[{'field':'电镀磕边重量','weight':None,'count':4120,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}},
|
||||
{'field':'镀废合计','weight':855.801072,'count':431,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}]}]*4
|
||||
self.assertEqual(unpack(finalizer.packed_evidence(evidence)),evidence)
|
||||
|
||||
def test_oversize_evidence_fails_closed_without_discarding_statistics(self):
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||||
finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}])
|
||||
|
||||
def test_real_second_turn_replay_fits_and_preserves_every_summary_value(self):
|
||||
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
|
||||
directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence'))
|
||||
if not directories: self.skipTest('Private captured acceptance evidence not present')
|
||||
with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')):
|
||||
docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c')
|
||||
self.assertEqual(len(docs),15)
|
||||
self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
|
||||
# This captured run predates paired black/bright defect reads for each
|
||||
# highlighted product, so it is now correctly labelled incomplete.
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化'))
|
||||
# Ordinary analysis must not be blocked by minor source reconciliation issues.
|
||||
missing=[d for d in docs if not(d['result'].get('action')=='source' and d['result'].get('stage')=='bright')]
|
||||
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
|
||||
# Full raw difference pagination is optional for an ordinary statistics question.
|
||||
missing=[d for d in docs if not(d['result'].get('action')=='rows' and d['result'].get('stage')=='bright')]
|
||||
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
|
||||
|
||||
def test_previous_turn_evidence_is_excluded(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root=Path(directory)
|
||||
for index in range(2):
|
||||
path=root/('nao-quality-'+str(index)*32+'.json')
|
||||
path.write_text(json.dumps({'tool':'quality_data','releaseId':'release','evidenceRef':'Q0000000'+str(index),
|
||||
'mode':'compare','result':{'releaseId':'release'},'createdAt':str(index)}))
|
||||
path.chmod(0o600)
|
||||
docs=finalizer.evidence_summaries(root,'release',exclude_files={'nao-quality-'+'0'*32+'.json'})
|
||||
self.assertEqual([d['evidenceRef'] for d in docs],['Q00000001'])
|
||||
|
||||
def test_identity_permissions_and_duplicate_refs_are_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root=Path(directory);path=root/('nao-quality-'+'a'*32+'.json')
|
||||
doc={'tool':'quality_data','releaseId':'r','evidenceRef':'Q00000001','mode':'query','result':{'releaseId':'r'}}
|
||||
path.write_text(json.dumps(doc));path.chmod(0o644)
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
|
||||
path.chmod(0o600)
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'other')
|
||||
twin=root/('nao-quality-'+'b'*32+'.json');twin.write_text(json.dumps(doc));twin.chmod(0o600)
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
|
||||
|
||||
def test_cancel_failed_stop_and_incomplete_tools_never_start_finalization(self):
|
||||
calls=[{'status':'completed'}]
|
||||
args=['mimo-v2.6-flash','PARTIAL','timeout',calls,False,True]
|
||||
self.assertTrue(finalizer.eligible(*args))
|
||||
for position,value in ((4,True),(5,False),(5,None),(3,[{'status':'started'}]),(2,'transport_or_stream_error')):
|
||||
changed=list(args);changed[position]=value
|
||||
self.assertFalse(finalizer.eligible(*changed))
|
||||
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','timeout',calls,False,True))
|
||||
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','coverage_ready',calls,False,True))
|
||||
|
||||
def test_full_detail_requires_all_pages_but_accepts_contiguous_completed_pagination(self):
|
||||
evidence=[{'mode':'compare','result':{}}]
|
||||
for offset,count,more in ((0,2,True),(2,1,False)):
|
||||
evidence.append({'mode':'investigate','result':{'action':'rows','current':{
|
||||
'context':{'month':9},'matchingRows':3,'offset':offset,'returnedRows':count,
|
||||
'hasMore':more,'differenceRecords':[{}]*count}}})
|
||||
self.assertTrue(finalizer.coverage_sufficient(evidence,'全部明细'))
|
||||
self.assertFalse(finalizer.coverage_sufficient(evidence[:-1],'全部明细'))
|
||||
|
||||
def test_synthesis_is_one_bounded_openai_compatible_request_with_verified_citations(self):
|
||||
calls=[]
|
||||
def opener(request,timeout):
|
||||
calls.append((json.loads(request.data),timeout))
|
||||
return io.BytesIO(json.dumps({'model':'mimo-v2.6-flash','choices':[{'finish_reason':'stop','message':{
|
||||
'content':json.dumps({'otherFactIds':['F1'],'groupFactIds':[],'productFactIds':[],'defectFactIds':[],'answer':'伪造上升方向和QDEADBEEF引用必须忽略','evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||||
'delta':{'percentagePoints':-1}}}]
|
||||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertTrue(result['evidenceSufficient']);self.assertEqual(len(calls),1)
|
||||
self.assertIn('下降1.00个百分点',result['answer']);self.assertNotIn('QDEADBEEF',result['answer']);self.assertNotIn('伪造',result['answer'])
|
||||
self.assertEqual(calls[0][0]['thinking'],{'type':'disabled'})
|
||||
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
||||
self.assertEqual(calls[0][0]['max_completion_tokens'],MAX_OUTPUT_TOKENS)
|
||||
self.assertEqual(calls[0][1],finalizer.FINAL_TIMEOUT_SECONDS)
|
||||
cancelled=threading.Event();cancelled.set()
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||||
finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||||
question='质量如何',evidence=evidence,cancel_event=cancelled,opener=opener)
|
||||
self.assertEqual(len(calls),1)
|
||||
|
||||
def test_selector_timeout_uses_verified_report_without_claiming_missing_coverage(self):
|
||||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{
|
||||
'kind':'quality_comparison',
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||||
'delta':{'percentagePoints':-1}}}]
|
||||
def timeout(request, timeout):
|
||||
self.assertEqual(timeout, finalizer.FINAL_TIMEOUT_SECONDS)
|
||||
raise TimeoutError('model selector timed out')
|
||||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',
|
||||
model='mimo-v2.6-flash',question='质量如何',evidence=evidence,
|
||||
cancel_event=threading.Event(),opener=timeout)
|
||||
self.assertTrue(result['selectionFallback'])
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertIn('下降1.00个百分点',result['answer'])
|
||||
|
||||
def test_model_cannot_override_missing_stage_or_ambiguous_frozen_frame(self):
|
||||
from test_nao_evidence_report import coverage_comparison, comparison
|
||||
def opener(request,timeout):
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||||
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||||
missing=coverage_comparison()
|
||||
missing['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
|
||||
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||||
question='比较8月相对7月',evidence=[missing],cancel_event=threading.Event(),opener=opener)
|
||||
self.assertTrue(result['modelDeclaredEvidenceSufficient'])
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertFalse(finalizer.coverage_sufficient([missing],'比较8月相对7月'))
|
||||
self.assertIn('没有可引用的正式综合合格率',result['answer'])
|
||||
self.assertNotIn('100.00%',result['answer'])
|
||||
docs=[comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)]
|
||||
unresolved=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||||
question='只看组A',evidence=docs,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertFalse(unresolved['evidenceSufficient'])
|
||||
self.assertIn('组A',unresolved['answer']);self.assertIn('80.00%',unresolved['answer'])
|
||||
context={'year':2026,'month':7,'group':'组A'}
|
||||
bound=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||||
question='只看组A',evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertTrue(bound['evidenceSufficient'])
|
||||
self.assertTrue(finalizer.coverage_sufficient(docs,'只看组A',context))
|
||||
self.assertIn('80.00%',bound['answer']);self.assertNotIn('97.00%',bound['answer'])
|
||||
|
||||
def test_comparison_required_task_does_not_get_rechecked_as_single_period(self):
|
||||
question = '全面分析质量变化'
|
||||
current = {'year': 2026, 'month': 9}
|
||||
comparison = {'year': 2026, 'month': 8}
|
||||
context = {'releaseId': 'release-a', '_agentTask': {
|
||||
'version': 1, 'question': question, 'kind': 'analysis', 'freshQuery': True,
|
||||
'dimensions': ['department', 'group', 'product', 'defect'],
|
||||
'stages': ['black', 'bright'], 'fields': ['rate'],
|
||||
'period': current, 'comparisonPeriod': None, 'comparisonRequired': True,
|
||||
'filters': {}, 'verifySource': False, 'fullDetails': False,
|
||||
}}
|
||||
evidence = [{'evidenceRef': 'Q' + '4' * 8, 'mode': 'compare',
|
||||
'result': {'kind': 'quality_comparison', 'releaseId': 'release-a',
|
||||
'filterBinding': {},
|
||||
'current': {'period': current},
|
||||
'comparison': {'period': comparison}}}]
|
||||
with patch('nao_analysis_coverage.coverage_gaps', return_value=[]), \
|
||||
patch('nao_analysis_coverage.semantic_single_period_gaps',
|
||||
side_effect=AssertionError('comparison task was treated as single period')), \
|
||||
patch('nao_analysis_coverage.formal_metric_coverage', return_value=None), \
|
||||
patch('nao_evidence_report.build_report', return_value={'coverageSufficient': True}):
|
||||
self.assertTrue(finalizer.coverage_sufficient(evidence, question, context))
|
||||
|
||||
def test_optional_focus_metadata_cannot_discard_or_invent_verified_statistics(self):
|
||||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||||
'delta':{'percentagePoints':-1}}}]
|
||||
def opener(request,timeout):
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||||
'otherFactIds':['invented','F1','F1','F2','F3'], 'groupFactIds':['F1'],
|
||||
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertTrue(result['selectionAdjusted'])
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertEqual(result['selectedFactIds'],['F1','F2'])
|
||||
self.assertNotIn('invented',result['answer'])
|
||||
self.assertIn('下降1.00个百分点',result['answer'])
|
||||
|
||||
def test_completion_requires_concrete_missing_items_and_keeps_model_diagnostics(self):
|
||||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||||
'delta':{'percentagePoints':-1}}}]
|
||||
for declared,missing,expected in [(False,[],True),(False,[' ',''],True),
|
||||
(False,[' 缺少对照期 ','缺少对照期'],False),(True,['缺少对照期'],False),(True,[],True)]:
|
||||
with self.subTest(declared=declared,missing=missing):
|
||||
def opener(request,timeout):
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||||
'evidenceSufficient':declared,'missingEvidence':missing})}}]}).encode())
|
||||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(result['evidenceSufficient'],expected)
|
||||
self.assertEqual(result['modelDeclaredEvidenceSufficient'],declared)
|
||||
self.assertEqual(result['modelMissingEvidence'],missing)
|
||||
self.assertEqual(result['completionAdjusted'],not declared and expected)
|
||||
self.assertEqual(result['missingEvidence'],[] if expected else ['缺少对照期'])
|
||||
if not expected:self.assertIn('缺少对照期',result['answer'])
|
||||
|
||||
def test_captured_3c_with_missing_paired_stage_remains_partial(self):
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
paths=list((root/'.runtime/nao-users').glob('*/sessions/98829d63-0158-4e70-89db-ef0d58fe546a'))
|
||||
if not paths:self.skipTest('Private captured session not present')
|
||||
original=json.loads((paths[0]/'finalization.json').read_text())
|
||||
self.assertFalse(original['modelDeclaredEvidenceSufficient'])
|
||||
self.assertEqual(original['missingEvidence'],[])
|
||||
self.assertTrue(original['serverCoverageSufficient'])
|
||||
release=json.loads((root/'output/20261023-incomplete-reason/session.json').read_text())['releaseId']
|
||||
docs=finalizer.evidence_summaries(paths[0]/'evidence',release)
|
||||
def opener(request,timeout):
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||||
'evidenceSufficient':False,'missingEvidence':[]})}}]}).encode())
|
||||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||||
question='3C哪些产品近期表现最差',evidence=docs,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs,'3C哪些产品近期表现最差'))
|
||||
self.assertFalse(result['completionAdjusted'])
|
||||
self.assertTrue(any('缺少' in item for item in result['missingEvidence']))
|
||||
self.assertIn('Q039D5FA7',result['answer'])
|
||||
self.assertIn('90.11%',result['answer'])
|
||||
|
||||
def test_exact_formal_metrics_refute_only_the_model_claims_they_prove_false(self):
|
||||
captured = []
|
||||
def opener(request, timeout):
|
||||
captured.append(json.loads(request.data))
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||||
'evidenceSufficient': False, 'missingEvidence': FORMAL_PRODUCT_GAPS})}}]}).encode())
|
||||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertEqual(result['missingEvidence'], [])
|
||||
self.assertEqual(result['modelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||||
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||||
self.assertTrue(result['completionAdjusted'])
|
||||
selector = json.loads(captured[0]['messages'][1]['content'])
|
||||
contract = selector['formalMetricCoverage']
|
||||
self.assertEqual(contract['filters'], {'product': 'SYNTH-16X15+T'})
|
||||
self.assertEqual(len(contract['metrics']), 12)
|
||||
self.assertTrue(all(row['covered'] for row in contract['metrics']))
|
||||
self.assertIn(1200.5, [row['value'] for row in contract['metrics']])
|
||||
self.assertNotIn('未覆盖内容', result['answer'])
|
||||
|
||||
def test_formal_metric_contract_rejects_wrong_scope_period_release_and_incomplete_values(self):
|
||||
mutations = {
|
||||
'product': lambda r: r['filterBinding'].update(product='SYNTH-16X15+T-OTHER'),
|
||||
'extra_filter': lambda r: r['filterBinding'].update(group='其他分组'),
|
||||
'period': lambda r: r['comparison']['period'].update(month=7),
|
||||
'release': lambda r: r.update(releaseId='release-other'),
|
||||
'input': lambda r: r['current']['black'].pop('totalWeight'),
|
||||
'scrap': lambda r: r['comparison']['bright'].pop('scrapWeight'),
|
||||
'rate': lambda r: r['current']['bright'].update(rate=None),
|
||||
'nonfinite': lambda r: r['current']['black'].update(totalWeight='NaN'),
|
||||
'source_months': lambda r: r['comparison'].pop('coverage'),
|
||||
'source_stage': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0),
|
||||
}
|
||||
for name, mutate in mutations.items():
|
||||
with self.subTest(name=name):
|
||||
docs = copy.deepcopy(formal_product_evidence()); mutate(docs[0]['result'])
|
||||
def opener(request, timeout):
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||||
'evidenceSufficient': True, 'missingEvidence': []})}}]}).encode())
|
||||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||||
question=FORMAL_PRODUCT_QUESTION, evidence=docs, context={'releaseId': 'release-a'},
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertTrue(result['missingEvidence'])
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, {'releaseId': 'release-a'}))
|
||||
|
||||
def test_unknown_mixed_and_other_scope_model_gaps_still_block_completion(self):
|
||||
for missing in ['缺少该规格投入重量和现场原因证据', '缺少该规格投入重量原始明细',
|
||||
'缺少其他规格的投入重量正式数据', '缺少2026年7月投入重量正式数据',
|
||||
'缺少对照期', '缺少生产原因证据']:
|
||||
with self.subTest(missing=missing):
|
||||
def opener(request, timeout):
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||||
'evidenceSufficient': False, 'missingEvidence': [*FORMAL_PRODUCT_GAPS, missing]})}}]}).encode())
|
||||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertEqual(result['missingEvidence'], [missing])
|
||||
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||||
|
||||
def test_metric_reconciliation_does_not_infer_unspecified_scope_or_hide_later_missing_items(self):
|
||||
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps
|
||||
for question in [FORMAL_PRODUCT_QUESTION.replace('只看SYNTH-16X15+T,', ''),
|
||||
FORMAL_PRODUCT_QUESTION.replace('2026年', ''),
|
||||
FORMAL_PRODUCT_QUESTION.replace('黑片、亮片', ''),
|
||||
FORMAL_PRODUCT_QUESTION.replace('9月与8月', '13月与8月')]:
|
||||
with self.subTest(question=question):
|
||||
contract = formal_metric_coverage(formal_product_evidence(), question, {'releaseId': 'release-a'})
|
||||
self.assertIsNone(contract)
|
||||
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
|
||||
contract = formal_metric_coverage(formal_product_evidence(), FORMAL_PRODUCT_QUESTION.replace('报废重量和', ''), {'releaseId': 'release-a'})
|
||||
self.assertEqual(resolve_proven_metric_gaps(contract, [FORMAL_PRODUCT_GAPS[1]]), ([FORMAL_PRODUCT_GAPS[1]], []))
|
||||
gaps = ['缺少' + '的' * index + '该规格投入重量正式数据' for index in range(12)] + ['缺少生产原因证据']
|
||||
def opener(request, timeout):
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||||
'evidenceSufficient': False, 'missingEvidence': gaps})}}]}).encode())
|
||||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||||
cancel_event=threading.Event(), opener=opener)
|
||||
self.assertFalse(result['evidenceSufficient'])
|
||||
self.assertEqual(result['missingEvidence'], ['缺少生产原因证据'])
|
||||
self.assertEqual(result['modelMissingEvidence'], gaps)
|
||||
|
||||
def test_formal_contract_accepts_the_same_explicit_frozen_filters(self):
|
||||
from nao_analysis_coverage import formal_metric_coverage
|
||||
for filters in [{'group': '合成分组'}, {'department': '合成部门'},
|
||||
{'group': '合成分组', 'extraFilters': {'region': '合成地区'}, 'includeUnassigned': True}]:
|
||||
with self.subTest(filters=filters):
|
||||
docs = formal_product_evidence()
|
||||
docs[0]['result']['filterBinding'].update(filters)
|
||||
expected = {'product': 'SYNTH-16X15+T', **filters}
|
||||
context = {'releaseId': 'release-a', 'filters': expected}
|
||||
contract = formal_metric_coverage(docs, FORMAL_PRODUCT_QUESTION, context)
|
||||
self.assertEqual(contract['filters'], expected)
|
||||
self.assertEqual(contract['missingEvidence'], [])
|
||||
self.assertTrue(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, context))
|
||||
|
||||
def test_unknown_extra_scope_or_analysis_language_does_not_activate_the_contract(self):
|
||||
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps, coverage_gaps
|
||||
docs = formal_product_evidence()
|
||||
docs[0]['result']['filterBinding']['group'] = '合成分组'
|
||||
for question in [FORMAL_PRODUCT_QUESTION + '仅统计合成分组。',
|
||||
FORMAL_PRODUCT_QUESTION.replace(',请', ',合成部门,分析人员因素,请'),
|
||||
'合成分组中,' + FORMAL_PRODUCT_QUESTION,
|
||||
FORMAL_PRODUCT_QUESTION + '并提供现场生产原因。']:
|
||||
with self.subTest(question=question):
|
||||
contract = formal_metric_coverage(docs, question, {'releaseId': 'release-a'})
|
||||
self.assertIsNone(contract)
|
||||
self.assertEqual(coverage_gaps(docs, question, {'releaseId': 'release-a'}), [])
|
||||
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,211 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_evidence_report import build_report,render_report,product_priority_summary
|
||||
from nao_evidence_finalizer import evidence_summaries
|
||||
|
||||
|
||||
def comparison(ref='Q00000001', filters=None, rate=.95, month=7):
|
||||
def coverage():
|
||||
return {'sourceMonths': [{'year': 2026, 'month': selected,
|
||||
'blackRows': 2, 'brightRows': 2}
|
||||
for selected in range(1, 13)]}
|
||||
return {'evidenceRef':ref,'mode':'compare','result':{'kind':'quality_comparison','filterBinding':filters or {},
|
||||
'current':{'period':{'year':2026,'month':month},'compositeRate':rate,'coverage':coverage()},
|
||||
'comparison':{'period':{'year':2026,'month':month-1},'compositeRate':.96,'coverage':coverage()},
|
||||
'delta':{'available':True,'percentagePoints':(rate-.96)*100}}}
|
||||
|
||||
|
||||
def coverage_comparison():
|
||||
from nao_quality_tools import QualityTools
|
||||
def metrics(month):
|
||||
missing=month==8
|
||||
return {'summary':{'compositeRate':.9 if missing else .95,
|
||||
'black':{'rate':1 if missing else .97,'qualifiedWeight':None if missing else 97,'totalWeight':None if missing else 100,'scrapWeight':None if missing else 3},
|
||||
'bright':{'rate':.9 if missing else .98,'qualifiedWeight':90 if missing else 98,'totalWeight':100,'scrapWeight':10 if missing else 2}},
|
||||
'coverage':{'rawRows':50 if missing else 100,'includedRows':50 if missing else 100,
|
||||
'sourceMonths':[{'year':2026,'month':month,'blackRows':0 if missing else 50,'brightRows':50}]}}
|
||||
tools=QualityTools.__new__(QualityTools)
|
||||
tools._metrics=lambda release,period,filters,view:metrics(period['month'])
|
||||
return {'mode':'compare','evidenceRef':'Q00000004','result':tools._compare({'id':'synthetic'},
|
||||
{'year':2026,'month':8},{'year':2026,'month':7},{},None,None)}
|
||||
|
||||
|
||||
class BoundReportTest(unittest.TestCase):
|
||||
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
|
||||
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
|
||||
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
|
||||
'rateDrop':[{'label':'小量规格','rateDrop':0.8,
|
||||
'comparison':{'blackInputWeight':400,'brightInputWeight':300},
|
||||
'current':{'blackInputWeight':4,'brightInputWeight':3}}]}
|
||||
text=product_priority_summary(signals,None)
|
||||
self.assertIn('放量规格(+120.00 kg)',text)
|
||||
self.assertIn('存量规格(500.00 kg)',text)
|
||||
self.assertIn('小量规格(下降80.00个百分点',text)
|
||||
self.assertIn('黑片投入400.00→4.00 kg',text)
|
||||
self.assertIn('不能相加为总体贡献',text)
|
||||
|
||||
def test_frozen_group_period_anchor_wins_over_global_regardless_of_order(self):
|
||||
global_doc=comparison('Q00000001',{},.97)
|
||||
local=comparison('Q00000002',{'group':'组A'},.80)
|
||||
another_month=comparison('Q00000003',{'group':'组A'},.70,6)
|
||||
for docs in ([global_doc,local,another_month],[another_month,local,global_doc]):
|
||||
report=build_report(docs,{'year':2026,'month':7,'group':'组A'})
|
||||
self.assertIn('80.00%',report['headline']);self.assertNotIn('97.00%',report['headline'])
|
||||
self.assertIn('业务分组=组A',report['policy'])
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
|
||||
def test_ambiguous_frames_stay_separate_and_cannot_claim_completion(self):
|
||||
report=build_report([comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)])
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
text=render_report(report,[report['facts'][0]['id']],report['missingEvidence'])
|
||||
self.assertIn('97.00%',text);self.assertIn('80.00%',text);self.assertIn('业务分组=组A',text)
|
||||
self.assertIn('多个期间和范围',text)
|
||||
mismatch=build_report([comparison()],{'year':2026,'month':7,'group':'未查组'})
|
||||
self.assertFalse(mismatch['coverageSufficient'])
|
||||
self.assertIn('未匹配',mismatch['missingEvidence'][0])
|
||||
|
||||
def test_annual_and_quarter_anchors_do_not_match_months(self):
|
||||
monthly=comparison('Q00000001',{},.95)
|
||||
annual=comparison('Q00000002',{},.85)
|
||||
annual['result']['current']['period']={'year':2026}
|
||||
annual['result']['comparison']['period']={'year':2025}
|
||||
quarterly=comparison('Q00000003',{},.75)
|
||||
quarterly['result']['current']['period']={'year':2026,'quarter':3}
|
||||
quarterly['result']['comparison']['period']={'year':2026,'quarter':2}
|
||||
docs=[monthly,annual,quarterly]
|
||||
self.assertIn('85.00%',build_report(docs,{'year':2026})['headline'])
|
||||
self.assertIn('75.00%',build_report(docs,{'year':2026,'quarter':3})['headline'])
|
||||
|
||||
def test_other_period_defects_cannot_borrow_root_column_labels(self):
|
||||
docs=[comparison(),{'evidenceRef':'Q00000003','mode':'defects','result':{
|
||||
'kind':'quality_fine_defects','stage':'black','dimension':'defect','filters':{},
|
||||
'period':{'year':2026,'month':6},'comparisonPeriod':{'year':2026,'month':5},
|
||||
'categoryReconciliation':[],
|
||||
'defectColumns':['field','category','currentWeight','comparisonWeight','deltaWeight','deltaAvailable'],
|
||||
'defectValues':[['合成磕边重量','料废',30,10,20,True]]}}]
|
||||
report=build_report(docs)
|
||||
text=render_report(report,[report['facts'][0]['id']],[])
|
||||
self.assertNotIn('合成磕边重量',text)
|
||||
self.assertIn('其他期间证据未合并',text)
|
||||
self.assertIn('[Q00000003]',text)
|
||||
|
||||
def test_real_tool_missing_stage_remains_missing_and_raw_value_is_traceable(self):
|
||||
doc=coverage_comparison()
|
||||
self.assertTrue(doc['result']['delta']['available'])
|
||||
report=build_report([doc])
|
||||
text=render_report(report,[report['facts'][0]['id']],report['missingEvidence'])
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('缺失的黑片来源月份:2026-08',text)
|
||||
self.assertNotIn('100.00%',text)
|
||||
self.assertIn('| 黑片合格率 | 97.00% | — | 不可比 |',text)
|
||||
self.assertEqual(report['rawMetrics']['current']['black']['rate'],1)
|
||||
self.assertEqual(doc['result']['current']['black']['rate'],1)
|
||||
|
||||
def test_unknown_comparison_coverage_stays_incomplete(self):
|
||||
doc=comparison()
|
||||
for side in ('current','comparison'):
|
||||
doc['result'][side].pop('coverage')
|
||||
report=build_report([doc])
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn('没有可引用的正式综合合格率', ';'.join(report['missingEvidence']))
|
||||
self.assertIn('—→', report['headline'])
|
||||
|
||||
def test_frozen_stage_uses_its_formal_delta_and_other_stage_does_not_block_it(self):
|
||||
doc=coverage_comparison()
|
||||
# The synthetic tool result has bright coverage in both periods, even
|
||||
# though no composite comparison is possible without the black source.
|
||||
report=build_report([doc],{'year':2026,'month':8,'stage':'bright'})
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertIn('亮片合格率',report['headline'])
|
||||
self.assertIn('98.00%→2026年8月90.00%',report['headline'])
|
||||
self.assertIn('下降8.00个百分点',report['headline'])
|
||||
self.assertIn('缺失的黑片来源月份',report['policy'])
|
||||
black=build_report([doc],{'year':2026,'month':8,'stage':'black'})
|
||||
self.assertFalse(black['coverageSufficient'])
|
||||
self.assertNotIn('100.00%',black['headline'])
|
||||
|
||||
def test_chapter_leads_are_bound_facts_before_their_data_tables(self):
|
||||
docs=[{'evidenceRef':'Q00000001','result':{'kind':'quality_comparison','filterBinding':{},
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||||
'black':{'rate':0.97,'totalWeight':100,'qualifiedWeight':97,'defects':[{'name':'责废','weight':12},{'name':'料废','weight':5}]},
|
||||
'bright':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98},
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2,'brightRows':2}]}},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||||
'black':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98,'defects':[{'name':'责废','weight':8},{'name':'料废','weight':4}]},
|
||||
'bright':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98},
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':2,'brightRows':2}]}},
|
||||
'delta':{'percentagePoints':-1},'stageRateChangesPP':{'black':-1,'bright':0}}},
|
||||
{'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'group','filters':{},
|
||||
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
|
||||
'rows':[{'label':'改善组','current':{'rate':0.93},'comparison':{'rate':0.90},'rateDrop':-0.03,'scrapDelta':-10},
|
||||
{'label':'下降组','current':{'rate':0.93},'comparison':{'rate':0.95},'rateDrop':0.02,'scrapDelta':12}]}}]
|
||||
report=build_report(docs)
|
||||
stage=next(s for s in report['sections'] if s.startswith('### 总体与阶段'))
|
||||
self.assertLess(stage.index('黑片合格率:98.00%→97.00%,下降1.00个百分点。[Q00000001]'),stage.index('| 指标'))
|
||||
category=next(s for s in report['sections'] if s.startswith('### 正式报废类别'))
|
||||
self.assertIn('黑片责废:8.00→12.00 kg,增减+4.00 kg。[Q00000001]',category.split('| 类别')[0])
|
||||
group=next(s for s in report['sections'] if s.startswith('### 业务分组变化'))
|
||||
self.assertIn('本次返回的组织中,下降组',group.split('| 范围')[0])
|
||||
self.assertNotIn('生产主因', '\n'.join(report['sections']))
|
||||
|
||||
def test_actual_wrong_group_directions_and_product_citations_cannot_be_generated(self):
|
||||
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
|
||||
paths=list(root.glob('.runtime/nao-users/*/sessions/4c3c5e05-3c81-4c43-acf4-3e70af3ab7c7/evidence'))
|
||||
if not paths:self.skipTest('Private real acceptance evidence not present')
|
||||
docs=evidence_summaries(paths[0],'194b0287-9044-4d80-9138-ecd3c481015c')
|
||||
report=build_report(docs)
|
||||
groups={row['label']:row for row in report['facts'] if row['kind']=='group'}
|
||||
self.assertIn('下降0.38个百分点',groups['工业电机-中国区']['text'])
|
||||
self.assertIn('下降0.05个百分点',groups['工业电机-VCM']['text'])
|
||||
self.assertIn('上升3.12个百分点',groups['工业电机-亚太']['text'])
|
||||
product=next(row for row in report['facts'] if row['kind']=='product')
|
||||
self.assertEqual(product['evidenceRef'],'QE8DBF76C')
|
||||
text=render_report(report,[groups['工业电机-中国区']['id'],groups['工业电机-VCM']['id'],product['id']],[])
|
||||
self.assertIn('本部报废',text);self.assertIn('VCM亮片仅录合计',text)
|
||||
self.assertNotIn('中国区、VCM、亚太合格率小幅上行',text)
|
||||
with self.assertRaises(ValueError):render_report(report,['F999999'],[])
|
||||
|
||||
def test_missing_rates_are_not_imputed_to_zero(self):
|
||||
docs=[{'evidenceRef':'Q00000001','result':{'kind':'quality_comparison','filterBinding':{},
|
||||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95},
|
||||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96},'delta':{'percentagePoints':-1}}},
|
||||
{'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'group','filters':{},
|
||||
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
|
||||
'rows':[{'label':'新进组','current':{'rate':0.93},'comparison':{'rate':None},'rateDrop':None,'scrapDelta':12}]}}]
|
||||
report=build_report(docs);group=next(row for row in report['facts'] if row['kind']=='group')
|
||||
self.assertIn('不可比',group['text']);self.assertIn('—→93.00%',group['text']);self.assertNotIn('0.00%→',group['text'])
|
||||
|
||||
def test_rate_drop_product_remains_a_volume_qualified_signal_with_exact_pp(self):
|
||||
root=comparison(month=9)
|
||||
product={'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'product','stage':None,
|
||||
'filters':{},'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
|
||||
'sort':'rate_drop','totalRows':3566,'rows':[{'label':'小量规格','rateDrop':0.9943484950890867,
|
||||
'scrapDelta':6.737,'current':{'rate':0,'scrapWeight':9.095,'blackInputWeight':7.1,'brightInputWeight':39.656},
|
||||
'comparison':{'rate':0.9943484950890867,'scrapWeight':2.358,'blackInputWeight':481.48,'brightInputWeight':417.234}}]}}
|
||||
report=build_report([root,product])
|
||||
section=next(s for s in report['sections'] if s.startswith('### 产品率变化线索'))
|
||||
self.assertNotIn('### 重点产品',section)
|
||||
self.assertIn('下降99.43个百分点',section)
|
||||
self.assertIn('黑片投入481.48→7.10 kg',section)
|
||||
self.assertIn('亮片投入417.23→39.66 kg',section)
|
||||
self.assertIn('不能只凭率降幅确定重点',section)
|
||||
self.assertIn('仅是率变化线索',next(f for f in report['facts'] if f['kind']=='product_rate_signal')['text'])
|
||||
self.assertFalse(any(f['kind']=='product' for f in report['facts']))
|
||||
|
||||
def test_stage_rates_and_totals_only_vcm_detail_keep_their_actual_meaning(self):
|
||||
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
|
||||
paths=list(root.glob('.runtime/nao-users/*/sessions/24e16a62-0527-4bff-ba9f-535005acbfaf/evidence'))
|
||||
if not paths:self.skipTest('Private real acceptance evidence not present')
|
||||
report=build_report(evidence_summaries(paths[0],'194b0287-9044-4d80-9138-ecd3c481015c'))
|
||||
text=render_report(report,[report['facts'][0]['id']],[])
|
||||
self.assertIn('黑片业务分组变化',text)
|
||||
self.assertIn('亮片业务分组变化',text)
|
||||
self.assertIn('黑片合格率:比较期→本期',text)
|
||||
self.assertFalse(any(row['kind']=='cross' and 'VCM607+Z' in row['label'] for row in report['facts']))
|
||||
self.assertIn('VCM亮片仅录合计',text)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -16,8 +16,12 @@ class FinalizerFallbackTests(unittest.TestCase):
|
||||
def test_malformed_optional_focus_uses_verified_report_without_failure(self):
|
||||
evidence = [{"evidenceRef": "Q00000001", "mode": "compare", "result": {
|
||||
"kind": "quality_comparison",
|
||||
"current": {"period": {"year": 2026, "month": 9}, "compositeRate": 0.95},
|
||||
"comparison": {"period": {"year": 2026, "month": 8}, "compositeRate": 0.96},
|
||||
"current": {"period": {"year": 2026, "month": 9}, "compositeRate": 0.95,
|
||||
"coverage": {"sourceMonths": [{"year": 2026, "month": 9,
|
||||
"blackRows": 1, "brightRows": 1}]}},
|
||||
"comparison": {"period": {"year": 2026, "month": 8}, "compositeRate": 0.96,
|
||||
"coverage": {"sourceMonths": [{"year": 2026, "month": 8,
|
||||
"blackRows": 1, "brightRows": 1}]}},
|
||||
"delta": {"percentagePoints": -1}}}]
|
||||
bad_responses = [b"not-json",
|
||||
json.dumps({"choices": [{"finish_reason": "length", "message": {
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
||||
from nao_history_report import (HistorySnapshotError, execute_history_report,
|
||||
select_history_headline, validate_history_snapshot,
|
||||
_validate_model_selection)
|
||||
from nao_model_gate import ModelGate
|
||||
from nao_protocol_execution import execute
|
||||
|
||||
|
||||
RELEASE = 'release-2026-09'
|
||||
ATTEMPT = '11111111-1111-4111-8111-111111111111'
|
||||
SESSION = '22222222-2222-4222-8222-222222222222'
|
||||
TURN = '33333333-3333-4333-8333-333333333333'
|
||||
SOURCE_A = '44444444-4444-4444-8444-444444444444'
|
||||
SOURCE_B = '55555555-5555-4555-8555-555555555555'
|
||||
|
||||
|
||||
def report_task(source_ids=(SOURCE_A, SOURCE_B)):
|
||||
return {
|
||||
'version': 1, 'kind': 'report', 'historyMode': 'all', 'freshQuery': False,
|
||||
'goal': '整合已确认历史质量结果', 'dimensions': [], 'stages': [], 'fields': [],
|
||||
'comparisonRequired': False, 'trendRequired': False, 'verifySource': False,
|
||||
'fullDetails': False, 'filters': {}, 'period': None, 'comparisonPeriod': None,
|
||||
'comparisonMode': None, 'dimensionStages': {},
|
||||
'historySourceTurnIds': list(source_ids), 'needsClarification': False,
|
||||
}
|
||||
|
||||
|
||||
def authority(task=None):
|
||||
return {'attemptId': ATTEMPT, 'sessionId': SESSION, 'turnId': TURN,
|
||||
'revision': 3, 'executionId': '66666666-6666-4666-8666-666666666666',
|
||||
'task': copy.deepcopy(task if task is not None else report_task())}
|
||||
|
||||
|
||||
def snapshot(*, unavailable_second=False, facts=None):
|
||||
sources = [
|
||||
{'turnId': SOURCE_A, 'revision': 2, 'executionId': 'execution-a',
|
||||
'status': 'COMPLETED', 'available': True, 'queryIds': ['query-a']},
|
||||
{'turnId': SOURCE_B, 'revision': 1, 'executionId': 'execution-b',
|
||||
'status': 'PARTIAL' if unavailable_second else 'COMPLETED',
|
||||
'available': not unavailable_second,
|
||||
**({'revision': None, 'executionId': None} if unavailable_second else {'queryIds': ['query-b']}),
|
||||
**({'reason': '2025年9月海外组没有可用历史来源。'} if unavailable_second else {})},
|
||||
]
|
||||
sections = [{
|
||||
'title': '2026年9月黑片正式报废重量', 'sourceTurnId': SOURCE_A,
|
||||
'period': {'year': 2026, 'month': 9},
|
||||
'filters': {'extraFilters': {'customerCode': ['800001']},
|
||||
'columnFilters': [{'field': 'orderNo', 'stages': ['black'],
|
||||
'operator': 'in', 'values': ['WO-1']}]},
|
||||
'stage': 'black',
|
||||
'columns': [{'key': 'label', 'label': '对象'},
|
||||
{'key': 'scrapWeight', 'label': '报废重量(kg)'}],
|
||||
'rows': [{'label': '整体', 'scrapWeight': 1163.6817}],
|
||||
'note': '历史正式查询结果,本轮未重新计算',
|
||||
}]
|
||||
if not unavailable_second:
|
||||
sections.append({
|
||||
'title': '2025年9月亮片正式报废重量', 'sourceTurnId': SOURCE_B,
|
||||
'period': {'year': 2025, 'month': 9}, 'filters': {'group': '汽车交通-海外'},
|
||||
'stage': 'bright',
|
||||
'columns': [{'key': 'label', 'label': '对象'},
|
||||
{'key': 'scrapWeight', 'label': '报废重量(kg)'}],
|
||||
'rows': [{'label': '整体', 'scrapWeight': 17.25}],
|
||||
})
|
||||
return {
|
||||
'kind': 'protocol_history_snapshot', 'version': 1, 'releaseId': RELEASE,
|
||||
'historySources': sources, 'sections': sections,
|
||||
'facts': facts if facts is not None else [
|
||||
{'id': 'history-fact:F1', 'text': '2026年9月,客户800001,黑片正式报废重量1163.6817 kg。',
|
||||
'sourceTurnId': SOURCE_A},
|
||||
*([] if unavailable_second else [{
|
||||
'id': 'history-fact:F2',
|
||||
'text': '2025年9月,汽车交通-海外亮片正式报废重量17.25 kg。',
|
||||
'sourceTurnId': SOURCE_B,
|
||||
}]),
|
||||
],
|
||||
'historyCoverage': {
|
||||
'complete': not unavailable_second, 'expectedSources': 2,
|
||||
'importedSources': 1 if unavailable_second else 2,
|
||||
'missingEvidence': ['2025年9月一条确认历史来源暂不可用。'] if unavailable_second else [],
|
||||
},
|
||||
'historyEvidence': {
|
||||
'importId': 'import-7', 'attemptId': ATTEMPT, 'sessionId': SESSION,
|
||||
'turnId': TURN, 'revision': 3,
|
||||
'executionId': '66666666-6666-4666-8666-666666666666',
|
||||
'snapshotHash': 'sha256:' + 'a' * 64,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class FakeDelegated:
|
||||
def __init__(self, response):
|
||||
self.response = response
|
||||
self.calls = []
|
||||
|
||||
def call(self, path, payload=None, method=None):
|
||||
self.calls.append((path, copy.deepcopy(payload), method))
|
||||
if isinstance(self.response, Exception):
|
||||
raise self.response
|
||||
return copy.deepcopy(self.response)
|
||||
|
||||
|
||||
class HistorySnapshotValidationTest(unittest.TestCase):
|
||||
def test_first_mode_keeps_server_confirmed_sources_and_identity_checks(self):
|
||||
task = report_task((SOURCE_A,)); task['historyMode'] = 'first'
|
||||
candidate = snapshot()
|
||||
candidate['historySources'] = candidate['historySources'][:1]
|
||||
candidate['sections'] = candidate['sections'][:1]
|
||||
candidate['facts'] = candidate['facts'][:1]
|
||||
candidate['historyCoverage'].update(expectedSources=1, importedSources=1)
|
||||
self.assertEqual(validate_history_snapshot(candidate, task=task, authority=authority(task),
|
||||
release_id=RELEASE, request_turn_id=TURN)['historySources'][0]['turnId'], SOURCE_A)
|
||||
wrong = copy.deepcopy(candidate); wrong['historySources'][0]['turnId'] = SOURCE_B
|
||||
with self.assertRaises(HistorySnapshotError):
|
||||
validate_history_snapshot(wrong, task=task, authority=authority(task), release_id=RELEASE, request_turn_id=TURN)
|
||||
with self.assertRaises(HistorySnapshotError):
|
||||
validate_history_snapshot(candidate, task=task, authority=authority(), release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
def test_autonomous_helper_rejects_unvalidated_identity_before_model_request(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
candidate = snapshot()
|
||||
candidate['historyEvidence']['turnId'] = SOURCE_A
|
||||
with patch('nao_evidence_finalizer._open_model') as model, self.assertRaises(HistorySnapshotError):
|
||||
synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='汇总', snapshot=candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event())
|
||||
model.assert_not_called()
|
||||
|
||||
def test_all_published_source_period_is_exact_and_not_a_calendar_override(self):
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0]['period'] = {'allPublished': True}
|
||||
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
for period in ({'allPublished': False}, {'allPublished': 1},
|
||||
{'allPublished': True, 'year': 2026}):
|
||||
candidate['sections'][0]['period'] = period
|
||||
with self.subTest(period=period), self.assertRaises(HistorySnapshotError):
|
||||
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
def test_rejects_wrong_source_release_and_identity(self):
|
||||
task = report_task()
|
||||
auth = authority(task)
|
||||
base = snapshot()
|
||||
for mutate in (
|
||||
lambda value: value.update(releaseId='other-release'),
|
||||
lambda value: value['historySources'][0].update(turnId='unconfirmed'),
|
||||
lambda value: value['historyEvidence'].update(executionId='other-execution')):
|
||||
candidate = copy.deepcopy(base)
|
||||
mutate(candidate)
|
||||
with self.subTest(candidate=candidate['releaseId']):
|
||||
with self.assertRaises(HistorySnapshotError):
|
||||
validate_history_snapshot(candidate, task=task, authority=auth,
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
def test_rejects_old_protocol_evidence_even_when_nested_in_section(self):
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0]['protocolEvidence'] = {'queryId': 'old-query'}
|
||||
with self.assertRaisesRegex(HistorySnapshotError, '普通查询证明'):
|
||||
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
def test_rejects_unavailable_source_rows_and_inconsistent_complete_coverage(self):
|
||||
candidate = snapshot(unavailable_second=True)
|
||||
candidate['sections'].append(copy.deepcopy(snapshot()['sections'][1]))
|
||||
with self.assertRaises(HistorySnapshotError):
|
||||
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
candidate = snapshot()
|
||||
candidate['historyCoverage']['importedSources'] = 1
|
||||
with self.assertRaisesRegex(HistorySnapshotError, '可用来源不一致'):
|
||||
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
def test_preserves_long_record_cells_but_enforces_global_snapshot_capacity(self):
|
||||
long_cell = '原始长文本' * 600
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0]['columns'].append({'key': 'sourceText', 'label': '原始文本'})
|
||||
candidate['sections'][0]['rows'][0]['sourceText'] = long_cell
|
||||
validated = validate_history_snapshot(candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
self.assertEqual(validated['sections'][0]['rows'][0]['sourceText'], long_cell)
|
||||
|
||||
oversized = snapshot()
|
||||
large_value = 'x' * 3_500_000
|
||||
columns = [{'key': f'cell{index}', 'label': f'文本{index}'} for index in range(5)]
|
||||
oversized['sections'][0]['columns'] = columns
|
||||
oversized['sections'][0]['rows'] = [{column['key']: large_value for column in columns}]
|
||||
with self.assertRaisesRegex(HistorySnapshotError, '快照超出允许大小'):
|
||||
validate_history_snapshot(oversized, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN)
|
||||
|
||||
|
||||
class HistoryReportExecutionTest(unittest.TestCase):
|
||||
def test_scoped_formula_context_reaches_synthesis_and_the_same_review(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
from test_native_answer import response
|
||||
for product in (True, False):
|
||||
candidate = snapshot()
|
||||
task = report_task((SOURCE_A,))
|
||||
candidate['historySources'] = candidate['historySources'][:1]
|
||||
candidate['sections'] = candidate['sections'][:1]
|
||||
candidate['facts'] = candidate['facts'][:1]
|
||||
candidate['historyCoverage'].update(expectedSources=1, importedSources=1)
|
||||
rules = {'supportsStageContribution': product, 'metricRules': {'compositeRate': {
|
||||
'formula': 'blackRate*brightRate' if product else '(blackRate+brightRate)/2'}}}
|
||||
context = {'ruleVersionId': 'formal-version', 'calculationRules': rules,
|
||||
'metricFormulaDescriptions': {'compositeRate': '正式规则说明'}}
|
||||
candidate['sections'][0]['metricContext'] = context
|
||||
candidate['historySources'][0]['metricContexts'] = [context]
|
||||
candidate['facts'][0].update(sectionIndex=0, text='2026年9月综合合格率 = 0.9531',
|
||||
displayText='2026年9月综合合格率95.31%。')
|
||||
phases = []
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data); payload = json.loads(body['messages'][1]['content'])
|
||||
semantics = payload['metricSemantics']
|
||||
self.assertIs(semantics['productConfirmed'], product)
|
||||
supplied = semantics['scopedMetricContexts'][0]
|
||||
self.assertEqual(supplied['calculationRules'], rules)
|
||||
self.assertEqual(payload['scopes'][0]['metricContextId'], supplied['id'])
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
phases.append('review')
|
||||
return response({'reviews': [{'supported': True, 'statisticalClaimsSupported': True}], 'missingGoalDescriptions': []})
|
||||
phases.append('synthesis')
|
||||
return response({'blocks': [{'role': 'core', 'text': '本期综合合格率95.31%,沿用本轮正式组合规则解释。', 'factIds': ['F1']}]})
|
||||
with self.subTest(product=product):
|
||||
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='汇总历史结果', snapshot=candidate, task=task, authority=authority(task),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
|
||||
self.assertEqual(phases, ['synthesis', 'review'])
|
||||
|
||||
def test_display_percent_and_short_month_use_verified_history_metadata(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
from test_native_answer import response
|
||||
candidate = snapshot()
|
||||
candidate['facts'][0].update(text='2026年9月综合合格率 = 0.9530615',
|
||||
displayText='2026年9月综合合格率95.31%。', sectionIndex=0)
|
||||
for text, expected in (('9月综合合格率95.31%。', True), ('10月综合合格率95.31%。', False),
|
||||
('9月综合合格率95.31 kg。', False), ('9月综合合格率99.31%。', False)):
|
||||
def provider(request, timeout):
|
||||
if json.loads(request.data)['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
|
||||
return response({'blocks': [{'text': text, 'factIds': ['F1']}]})
|
||||
with self.subTest(text=text):
|
||||
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='历史指标', snapshot=candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertEqual(result['evidenceSufficient'], expected, result['compositionDiagnostics'])
|
||||
if expected:
|
||||
self.assertIn('95.31%', result['answer'])
|
||||
self.assertFalse(result['compositionRepairAttempted'])
|
||||
|
||||
def test_many_history_facts_share_compact_frames_and_keep_later_product(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
from test_native_answer import response
|
||||
candidate = snapshot()
|
||||
source = candidate['historySources'][0]['turnId']
|
||||
product = '16.33X15.5X3.75+T'
|
||||
candidate['sections'].append({**copy.deepcopy(candidate['sections'][0]),
|
||||
'title': '重点规格指标', 'filters': {'product': product},
|
||||
'columns': [{'key': 'product', 'label': '产品'}, {'key': 'weight', 'label': '报废重量'}],
|
||||
'rows': [{'product': product, 'weight': 100}]})
|
||||
candidate['facts'] = [{'id': f'history:{index}', 'sourceTurnId': source, 'sectionIndex': 0,
|
||||
'text': f'2026年9月黑片报废重量{index}.00 kg。'} for index in range(120)]
|
||||
candidate['facts'].append({'id': 'history:late-product', 'sourceTurnId': source, 'sectionIndex': 2,
|
||||
'text': '2026年9月'+product+'黑片报废重量100.00 kg。'})
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data)
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
|
||||
payload = json.loads(body['messages'][1]['content'])
|
||||
self.assertEqual(len(payload['scopes']), 2)
|
||||
self.assertGreaterEqual(len(payload['facts']), 60)
|
||||
self.assertFalse(any('rows' in scope or 'historyFactId' in scope for scope in payload['scopes']))
|
||||
late = next(fact for fact in payload['facts'] if fact['evidenceRef'] == 'history:late-product')
|
||||
return response({'blocks': [{'text': product+'黑片报废重量100.00 kg,是应核查的规格线索。', 'factIds': [late['id']]}]})
|
||||
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='汇总重点规格', snapshot=candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
|
||||
self.assertIn(product, result['readingSummary']['lead'])
|
||||
self.assertLess(result['compositionDiagnostics']['factPool']['modelEvidenceChars'], 30000)
|
||||
|
||||
def test_verified_row_values_remain_visible_to_synthesis_and_same_review(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
from test_native_answer import response
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0].update(title='已保存阶段聚合', stage='black',
|
||||
columns=[{'key': 'metric', 'label': '指标'}, {'key': 'value', 'label': '正式值'}],
|
||||
rows=[{'metric': 'rate', 'value': .9531}, {'metric': 'totalWeight', 'value': 100.00}])
|
||||
candidate['facts'][0].update(text='2026年9月黑片合格率 = 0.9531', displayText='2026年9月黑片合格率95.31%。', sectionIndex=0,
|
||||
metricCode='rate', sourceKind='native_saved_aggregate', scope={'period': {'year': 2026, 'month': 9}, 'filters': candidate['sections'][0]['filters'], 'stage': 'black'},
|
||||
values=[{'metricCode': 'currentBlackInput', 'unit': 'source_weight', 'displayValue': '100.00'}])
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data); payload = json.loads(body['messages'][1]['content'])
|
||||
current = next(fact for fact in payload['facts'] if fact['id'] == 'F1')
|
||||
self.assertEqual(current['kind'], 'stage')
|
||||
self.assertEqual(current['values'][0]['metricCode'], 'currentBlackInput')
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
self.assertIn('来源重量100.00', payload['blocks'][0]['text'])
|
||||
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
|
||||
return response({'blocks': [{'text': '本期黑片合格率95.31%,黑片投入来源重量100.00,应结合阶段变化核查。', 'factIds': ['F1']}]})
|
||||
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='汇总历史', snapshot=candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
|
||||
self.assertNotIn('kg', result['answer'])
|
||||
|
||||
def test_global_product_history_keeps_server_row_identifier_and_rejects_dimension_typo(self):
|
||||
from nao_native_answer import synthesize_history_snapshot
|
||||
from test_native_answer import response, fixture_payload
|
||||
product = '16.33X15.5X3.75+T'
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0].update(title='Top 2 product排名', filters={},
|
||||
columns=[{'key': key, 'label': key} for key in ('rank', 'key', 'label', 'weight')],
|
||||
rows=[{'rank': 1, 'key': product, 'label': product, 'weight': 100.00},
|
||||
{'rank': 2, 'key': '18X15X4+T', 'label': '18X15X4+T', 'weight': 90.00}])
|
||||
candidate['facts'][0]['text'] = '2026年9月;全局产品排名:'+product+',黑片报废重量100.00 kg。'
|
||||
for typo in (False, True):
|
||||
with self.subTest(typo=typo):
|
||||
calls = []
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data); calls.append(body)
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
|
||||
payload = fixture_payload(request)
|
||||
current = next(fact for fact in payload['facts'] if fact['id'] == 'F1')
|
||||
scope = next(scope for scope in payload['scopes'] if scope['scopeId'] == current['scopeId'])
|
||||
self.assertTrue(any(item['value'] == product and item['factId'] == 'F1' for item in payload['identifierTokens'].values()))
|
||||
value_id = next(key for key, token in payload['numericTokens'].items()
|
||||
if token['factId'] == 'F1' and token['value'] == '100.00')
|
||||
name = product.replace('3.75', '3.76') if typo else product
|
||||
return response({'blocks': [{'text': '全局产品排名中,'+name+'的黑片报废重量为{{'+value_id+'}} kg,应结合投入规模核查。',
|
||||
'factIds': ['F1'], 'evidenceRefs': ['history-fact:F1'], 'claimType': 'inference'}]})
|
||||
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
||||
question='解读全局产品历史排名', snapshot=candidate, task=report_task(), authority=authority(),
|
||||
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertEqual(result['evidenceSufficient'], not typo, result.get('compositionDiagnostics'))
|
||||
if not typo:
|
||||
self.assertIn(product, result['answer'])
|
||||
self.assertEqual(result['readingSummary']['evidenceRefs'], ['history-fact:F1'])
|
||||
self.assertTrue(result['compositionDiagnostics']['identifierBindings'])
|
||||
else:
|
||||
self.assertTrue(result['compositionRepairAttempted'])
|
||||
self.assertNotIn('3.76', result['answer'])
|
||||
|
||||
def test_autonomous_report_uses_shared_free_synthesis_and_actual_history_references(self):
|
||||
from test_native_answer import response
|
||||
calls = []
|
||||
def provider(request, timeout):
|
||||
body = json.loads(request.data); calls.append(body)
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
|
||||
payload = json.loads(body['messages'][1]['content'])
|
||||
self.assertEqual([fact['evidenceRef'] for fact in payload['facts']], ['history-fact:F1', 'history-fact:F2'])
|
||||
scopes = {scope['scopeId']: scope for scope in payload['scopes']}
|
||||
self.assertTrue(all('period' in scopes[fact['scopeId']] for fact in payload['facts']))
|
||||
self.assertTrue(all('historyFactId' not in scope for scope in scopes.values()))
|
||||
return response({'blocks': [{'text': '历史来源各自适用不同期间和筛选,应分别解读各自的报废重量。',
|
||||
'factIds': ['F1', 'F2'], 'evidenceRefs': ['history-fact:F1', 'history-fact:F2'], 'claimType': 'inference'}]})
|
||||
with tempfile.TemporaryDirectory() as folder, patch('nao_evidence_finalizer._open_model', side_effect=provider), patch(
|
||||
'nao_history_report.select_history_headline', side_effect=AssertionError('must use shared synthesis')):
|
||||
result = execute_history_report(request={'releaseId': RELEASE, 'turnId': TURN, 'question': '汇总历史', '_autonomousAnswer': True},
|
||||
task=report_task(), authority=authority(), actor_root=Path(folder), delegated=FakeDelegated(snapshot()),
|
||||
connection=('https://offline.invalid/v1', 'fixture', 'fixture'), cancel_event=threading.Event(), emit=lambda _: None, timings={})
|
||||
private = json.loads((Path(folder) / 'private/native-composition.json').read_text())
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertIn('各自适用不同期间和筛选', result['answer'])
|
||||
self.assertEqual(result['readingSummary']['evidenceRefs'], ['history-fact:F1', 'history-fact:F2'])
|
||||
self.assertEqual(result['historyHeadlineFactIds'], ['history-fact:F1', 'history-fact:F2'])
|
||||
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(len(private['rawModelResponses']), 2)
|
||||
self.assertNotIn('rawModelResponses', result)
|
||||
|
||||
def test_autonomous_raw_or_member_only_snapshot_has_explicit_limitation(self):
|
||||
with tempfile.TemporaryDirectory() as folder, patch('nao_evidence_finalizer._open_model') as model:
|
||||
result = execute_history_report(request={'releaseId': RELEASE, 'turnId': TURN, 'question': '汇总历史', '_autonomousAnswer': True},
|
||||
task=report_task(), authority=authority(), actor_root=Path(folder), delegated=FakeDelegated(snapshot(facts=[])),
|
||||
connection=('https://offline.invalid/v1', 'fixture', 'fixture'), cancel_event=threading.Event(), emit=lambda _: None, timings={})
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertIn('平台附录展示', result['answer'])
|
||||
self.assertTrue(result['readingSummary']['notices'])
|
||||
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
|
||||
model.assert_not_called()
|
||||
|
||||
def run_report(self, response, model_result=None, *, model_results=None, cancel_after_model=False):
|
||||
delegated = FakeDelegated(response)
|
||||
emitted, model_payloads = [], []
|
||||
cancel_event = threading.Event()
|
||||
queued_results = list(model_results) if model_results is not None else []
|
||||
def fake_model_json(**kwargs):
|
||||
model_payloads.append(kwargs['payload'])
|
||||
if cancel_after_model:
|
||||
cancel_event.set()
|
||||
result = queued_results.pop(0) if queued_results else model_result
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
with tempfile.TemporaryDirectory() as temp, patch('nao_agent_task._model_json', side_effect=fake_model_json):
|
||||
result = execute(
|
||||
request={'releaseId': RELEASE, 'turnId': TURN,
|
||||
'question': '请汇总已确认历史质量报告'},
|
||||
task=report_task(), authority=authority(), actor_root=Path(temp),
|
||||
delegated=delegated, connection=('https://offline.invalid/v1', 'secret', 'model-x'),
|
||||
cancel_event=cancel_event, emit=emitted.append, phase_timings={})
|
||||
return result, delegated, emitted, model_payloads
|
||||
|
||||
def test_imports_server_snapshot_and_models_only_title_and_existing_fact_ids(self):
|
||||
result, delegated, emitted, payloads = self.run_report(
|
||||
snapshot(), {'reportTitle': '2025至2026年质量表现',
|
||||
'headlineFactIds': ['history-fact:F1', 'history-fact:F2']})
|
||||
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['completionMethod'], 'confirmed_history_report')
|
||||
self.assertEqual(result['stopReason'], 'history_report_complete')
|
||||
self.assertTrue(result['titleGeneratedByModel'])
|
||||
self.assertEqual(result['protocolEvidence'], [])
|
||||
self.assertEqual(result['historyHeadlineFactIds'], ['history-fact:F1', 'history-fact:F2'])
|
||||
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
|
||||
self.assertEqual(result['agentTask'], authority()['task'])
|
||||
self.assertNotIn('agentContext', result)
|
||||
self.assertEqual(result['visualState'], 'not_requested')
|
||||
self.assertEqual(result['visuals'], [])
|
||||
self.assertEqual(len(emitted), 2)
|
||||
self.assertEqual(emitted[-1]['type'], 'result')
|
||||
self.assertEqual(len(payloads), 1)
|
||||
serialized = json.dumps(payloads[0], ensure_ascii=False)
|
||||
self.assertNotIn('rows', payloads[0])
|
||||
self.assertNotIn('整体', serialized)
|
||||
self.assertIn('history-fact:F1', serialized)
|
||||
self.assertEqual(len(payloads[0]['historySources']), 2)
|
||||
self.assertIn('columnFilters', serialized)
|
||||
|
||||
def test_incomplete_history_remains_partial_and_includes_unavailable_source_reason(self):
|
||||
result, _, _, _ = self.run_report(
|
||||
snapshot(unavailable_second=True),
|
||||
{'reportTitle': '2026年9月客户质量结果', 'headlineFactIds': ['history-fact:F1']})
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(result['stopReason'], 'history_report_partial')
|
||||
self.assertIn('2025年9月海外组没有可用历史来源。',
|
||||
result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
def test_available_partial_source_is_preserved_but_not_marked_complete(self):
|
||||
partial = snapshot()
|
||||
partial['historySources'][1]['status'] = 'PARTIAL'
|
||||
partial['historyCoverage'] = {
|
||||
'complete': False, 'expectedSources': 2, 'importedSources': 2,
|
||||
'missingEvidence': ['历史范围中有一项来源覆盖不完整。'],
|
||||
}
|
||||
result, _, _, _ = self.run_report(
|
||||
partial, {'reportTitle': '2025至2026年质量表现',
|
||||
'headlineFactIds': ['history-fact:F1']})
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'], False)
|
||||
|
||||
def test_unknown_fact_invalid_title_and_model_failure_fall_back_to_partial(self):
|
||||
invalid_results = [
|
||||
{'reportTitle': '2026年9月客户质量结果', 'headlineFactIds': ['not-a-fact']},
|
||||
{'reportTitle': '<b>质量报告</b>', 'headlineFactIds': ['history-fact:F1']},
|
||||
RuntimeError('model unavailable'),
|
||||
]
|
||||
for invalid in invalid_results:
|
||||
with self.subTest(invalid=repr(invalid)):
|
||||
result, _, _, _ = self.run_report(snapshot(), invalid)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['titleGeneratedByModel'])
|
||||
self.assertEqual(result['reportTitle'], '历史质量分析报告')
|
||||
self.assertEqual(result['historyHeadlineFactIds'], [])
|
||||
|
||||
def test_mixed_customer_and_unfiltered_source_repairs_title_to_multi_scope(self):
|
||||
candidate = snapshot()
|
||||
candidate['sections'][1]['filters'] = {'group': '汽车交通-海外'}
|
||||
bad = {'reportTitle': '2026年9月客户800001黑片亮片报废重量',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
repaired = {'reportTitle': '多范围汇总质量分析',
|
||||
'headlineFactIds': ['history-fact:F1', 'history-fact:F2']}
|
||||
result, delegated, _, payloads = self.run_report(candidate, model_results=[bad, repaired])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['reportTitle'], '多范围汇总质量分析')
|
||||
self.assertEqual(len(payloads), 2)
|
||||
self.assertEqual(payloads[0]['facts'], payloads[1]['facts'])
|
||||
self.assertEqual(payloads[0]['historySources'], payloads[1]['historySources'])
|
||||
self.assertIn('repairInstructions', payloads[1])
|
||||
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
|
||||
self.assertNotIn('rows', payloads[1])
|
||||
|
||||
def test_mixed_product_scope_is_not_named_but_common_product_can_be_named(self):
|
||||
mixed = snapshot()
|
||||
mixed['sections'][0]['filters']['product'] = '磁钢规格-X'
|
||||
bad = {'reportTitle': '2026年磁钢规格-X质量汇总',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
repaired = {'reportTitle': '多范围汇总质量分析',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
result, _, _, payloads = self.run_report(mixed, model_results=[bad, repaired])
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['reportTitle'], '多范围汇总质量分析')
|
||||
self.assertEqual(len(payloads), 2)
|
||||
|
||||
common = snapshot()
|
||||
common['sections'][0]['filters']['product'] = '磁钢规格-X'
|
||||
common['sections'][1]['filters']['product'] = '磁钢规格-X'
|
||||
common_title = {'reportTitle': '2026年磁钢规格-X质量汇总',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
result, _, _, payloads = self.run_report(common, common_title)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['reportTitle'], common_title['reportTitle'])
|
||||
self.assertEqual(len(payloads), 1)
|
||||
|
||||
def test_noncommon_column_filter_values_are_blocked_but_field_ids_are_not(self):
|
||||
candidate = snapshot()
|
||||
facts = candidate['facts']
|
||||
self.assertIsNone(_validate_model_selection(
|
||||
{'reportTitle': '2026年WO-1订单质量汇总', 'headlineFactIds': ['history-fact:F1']},
|
||||
candidate, '整理历史报告', facts))
|
||||
self.assertIsNotNone(_validate_model_selection(
|
||||
{'reportTitle': 'orderNo质量汇总', 'headlineFactIds': ['history-fact:F1']},
|
||||
candidate, '整理历史报告', facts))
|
||||
|
||||
def test_negative_filter_literals_are_never_named_even_when_common(self):
|
||||
for negative_filter in (
|
||||
{'columnFilters': [{'field': 'customerCode', 'operator': 'not_in',
|
||||
'values': ['800001']}]},
|
||||
{'exclusions': {'product': ['磁钢规格-X']}}):
|
||||
candidate = snapshot()
|
||||
candidate['sections'][0]['filters'] = copy.deepcopy(negative_filter)
|
||||
candidate['sections'][1]['filters'] = copy.deepcopy(negative_filter)
|
||||
title = ('2026年客户800001质量汇总' if 'columnFilters' in negative_filter
|
||||
else '2026年磁钢规格-X质量汇总')
|
||||
with self.subTest(negative_filter=negative_filter):
|
||||
self.assertIsNone(_validate_model_selection(
|
||||
{'reportTitle': title, 'headlineFactIds': ['history-fact:F1']},
|
||||
candidate, '整理历史报告', candidate['facts']))
|
||||
|
||||
def test_common_positive_in_filter_literal_may_be_named(self):
|
||||
candidate = snapshot()
|
||||
common_filter = {'columnFilters': [{'field': 'customerCode', 'operator': 'in',
|
||||
'values': ['800001']}]}
|
||||
candidate['sections'][0]['filters'] = copy.deepcopy(common_filter)
|
||||
candidate['sections'][1]['filters'] = copy.deepcopy(common_filter)
|
||||
self.assertIsNotNone(_validate_model_selection(
|
||||
{'reportTitle': '2026年客户800001质量汇总',
|
||||
'headlineFactIds': ['history-fact:F1']},
|
||||
candidate, '整理历史报告', candidate['facts']))
|
||||
|
||||
def test_title_repair_is_limited_to_one_retry(self):
|
||||
candidate = snapshot()
|
||||
candidate['sections'][1]['filters'] = {}
|
||||
bad = {'reportTitle': '2026年9月客户800001质量汇总',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
result, _, _, payloads = self.run_report(candidate, model_results=[bad, bad])
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['titleGeneratedByModel'])
|
||||
self.assertEqual(len(payloads), 2)
|
||||
|
||||
def test_no_facts_skips_model_and_is_partial(self):
|
||||
empty = snapshot(facts=[])
|
||||
result, _, _, payloads = self.run_report(empty, None)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(payloads, [])
|
||||
self.assertIn('没有可引用事实', result['evidenceCompletion']['missingEvidence'][0])
|
||||
|
||||
def test_cancellation_after_snapshot_keeps_partial_and_does_not_accept_late_title(self):
|
||||
result, _, _, payloads = self.run_report(
|
||||
snapshot(), {'reportTitle': '2025至2026年质量表现',
|
||||
'headlineFactIds': ['history-fact:F1']}, cancel_after_model=True)
|
||||
self.assertEqual(len(payloads), 1)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['titleGeneratedByModel'])
|
||||
|
||||
def test_api_failure_is_not_replaced_by_a_local_history_scan(self):
|
||||
delegated = FakeDelegated(RuntimeError('history service unavailable'))
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
with self.assertRaisesRegex(RuntimeError, 'history service unavailable'):
|
||||
execute_history_report(
|
||||
request={'releaseId': RELEASE, 'turnId': TURN, 'question': '报告'},
|
||||
task=report_task(), authority=authority(), actor_root=Path(temp),
|
||||
delegated=delegated, connection=('u', 'k', 'm'),
|
||||
cancel_event=threading.Event(), emit=lambda _: None, timings={})
|
||||
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
|
||||
|
||||
def test_headline_uses_authorized_finalizing_phase_through_real_model_gate(self):
|
||||
class ProviderHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
owner.provider_calls.append((self.headers.get('Authorization'), body))
|
||||
selection = ({'reportTitle': '2026年9月客户800001质量汇总',
|
||||
'headlineFactIds': ['history-fact:F1']}
|
||||
if len(owner.provider_calls) == 1 else
|
||||
{'reportTitle': '多范围汇总质量表现',
|
||||
'headlineFactIds': ['history-fact:F1']})
|
||||
content = json.dumps(selection, ensure_ascii=False)
|
||||
response = json.dumps({'model': 'history-model', 'choices': [{
|
||||
'finish_reason': 'stop', 'message': {'content': content},
|
||||
}]}, ensure_ascii=False).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(response)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
|
||||
owner = self
|
||||
self.provider_calls = []
|
||||
provider = ThreadingHTTPServer(('127.0.0.1', 0), ProviderHandler)
|
||||
provider.daemon_threads = True
|
||||
provider_thread = threading.Thread(target=provider.serve_forever, daemon=True)
|
||||
provider_thread.start()
|
||||
authorize_calls = []
|
||||
gate = ModelGate(
|
||||
upstream_base=f'http://127.0.0.1:{provider.server_port}/v1',
|
||||
upstream_key='fake-provider-key', model='history-model', port=0,
|
||||
client_key='fake-worker-key', authorize=lambda: authorize_calls.append(True),
|
||||
native_turn_marker='quality-turn:current-history-turn')
|
||||
try:
|
||||
selection = select_history_headline(
|
||||
snapshot=snapshot(), question='整理已确认历史报告',
|
||||
connection=(f'http://127.0.0.1:{gate._server.server_port}/v1',
|
||||
gate.client_key, 'history-model'),
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(selection, ('多范围汇总质量表现', ['history-fact:F1']))
|
||||
self.assertGreaterEqual(len(authorize_calls), 2)
|
||||
self.assertEqual(gate.rejected_stale_requests, 0)
|
||||
self.assertEqual(len(self.provider_calls), 2)
|
||||
for authorization, body in self.provider_calls:
|
||||
self.assertEqual(authorization, 'Bearer fake-provider-key')
|
||||
self.assertEqual(body['model'], 'history-model')
|
||||
self.assertEqual(len(body['messages']), 2)
|
||||
self.assertNotIn('sections', body['messages'][1]['content'])
|
||||
self.assertIn('repairInstructions', self.provider_calls[1][1]['messages'][1]['content'])
|
||||
finally:
|
||||
gate.close()
|
||||
provider.shutdown()
|
||||
provider.server_close()
|
||||
provider_thread.join(timeout=1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,192 @@
|
||||
import copy
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_task import (TaskUnavailable, _explicit_history_refresh_intent,
|
||||
normalize_task, plan_task)
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
|
||||
def history_plan(**changes):
|
||||
value = task_value(kind='report', freshQuery=False, historyMode='all', dimensions=[],
|
||||
stages=[], fields=[], comparisonRequired=False, trendRequired=False,
|
||||
verifySource=False, fullDetails=False, filterUpdates={})
|
||||
value.update(changes)
|
||||
return value
|
||||
|
||||
|
||||
class HistoryReportTaskTest(unittest.TestCase):
|
||||
def test_refresh_intent_respects_negation_without_weakening_positive_refresh(self):
|
||||
for question in ('整理历史报告,不重新查询', '不要重新查数据,只整理已完成结果',
|
||||
'无需重查,汇总历史报告', '不需要重新核验数据',
|
||||
'不用最新数据,整理历史', '不需要重新查最新数据,只整理历史'):
|
||||
with self.subTest(question=question):
|
||||
self.assertFalse(_explicit_history_refresh_intent(question))
|
||||
for question in ('不要复用历史,重新查询当前数据', '重新跑数据后更新报告',
|
||||
'最新数据需要重新核验', '必须重查当前数据'):
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(_explicit_history_refresh_intent(question))
|
||||
task = normalize_task(history_plan(), '整理已完成历史结果,不重新查询',
|
||||
{'_protocol': {'version': 1}})
|
||||
self.assertEqual(task['historyMode'], 'all')
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task(history_plan(), '不要复用历史,重新查询数据',
|
||||
{'_protocol': {'version': 1}})
|
||||
|
||||
def test_history_report_does_not_inherit_local_filters_or_period(self):
|
||||
context = {
|
||||
'_protocol': {'version': 1},
|
||||
'filters': {'department': '新能源汽车磁钢',
|
||||
'extraFilters': {'customerCode': ['800001']}},
|
||||
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 9},
|
||||
'comparisonPeriod': {'year': 2025, 'monthStart': 1, 'monthEnd': 9},
|
||||
}
|
||||
task = normalize_task(history_plan(), '重新生成以上报告,正文突出重点', context)
|
||||
self.assertEqual(task['kind'], 'report')
|
||||
self.assertEqual(task['historyMode'], 'all')
|
||||
self.assertFalse(task['freshQuery'])
|
||||
self.assertEqual(task['filters'], {})
|
||||
self.assertIsNone(task['period'])
|
||||
self.assertIsNone(task['comparisonPeriod'])
|
||||
self.assertEqual(task['dimensions'], [])
|
||||
self.assertEqual(task['stages'], [])
|
||||
self.assertEqual(task['fields'], [])
|
||||
self.assertEqual(task['dimensionStages'], {})
|
||||
self.assertNotIn('queryPlan', task)
|
||||
self.assertFalse(task['comparisonRequired'])
|
||||
self.assertFalse(task['trendRequired'])
|
||||
self.assertFalse(task['verifySource'])
|
||||
self.assertFalse(task['fullDetails'])
|
||||
self.assertEqual(task['visualPreference'], 'none')
|
||||
|
||||
def test_new_filter_period_and_refresh_intent_cannot_be_silently_dropped(self):
|
||||
invalid = [
|
||||
(history_plan(filterUpdates={'department': '消费电子'}), '整理历史报告'),
|
||||
(history_plan(period={'year': 2026, 'month': 9}), '整理历史报告'),
|
||||
(history_plan(comparisonPeriod={'year': 2025, 'month': 9}), '整理历史报告'),
|
||||
(history_plan(), '重新查询数据并更新报告'),
|
||||
(history_plan(freshQuery=True), '整理历史报告'),
|
||||
(history_plan(fields=['rate']), '整理历史报告'),
|
||||
(history_plan(historyMode='none'), '整理历史报告'),
|
||||
]
|
||||
for value, question in invalid:
|
||||
with self.subTest(value=value, question=question):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task(value, question, {'_protocol': {'version': 1},
|
||||
'filters': {'columnFilters': [{'field': 'code'}]}})
|
||||
|
||||
column_rule = {'field': 'orderNo', 'stages': ['black'],
|
||||
'operator': 'in', 'values': ['WO-1']}
|
||||
context = {'_protocol': {'version': 1},
|
||||
'_businessCatalog': {'sourceFields': [{
|
||||
'key': 'orderNo', 'label': '委托单号', 'type': 'source_value',
|
||||
'stages': ['black'], 'columns': {'black': '委托单号'},
|
||||
'operators': ['in'],
|
||||
}]}}
|
||||
with self.assertRaises(TaskUnavailable) as error:
|
||||
normalize_task(history_plan(columnFilters=[column_rule]), '整理历史报告', context)
|
||||
self.assertEqual(error.exception.field, 'freshQuery')
|
||||
|
||||
def test_existing_record_and_source_receipts_can_be_preserved_without_new_reads(self):
|
||||
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False}
|
||||
with patch('nao_agent_task._model_json', side_effect=[history_plan(), review]):
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='把已完成的全部逐行记录和原件核验结果整合成历史报告,不重新打开原件',
|
||||
context={'_protocol': {'version': 1}}, cancel_event=threading.Event())
|
||||
self.assertEqual(planned['kind'], 'report')
|
||||
self.assertFalse(planned['fullDetails'])
|
||||
self.assertFalse(planned['verifySource'])
|
||||
|
||||
def test_history_report_does_not_override_an_independent_review_requiring_new_rows(self):
|
||||
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': True, 'requiresTrend': False, 'requiresComparison': False}
|
||||
with patch('nao_agent_task._model_json', side_effect=[history_plan(), review] * 3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='把该规格本月每条原始记录及全部字段列入报告',
|
||||
context={'_protocol': {'version': 1}}, cancel_event=threading.Event())
|
||||
|
||||
def test_legacy_non_protocol_report_is_not_changed(self):
|
||||
value = history_plan()
|
||||
task = normalize_task(value, '整理历史报告', {})
|
||||
self.assertEqual(task['kind'], 'report')
|
||||
self.assertEqual(task['historyMode'], 'all')
|
||||
|
||||
def test_protocol_planner_prompt_exposes_bounded_history_report_contract(self):
|
||||
replies = [history_plan(), {
|
||||
'aligned': True, 'requiresEvidence': True,
|
||||
'requiresSourceVerification': False, 'requiresAllSourceRows': False,
|
||||
'requiresTrend': False, 'requiresComparison': False,
|
||||
}]
|
||||
calls = []
|
||||
def fake_model(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return replies.pop(0)
|
||||
with patch('nao_agent_task._model_json', side_effect=fake_model):
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='整理当前会话已经完成的分析结果',
|
||||
context={'_protocol': {'version': 1}},
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(planned['kind'], 'report')
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertIn('仅在用户要求汇总/重排已完成历史时用kind=report', calls[0]['system'])
|
||||
self.assertIn('historyMode=last或all、freshQuery=false', calls[0]['system'])
|
||||
self.assertIn('previousVerifiedTurns为空或缺失不表示会话没有已分析来源', calls[0]['system'])
|
||||
self.assertIn('用户说不重新查询、不要重查、无需重新查时是复用历史', calls[0]['system'])
|
||||
self.assertEqual(calls[0]['payload']['previousVerifiedTurns'], [])
|
||||
self.assertIn('不能据此判断历史数据不存在', calls[1]['system'])
|
||||
self.assertEqual(calls[1]['payload']['previousVerifiedTurns'], [])
|
||||
capabilities = calls[0]['payload']['platformCapabilities']
|
||||
self.assertIn('confirmed_history_report', capabilities['enabled'])
|
||||
self.assertNotIn('report', capabilities['unsupported'])
|
||||
|
||||
def test_report_review_does_not_rebind_prior_customer_product_or_column_scope(self):
|
||||
column_rule = {'field': 'orderNo', 'stages': ['black'],
|
||||
'operator': 'in', 'values': ['WO-1']}
|
||||
previous = {
|
||||
'kind': 'analysis', 'goal': '客户800001规格SPEC-A的正式质量分析',
|
||||
'question': '查询客户800001规格SPEC-A', 'period': {'year': 2026, 'month': 9},
|
||||
'dimensions': ['product'], 'fields': ['rate'], 'stages': ['black'],
|
||||
'filters': {'extraFilters': {'customerCode': ['800001']}, 'product': 'SPEC-A',
|
||||
'columnFilters': [column_rule]},
|
||||
}
|
||||
context = {
|
||||
'_protocol': {'version': 1}, '_agentTask': previous,
|
||||
'filters': copy.deepcopy(previous['filters']), 'period': previous['period'],
|
||||
'_businessCatalog': {
|
||||
'extraFields': [{'key': 'customerCode', 'label': '客户代码'}],
|
||||
'sourceFields': [{'key': 'orderNo', 'label': '委托单号', 'type': 'source_value',
|
||||
'stages': ['black'], 'columns': {'black': '委托单号'},
|
||||
'operators': ['in']}],
|
||||
'productMatch': {'status': 'unique', 'query': 'SPEC-A',
|
||||
'candidates': [{'name': 'SPEC-A'}]},
|
||||
},
|
||||
}
|
||||
replies = [history_plan(), {
|
||||
'aligned': True, 'requiresEvidence': True,
|
||||
'requiresSourceVerification': False, 'requiresAllSourceRows': False,
|
||||
'requiresTrend': False, 'requiresComparison': False,
|
||||
'customerCodes': ['800001'], 'productScope': 'single',
|
||||
}]
|
||||
calls = []
|
||||
def fake_model(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return replies.pop(0)
|
||||
with patch('nao_agent_task._model_json', side_effect=fake_model):
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='整理当前会话已完成的历史报告,沿用各轮范围',
|
||||
context=context, previous=previous, cancel_event=threading.Event())
|
||||
self.assertEqual(planned['kind'], 'report')
|
||||
self.assertEqual(planned['filters'], {})
|
||||
self.assertIsNone(planned['period'])
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertFalse(calls[1]['payload']['columnFilterReviewRequired'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Strict tool scope and pagination/source completeness contracts."""
|
||||
import copy,json,sys,tempfile,unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
from test_nao_quality_tools import FakeApi
|
||||
from nao_quality_tools import QualityTools,QualityToolError
|
||||
class Api(FakeApi):
|
||||
mutation=None
|
||||
def call(self,path,payload=None,method=None):
|
||||
if path=='/api/analysis/investigate':
|
||||
context={k:payload.get(k) for k in ('releaseId','year','month','quarter','department','group','product','projectGroup','category','stage')};context['includeUnassigned']=payload.get('includeUnassigned',False);context['extraFilters']=payload.get('extraFilters',{})
|
||||
r={'kind':'quality_investigation','action':payload['action'],'context':context,'aggregateComplete':True,'totalRows':205,'categories':[],'lineage':[]}
|
||||
if payload['action']=='source':r['sourceVerification']={'complete':True,'expectedRows':205,'verifiedRows':205,'missingRows':0,'mismatchRows':0}
|
||||
if payload['action']=='rows':r.update(rows=[{'recordId':1}],offset=payload.get('offset',0),returnedRows=1,matchingRows=205,hasMore=True,nextOffset=payload.get('offset',0)+1)
|
||||
if self.mutation:self.mutation(r)
|
||||
return r
|
||||
return super().call(path,payload,method=method)
|
||||
class TestInvestigation(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.api=Api();self.tools=QualityTools(api_factory=lambda:self.api,evidence_dir=Path(self.tmp.name));self.args={'mode':'investigate','releaseId':'release-a','period':{'year':2026,'month':9},'stage':'bright','action':'source','filters':{'department':'工业电机产业部'}}
|
||||
def tearDown(self):self.tmp.cleanup()
|
||||
def test_both_periods_and_evidence_are_preserved(self):
|
||||
args={**self.args,'comparisonPeriod':{'year':2026,'month':8}};r=self.tools.quality_data(args);self.assertEqual(r['current']['totalRows'],205);self.assertEqual(r['comparison']['context']['month'],8);self.assertTrue(r['evidenceRef'].startswith('Q'));self.assertTrue(Path(r['evidencePath']).exists())
|
||||
def test_scope_mismatch_refused(self):
|
||||
self.api.mutation=lambda r:r['context'].update(department='其他');self.assertRaises(QualityToolError,self.tools.quality_data,self.args)
|
||||
def test_false_source_completeness_refused(self):
|
||||
self.api.mutation=lambda r:r['sourceVerification'].update(verifiedRows=5);self.assertRaises(QualityToolError,self.tools.quality_data,self.args)
|
||||
def test_bad_pagination_refused(self):
|
||||
self.api.mutation=lambda r:r.update(nextOffset=0);self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,'action':'rows'})
|
||||
def test_extra_parameters_never_ignored(self):
|
||||
for extra in ({'sql':'select 1'},{'offset':True},{'fields':['x']*33},{'sourceRevisionId':'../x'},{'sort':'rate_drop'}):
|
||||
with self.subTest(extra=extra):self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,**extra})
|
||||
def test_source_only_parameters_rejected_for_other_modes(self):
|
||||
self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,'mode':'query'})
|
||||
class CompactEvidenceTests(unittest.TestCase):
|
||||
def test_compression_preserves_full_evidence_and_proof_counts(self):
|
||||
full={"kind":"quality_investigation","action":"source","current":{"totalRows":205,"fieldDictionary":[{"duplicate":True}],"categories":[{"differenceFormalMinusSubitems":5.7822,"fields":[{"field":"x","coverage":{"rows":205,"numericRows":200,"nullOrBlankRows":5,"keyAbsentRows":0,"invalidNumericRows":0,"negativeRows":0,"numericSum":10}}]}],"sourceVerification":{"complete":True,"allSelectedValuesMatchStored":True,"verifiedRows":205,"files":[{"sha256":"original","fields":[{"field":"x"}]}]}}}
|
||||
view=QualityTools._model_view(full)
|
||||
self.assertIn("fields",full["current"]["categories"][0]);self.assertIn("fieldDictionary",full["current"])
|
||||
self.assertEqual(view["current"]["categories"][0]["fieldCoverage"][0],["x",205,200,5,0,0,0,10])
|
||||
self.assertEqual(view["current"]["sourceVerification"]["verifiedRows"],205)
|
||||
self.assertEqual(view["current"]["sourceVerification"]["files"][0]["verifiedFields"],["x"])
|
||||
self.assertEqual(view["current"]["categories"][0]["differenceFormalMinusSubitems"],5.7822)
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Synthetic regression cases for bounded metric intent and early completion."""
|
||||
import copy
|
||||
import json
|
||||
import io
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
import nao_evidence_finalizer as finalizer
|
||||
from nao_analysis_coverage import formal_metric_coverage
|
||||
from nao_metric_request import (resolve_metric_request, prepare_metric_context, frozen_metric_frame,
|
||||
metric_request, requires_fresh_query, is_history_restatement)
|
||||
|
||||
QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
|
||||
|
||||
|
||||
def metric_evidence():
|
||||
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
|
||||
'filterBinding': {'product': 'SYNTH-16X15+T'},
|
||||
'delta': {'available': True, 'percentagePoints': -2},
|
||||
'stageRateChangesPP': {'black': -1, 'bright': -1}}
|
||||
for side, month in [('current', 9), ('comparison', 8)]:
|
||||
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]},
|
||||
'black': {'rate': .97, 'totalWeight': 1200.5, 'scrapWeight': 36.1},
|
||||
'bright': {'rate': .98, 'totalWeight': 2400.5, 'scrapWeight': 72.1}}
|
||||
return [{'tool': 'quality_data', 'releaseId': 'release-a', 'evidenceRef': 'Q00000001',
|
||||
'mode': 'compare', 'result': result}]
|
||||
|
||||
|
||||
class MetricRequestRegressionTests(unittest.TestCase):
|
||||
def test_common_equivalent_questions_have_the_same_formal_contract(self):
|
||||
questions = [QUESTION,
|
||||
'请对比规格SYNTH-16X15+T在2026年9月和8月的黑片及亮片合格率、投入重量、报废重量。',
|
||||
'帮我查一下SYNTH-16X15+T的2026年9月与8月黑片、亮片合格率、投入量和报废量分别是多少?',
|
||||
'SYNTH-16X15+T,2026年9月与8月,黑片和亮片的合格率、投入重量及报废重量,给我正式数据。']
|
||||
for question in questions:
|
||||
with self.subTest(question=question):
|
||||
contract = formal_metric_coverage(metric_evidence(), question, {'releaseId': 'release-a'})
|
||||
self.assertIsNotNone(contract)
|
||||
self.assertEqual(contract['missingEvidence'], [])
|
||||
self.assertEqual(len(contract['metrics']), 12)
|
||||
|
||||
def _stream_readiness(self, *, mutate=None, question=QUESTION, model='mimo-v2.6-flash',
|
||||
excluded=False, cancelled=False, pending=False, failed_auxiliary=False):
|
||||
clock = [0.0]
|
||||
gates = []
|
||||
readiness = []
|
||||
class Stopper:
|
||||
cause = None
|
||||
stop_succeeded = None
|
||||
def __init__(self, stop, timeout, ready): self.ready = ready; gates.append(self)
|
||||
def set_chat_id(self, value): pass
|
||||
def set_response(self, value): pass
|
||||
def start(self): pass
|
||||
def close(self): pass
|
||||
def request_cancel(self, cause): self.cause = cause
|
||||
class Response:
|
||||
def __enter__(self): return self
|
||||
def __exit__(self, *args): pass
|
||||
def __iter__(self):
|
||||
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q1"}\n'
|
||||
if not pending:
|
||||
yield b'data: {"type":"tool-output-available","toolCallId":"q1","output":{"success":true}}\n'
|
||||
if failed_auxiliary:
|
||||
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q2"}\n'
|
||||
yield b'data: {"type":"tool-output-available","toolCallId":"q2","output":{"success":false,"error":"invalid auxiliary query"}}\n'
|
||||
readiness.append(gates[0].ready())
|
||||
clock[0] = 3.0
|
||||
readiness.append(gates[0].ready())
|
||||
yield b'data: {"type":"text-delta","delta":"done"}\n'
|
||||
yield b'data: {"type":"finish","finishReason":"stop"}\n'
|
||||
yield b'data: [DONE]\n'
|
||||
class Client:
|
||||
def open(self, *args, **kwargs): return Response()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir()
|
||||
evidence = root / 'evidence' / ('nao-quality-' + 'a' * 32 + '.json')
|
||||
document = metric_evidence()[0]
|
||||
if mutate: mutate(document['result'])
|
||||
evidence.write_text(json.dumps(document)); evidence.chmod(0o600)
|
||||
cancel_event = threading.Event()
|
||||
if cancelled: cancel_event.set()
|
||||
with patch.object(runtime, 'NativeStopController', Stopper), patch.object(runtime.time, 'monotonic', side_effect=lambda: clock[0]):
|
||||
runtime._sse_request(client=Client(), base_url='http://127.0.0.1:10000', model=model,
|
||||
question=question, release_id='release-a', context={'releaseId': 'release-a'}, chat_id='native',
|
||||
session_state=None, session_id=str(uuid.uuid4()), actor_root=root, emit=lambda event: None, timeout=60,
|
||||
cancel_event=cancel_event, prior_evidence_files={evidence.name} if excluded else frozenset())
|
||||
return readiness
|
||||
|
||||
def test_pure_metric_stream_is_ready_within_three_seconds_after_evidence(self):
|
||||
self.assertEqual(self._stream_readiness(), [False, True], 'Covered pure metrics still wait for native model completion')
|
||||
|
||||
def test_failed_auxiliary_does_not_block_finalizing_already_complete_evidence(self):
|
||||
self.assertEqual(self._stream_readiness(failed_auxiliary=True), [False, True])
|
||||
self.assertEqual(self._stream_readiness(failed_auxiliary=True, mutate=lambda r:r['current']['black'].pop('totalWeight')), [False,False])
|
||||
|
||||
def test_early_completion_keeps_missing_scope_pending_cancel_and_profile_guards(self):
|
||||
cases = [
|
||||
{'mutate': lambda r: r['current']['black'].pop('totalWeight')},
|
||||
{'mutate': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0)},
|
||||
{'mutate': lambda r: r['filterBinding'].update(group='额外范围')},
|
||||
{'mutate': lambda r: r['comparison']['period'].update(month=7)},
|
||||
{'question': QUESTION + '并分析现场原因。'},
|
||||
{'excluded': True}, {'pending': True}, {'cancelled': True},
|
||||
]
|
||||
for case in cases:
|
||||
with self.subTest(case=case): self.assertEqual(self._stream_readiness(**case), [False, False])
|
||||
self.assertEqual(self._stream_readiness(model='other-model'), [False, True])
|
||||
|
||||
def test_followups_inherit_only_verified_dimensions_in_the_same_frozen_frame(self):
|
||||
base = {'releaseId': 'release-a', 'filters': {'group': '合成分组', 'extraFilters': {'region': '合成地区'}}}
|
||||
bound, first = prepare_metric_context(QUESTION, base)
|
||||
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
|
||||
for question, stage, field in [('那亮片呢?', ['bright'], first['fields']),
|
||||
('再看黑片投入量。', ['black'], ['totalWeight']),
|
||||
('该规格报废量是多少?', first['stages'], ['scrapWeight'])]:
|
||||
with self.subTest(question=question):
|
||||
context, request = prepare_metric_context(question, base, record)
|
||||
self.assertEqual(request['stages'], stage)
|
||||
self.assertEqual(request['fields'], field)
|
||||
self.assertEqual(request['product'], first['product'])
|
||||
self.assertEqual(request['periods'], first['periods'])
|
||||
self.assertEqual(request['filters']['group'], '合成分组')
|
||||
self.assertEqual(metric_request(question, context), request)
|
||||
docs = metric_evidence(); docs[0]['result']['filterBinding'].update(base['filters'])
|
||||
self.assertEqual(formal_metric_coverage(docs, question, context)['missingEvidence'], [])
|
||||
for changed in [{**base, 'releaseId': 'release-b'}, {**base, 'filters': {'group': '另一分组'}},
|
||||
{**base, 'period': {'year': 2025, 'month': 9}}]:
|
||||
with self.subTest(changed=changed):
|
||||
self.assertIsNone(prepare_metric_context('那亮片呢?', changed, record)[1])
|
||||
self.assertIsNone(prepare_metric_context('那亮片呢?', base)[1])
|
||||
|
||||
def test_explicit_new_goal_updates_product_period_stage_and_retains_other_filters(self):
|
||||
base = {'releaseId': 'release-a', 'period': {'year': 2026, 'month': 9},
|
||||
'comparisonPeriod': {'year': 2026, 'month': 8}, 'stage': 'black',
|
||||
'filters': {'product': 'SYNTH-OLD', 'group': '合成分组', 'exclusions': {'product': ['排除品']}}}
|
||||
bound, first = prepare_metric_context(QUESTION, base)
|
||||
self.assertNotIn('stage', bound)
|
||||
self.assertEqual(bound['filters']['product'], 'SYNTH-16X15+T')
|
||||
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
|
||||
context, changed = prepare_metric_context('换成SYNTH-NEW,2026年10月和9月,亮片投入量是多少?', base, record)
|
||||
self.assertEqual(changed['product'], 'SYNTH-NEW')
|
||||
self.assertEqual(context['period'], {'year': 2026, 'month': 10})
|
||||
self.assertEqual(context['comparisonPeriod'], {'year': 2026, 'month': 9})
|
||||
self.assertEqual(context['stage'], 'bright')
|
||||
self.assertEqual(context['filters']['exclusions'], {'product': ['排除品']})
|
||||
|
||||
def test_unknown_language_and_ambiguous_dimensions_remain_outside_the_contract(self):
|
||||
for question in [QUESTION + '仅统计合成分组。', QUESTION + '并提供全部原始明细。',
|
||||
QUESTION.replace('2026年', ''), QUESTION.replace('8月', '8月和7月'),
|
||||
QUESTION.replace('8月', '9月'), QUESTION + '并提供生产原因。',
|
||||
QUESTION.replace('SYNTH-16X15+T', 'SYNTH-16X15+T和SYNTH-OTHER'),
|
||||
QUESTION + '去年相比如何?']:
|
||||
with self.subTest(question=question): self.assertIsNone(resolve_metric_request(question))
|
||||
|
||||
def test_only_requested_metrics_gate_coverage_and_appear_in_the_answer(self):
|
||||
question = '请对比规格SYNTH-16X15+T在2026年9月和8月的亮片投入量。'
|
||||
docs = metric_evidence()
|
||||
for side in ('current', 'comparison'):
|
||||
docs[0]['result'][side]['black'] = {}
|
||||
docs[0]['result'][side]['coverage']['sourceMonths'][0]['blackRows'] = 0
|
||||
docs[0]['result'][side]['compositeRate'] = None
|
||||
docs[0]['result'][side]['bright']['rate'] = None
|
||||
self.assertTrue(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
|
||||
reply = {'evidenceSufficient': True, 'missingEvidence': []}
|
||||
result = finalizer.synthesize(base_url='https://synthetic.invalid/v1', api_key='synthetic',
|
||||
model='mimo-v2.6-flash', question=question, evidence=docs, context={'releaseId': 'release-a'},
|
||||
cancel_event=threading.Event(), opener=lambda *_a, **_kw: io.BytesIO(json.dumps({
|
||||
'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(reply)}}]}).encode()))
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
self.assertIn('亮片投入重量', result['answer'])
|
||||
self.assertNotIn('综合合格率', result['answer'])
|
||||
self.assertNotIn('黑片', result['answer'])
|
||||
self.assertEqual(result['modelMissingEvidence'], [])
|
||||
docs[0]['result']['comparison']['bright']['totalWeight'] = None
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
|
||||
|
||||
def test_fresh_query_detection_and_restatement_are_distinct(self):
|
||||
for question in ['重新查询', '请重新分析刚才的数据', '再查一次', '再次查询亮片投入', '刷新数据']:
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(requires_fresh_query(question))
|
||||
self.assertFalse(is_history_restatement(question))
|
||||
for question in ['把刚才的结果再说一遍', '请复述上一轮结论', '总结刚才的结果']:
|
||||
with self.subTest(question=question):
|
||||
self.assertTrue(is_history_restatement(question))
|
||||
self.assertFalse(requires_fresh_query(question))
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Loopback fake-provider tests: no configured model, credential file or real network service."""
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_model_gate import ModelGate
|
||||
|
||||
|
||||
class HeldOpenProviderResponse:
|
||||
"""A provider that sends a terminal SSE event but keeps its transport alive."""
|
||||
status = 200
|
||||
headers = {'Content-Type': 'text/event-stream'}
|
||||
|
||||
def __init__(self, chunks):
|
||||
self.chunks = list(chunks)
|
||||
self.read_count = 0
|
||||
self.terminal_delivered = threading.Event()
|
||||
self.read_after_terminal = threading.Event()
|
||||
self.release = threading.Event()
|
||||
self.closed = threading.Event()
|
||||
|
||||
def __enter__(self): return self
|
||||
|
||||
def __exit__(self, *_): self.closed.set()
|
||||
|
||||
def read1(self, _size):
|
||||
self.read_count += 1
|
||||
if self.chunks:
|
||||
chunk = self.chunks.pop(0)
|
||||
if not self.chunks:
|
||||
self.terminal_delivered.set()
|
||||
return chunk
|
||||
self.read_after_terminal.set()
|
||||
# Heartbeats keep the connection readable without adding completion data.
|
||||
return b'' if self.release.wait(.02) else b': heartbeat\n\n'
|
||||
|
||||
|
||||
class ModelGateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.allowed = True
|
||||
self.calls = []
|
||||
self.provider_status = 200
|
||||
owner = self
|
||||
class Provider(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_): pass
|
||||
def do_POST(self):
|
||||
owner.calls.append(json.loads(self.rfile.read(int(self.headers['Content-Length']))))
|
||||
data = json.dumps({'choices': [{'message': {'content': 'fake model'}}]} if owner.provider_status == 200
|
||||
else {'error': 'fake-sensitive-provider-details'}).encode()
|
||||
self.send_response(owner.provider_status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.end_headers(); self.wfile.write(data)
|
||||
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
|
||||
self.provider.daemon_threads = True
|
||||
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True); self.thread.start()
|
||||
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1', upstream_key='dummy-provider-key',
|
||||
model='fake-model', port=0, client_key='dummy-native-key', authorize=self.authorize)
|
||||
# Port zero is only used by this isolated test server.
|
||||
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
|
||||
|
||||
def tearDown(self):
|
||||
self.gate.close(); self.provider.shutdown(); self.provider.server_close(); self.thread.join(timeout=1)
|
||||
|
||||
def authorize(self):
|
||||
if not self.allowed: raise PermissionError('revoked or expired delegation')
|
||||
|
||||
def request(self, *, key='dummy-native-key', model='fake-model', extra=None, phase=None):
|
||||
headers={'Content-Type':'application/json','Authorization':'Bearer '+key}
|
||||
if phase:headers['X-Quality-Analysis-Phase']=phase
|
||||
request = Request(self.base+'/chat/completions', data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': 'offline fixture'}],**(extra or {})}).encode(),
|
||||
headers=headers)
|
||||
try:
|
||||
with urlopen(request, timeout=3) as response: return response.status, response.read().decode()
|
||||
except HTTPError as error: return error.code, error.read().decode()
|
||||
|
||||
def test_revocation_blocks_native_and_final_synthesis_requests_before_fake_provider(self):
|
||||
self.assertEqual(self.request()[0], 200)
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.allowed = False
|
||||
self.assertEqual(self.request()[0], 403) # next native provider request
|
||||
self.assertEqual(self.request()[0], 403) # final synthesis uses this same connection
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
|
||||
def test_initial_denial_or_wrong_native_credential_never_reaches_provider(self):
|
||||
self.allowed = False
|
||||
self.assertEqual(self.request()[0], 403)
|
||||
self.allowed = True
|
||||
self.assertEqual(self.request(key='wrong-key')[0], 403)
|
||||
self.assertEqual(self.request(model='different-model')[0], 400)
|
||||
self.assertEqual(len(self.calls), 0)
|
||||
|
||||
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
|
||||
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
|
||||
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
|
||||
self.assertEqual(self.request(extra={'tools':[data_tool,chart_tool],
|
||||
'tool_choice':{'type':'function','function':{'name':'display_chart'}}})[0],200)
|
||||
self.assertEqual(self.calls[-1]['tools'],[data_tool])
|
||||
self.assertNotIn('tool_choice',self.calls[-1])
|
||||
self.assertEqual(self.request(extra={'tools':[chart_tool],'tool_choice':'required'})[0],200)
|
||||
self.assertNotIn('tools',self.calls[-1])
|
||||
self.assertNotIn('tool_choice',self.calls[-1])
|
||||
|
||||
def test_provider_errors_are_bounded_public_failures(self):
|
||||
self.provider_status = 503
|
||||
code, body = self.request()
|
||||
self.assertEqual(code, 503)
|
||||
self.assertNotIn('fake-sensitive-provider-details', body)
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
|
||||
def test_previous_turn_background_call_cannot_enter_current_thinking_or_provider(self):
|
||||
marker='quality-turn:current-request'
|
||||
self.gate._native_turn_marker=marker
|
||||
self.gate._thinking={'enabled':True,'effort':'medium','protocol':'thinking'}
|
||||
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':'quality-turn:old-request'}]})[0],403)
|
||||
self.assertEqual(self.request(extra={'messages':[{'role':'system','content':marker},{'role':'user','content':'old request'}]})[0],403)
|
||||
self.assertEqual(self.calls,[])
|
||||
self.assertEqual(self.gate._request_number,0)
|
||||
self.assertEqual(self.gate.request_audit,[])
|
||||
self.assertEqual(self.gate.rejected_stale_requests,2)
|
||||
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':[{'type':'text','text':marker+' current task'}]}]})[0],200)
|
||||
self.assertEqual(self.request(phase='planning')[0],200)
|
||||
self.assertEqual(len(self.calls),2)
|
||||
self.assertEqual(len(self.gate.request_audit),2)
|
||||
|
||||
def test_saved_enabled_effort_applies_to_every_request_despite_helper_defaults(self):
|
||||
self.gate._thinking={'enabled':True,'effort':'high','protocol':'thinking'}
|
||||
self.assertEqual(self.request(extra={'thinking':{'type':'disabled'},'reasoning_effort':'low','temperature':.2,'max_completion_tokens':4096})[0],200)
|
||||
body=self.calls[-1]
|
||||
self.assertEqual(body['thinking'],{'type':'enabled'})
|
||||
self.assertEqual(body['reasoning_effort'],'high')
|
||||
self.assertNotIn('temperature',body)
|
||||
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
||||
self.assertEqual(body['max_completion_tokens'],MAX_OUTPUT_TOKENS)
|
||||
self.assertNotIn('offline fixture',json.dumps(self.gate.request_audit))
|
||||
|
||||
def test_qwen38_native_effort_uses_template_low_medium_and_xhigh(self):
|
||||
self.gate._model='qwen3.8-27b-local'
|
||||
for selected,native in [('low','low'),('medium','medium'),('high','xhigh')]:
|
||||
self.gate._thinking={'enabled':True,'effort':selected,'protocol':'chat_template'}
|
||||
self.assertEqual(self.request(model='qwen3.8-27b-local',extra={'reasoning_effort':'high','chat_template_kwargs':{'reasoning_effort':'low','preserve_thinking':True}})[0],200)
|
||||
body=self.calls[-1]
|
||||
self.assertEqual(body['chat_template_kwargs'],{'reasoning_effort':native,'enable_thinking':True,'preserve_thinking':True})
|
||||
self.assertNotIn('reasoning_effort',body)
|
||||
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
|
||||
|
||||
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
|
||||
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
|
||||
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
|
||||
body=self.calls[-1]
|
||||
self.assertEqual(body['chat_template_kwargs']['enable_thinking'],False)
|
||||
self.assertNotIn('thinking',body)
|
||||
self.assertNotIn('reasoning_effort',body)
|
||||
|
||||
def test_explicit_sse_done_finishes_without_eof_for_bridge_and_passthrough(self):
|
||||
from unittest.mock import Mock
|
||||
rows = [
|
||||
{'id': 'fake-completion', 'model': 'fake-model', 'choices': [
|
||||
{'index': 0, 'delta': {'reasoning_content': '先核对。', 'content': '{"ok":'}}]},
|
||||
{'choices': [{'index': 0, 'delta': {'content': 'true}'}, 'finish_reason': 'stop'}]},
|
||||
# The usage-only tail must survive even though the choice already finished.
|
||||
{'choices': [], 'usage': {'prompt_tokens': 7, 'completion_tokens': 12}},
|
||||
]
|
||||
wire = [('data: ' + json.dumps(row, ensure_ascii=False) + '\n\n').encode() for row in rows]
|
||||
chunks = [*wire, b'data: [DO', b'NE]\n\n']
|
||||
for stream, observe in ((False, True), (True, True), (True, False)):
|
||||
with self.subTest(stream=stream, observe=observe):
|
||||
events = []
|
||||
self.gate._on_thinking = events.append if observe else None
|
||||
upstream = HeldOpenProviderResponse(chunks)
|
||||
self.gate._opener = Mock()
|
||||
self.gate._opener.open.return_value = upstream
|
||||
response = []
|
||||
returned = threading.Event()
|
||||
|
||||
def request():
|
||||
try:
|
||||
response.append(self.request(extra={'stream': stream}))
|
||||
except Exception as error:
|
||||
response.append(error)
|
||||
finally:
|
||||
returned.set()
|
||||
|
||||
worker = threading.Thread(target=request, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
self.assertTrue(upstream.terminal_delivered.wait(1), 'fake provider did not deliver DONE')
|
||||
completed_before_eof = returned.wait(1)
|
||||
finally:
|
||||
upstream.release.set()
|
||||
worker.join(timeout=2)
|
||||
self.assertTrue(completed_before_eof, 'ModelGate still waits for provider EOF after explicit DONE')
|
||||
self.assertFalse(upstream.read_after_terminal.is_set(), 'ModelGate read beyond explicit DONE')
|
||||
self.assertTrue(upstream.closed.is_set())
|
||||
self.assertEqual(response[0][0], 200)
|
||||
if stream:
|
||||
self.assertEqual(response[0][1].encode(), b''.join(chunks))
|
||||
else:
|
||||
completion = json.loads(response[0][1])
|
||||
self.assertEqual(completion['choices'][0]['message']['content'], '{"ok":true}')
|
||||
self.assertEqual(completion['choices'][0]['finish_reason'], 'stop')
|
||||
self.assertEqual(completion['usage'], {'prompt_tokens': 7, 'completion_tokens': 12})
|
||||
self.assertNotIn('reasoning_content', completion['choices'][0]['message'])
|
||||
if observe:
|
||||
self.assertEqual(events[-1]['event'], 'end')
|
||||
self.assertEqual(events[-1]['finishReason'], 'stop')
|
||||
self.assertFalse(events[-1]['interrupted'])
|
||||
|
||||
def test_explicit_done_without_finish_reason_is_not_a_successful_completion(self):
|
||||
from unittest.mock import Mock
|
||||
upstream = HeldOpenProviderResponse([
|
||||
b'data: {"choices":[{"index":0,"delta":{"content":"incomplete"}}]}\n\n',
|
||||
b'data: [DONE]\n\n',
|
||||
])
|
||||
self.gate._opener = Mock()
|
||||
self.gate._opener.open.return_value = upstream
|
||||
events = []
|
||||
self.gate._on_thinking = events.append
|
||||
response = []
|
||||
returned = threading.Event()
|
||||
|
||||
def request():
|
||||
try:
|
||||
response.append(self.request())
|
||||
finally:
|
||||
returned.set()
|
||||
|
||||
worker = threading.Thread(target=request, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
self.assertTrue(upstream.terminal_delivered.wait(1))
|
||||
completed_before_eof = returned.wait(1)
|
||||
finally:
|
||||
upstream.release.set()
|
||||
worker.join(timeout=2)
|
||||
self.assertTrue(completed_before_eof, 'invalid DONE must fail without waiting for EOF')
|
||||
self.assertEqual(response[0][0], 502)
|
||||
self.assertTrue(events[-1]['interrupted'])
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,25 @@
|
||||
import copy,json,sys,unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_quality_tools import QualityTools
|
||||
|
||||
class FineDefectProjectionTest(unittest.TestCase):
|
||||
def test_repeated_field_coverage_is_compact_without_changing_evidence_or_top_weights(self):
|
||||
fields=[{'key':f'镀废缺陷字段{i}重量','label':f'镀废缺陷字段{i}重量','category':'镀废','sourceField':f'镀废缺陷字段{i}重量'} for i in range(35)]
|
||||
components=[{'key':f['key'],'label':f['label'],'weight':None if i==2 else i/10,'presentRows':0 if i==2 else 72,'missingRows':72 if i==2 else 0,'invalidRows':0,'negativeRows':0,'sourceRows':72} for i,f in enumerate(fields)]
|
||||
category={'category':'镀废','sourceRows':72,'components':components,'subitemWeightSum':58.3,'diff':None,'reconciliationStatus':'INCOMPLETE','reconciliationReasons':['存在缺失字段'],'formalComponentCoverage':{'presentRows':72,'missingRows':0}}
|
||||
original={'kind':'quality_fine_defects','releaseId':'r1','stage':'bright','filters':{'group':'消费电子-美洲','product':'D34.5X5.5+C+Z'},'period':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},'rows':[{'key':fields[1]['key'],'weight':.1,'missingRows':0}],'pairedRows':[{'key':fields[1]['key'],'current':{'weight':.1},'comparison':{'weight':None},'deltaWeight':None,'deltaAvailable':False}],'sourceFields':fields,'comparisonSourceFields':fields,'categoryReconciliation':[category],'comparisonCategoryReconciliation':[category],'coverage':{'available':True,'categories':[category]},'comparisonCoverage':{'available':True,'categories':[category]},'rankingComplete':True,'pairedRankingComplete':True,'evidenceRef':'Q12345678'}
|
||||
untouched=copy.deepcopy(original);view=QualityTools._model_view(original)
|
||||
self.assertLess(len(json.dumps(view,ensure_ascii=False)),len(json.dumps(original,ensure_ascii=False))*.6)
|
||||
self.assertEqual(original,untouched)
|
||||
self.assertEqual(view['coverage'],{'available':True,'categoriesRef':'categoryReconciliation'})
|
||||
self.assertEqual(view['comparisonCoverage'],{'available':True,'categoriesRef':'comparisonCategoryReconciliation'})
|
||||
for key in ('rows','pairedRows','filters','period','comparisonPeriod','rankingComplete','pairedRankingComplete','evidenceRef'):self.assertEqual(view[key],original[key])
|
||||
for name in ('categoryReconciliation','comparisonCategoryReconciliation'):
|
||||
row=view[name][0];columns=row['componentColumns'];restored=[dict(zip(columns,values)) for values in row['componentValues']]
|
||||
self.assertEqual(restored,components)
|
||||
for key in category:
|
||||
if key!='components':self.assertEqual(row[key],category[key])
|
||||
self.assertEqual([f['key'] for f in view['sourceFields']],[f['key'] for f in fields])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,109 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_model_settings as settings
|
||||
|
||||
|
||||
class ModelSettingsTest(unittest.TestCase):
|
||||
def test_mimo26_legacy_effort_preserves_switch_but_reports_automatic_strength(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'config.json'
|
||||
with patch.object(settings,'SETTINGS_FILE',p):
|
||||
for enabled in (True,False):
|
||||
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='mimo-v2.6-flash',thinkingEnabled=enabled,thinkingEffort='high')))
|
||||
self.assertEqual(settings.model_configuration()[3],{'enabled':enabled,'effort':'auto','protocol':'thinking'})
|
||||
|
||||
def test_thinking_controls_are_loaded_atomically_with_profile_credentials(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'config.json'
|
||||
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='qwen-local',thinkingEnabled=False,thinkingEffort='high')))
|
||||
with patch.object(settings,'SETTINGS_FILE',p):
|
||||
result=settings.model_configuration()
|
||||
self.assertEqual(result[2],'qwen-local')
|
||||
self.assertEqual(result[3],{'enabled':False,'effort':'high','protocol':'chat_template'})
|
||||
for value in ('extreme',False):
|
||||
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='mimo-model',thinkingEnabled=True,thinkingEffort=value)))
|
||||
with self.assertRaises(ValueError):settings.model_configuration()
|
||||
|
||||
def test_ui_settings_override_legacy_and_refresh_each_read(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p = Path(tmp) / 'config.json'
|
||||
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
|
||||
p.write_text(json.dumps(dict(baseUrl='http://localhost:8000/v1', apiKey='private-one', model='local-model')))
|
||||
self.assertEqual(settings.model_settings(), ('http://localhost:8000/v1', 'private-one', 'local-model'))
|
||||
p.write_text(json.dumps(dict(baseUrl='http://localhost:8000/v1', apiKey='private-two', model='other-model')))
|
||||
self.assertEqual(settings.model_settings()[1:], ('private-two', 'other-model'))
|
||||
legacy.assert_not_called()
|
||||
|
||||
def test_missing_ui_config_bootstraps_selected_legacy(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / 'apikey.txt').write_text('https://first.example\nfirst-key\nfirst-model\nhttps://second.example/v1/chat/completions\nBearer selected-key\nsecond-model\n')
|
||||
selection = root / '.runtime/dbgpt-trial/selected-profile.json'
|
||||
selection.parent.mkdir(parents=True)
|
||||
selection.write_text('{"profile":2}')
|
||||
# A clean checkout has no dbgpt_trial module. Bootstrap must still work.
|
||||
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True), patch.dict(sys.modules, {'dbgpt_trial': None}):
|
||||
self.assertEqual(settings.model_settings(), ('https://second.example/v1', 'selected-key', 'second-model'))
|
||||
|
||||
def test_invalid_saved_config_does_not_silently_use_legacy(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p = Path(tmp) / 'config.json'
|
||||
p.write_text(json.dumps(dict(baseUrl='http://user:secret@localhost/v1', apiKey='key', model='model')))
|
||||
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
|
||||
with self.assertRaises(ValueError): settings.model_settings()
|
||||
legacy.assert_not_called()
|
||||
|
||||
def test_legacy_utf16_and_explicit_config_path(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
p = root / 'selected.txt'
|
||||
p.write_text('https://model.example/v1\nBEARER selected-key\nprovider/model:latest\n', encoding='utf-16')
|
||||
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {'QUALITY_MODEL_CONFIG': str(p)}):
|
||||
self.assertEqual(settings.model_settings(), ('https://model.example/v1', 'selected-key', 'provider/model:latest'))
|
||||
|
||||
def test_legacy_rejects_invalid_profile_without_exposing_content(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / 'apikey.txt').write_text('https://model.example\nprivate-secret\nmodel\n')
|
||||
selection = root / '.runtime/dbgpt-trial/selected-profile.json'
|
||||
selection.parent.mkdir(parents=True)
|
||||
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True):
|
||||
for profile in (0, 2, True, 1.5, '1'):
|
||||
with self.subTest(profile=profile):
|
||||
selection.write_text(json.dumps({'profile': profile}))
|
||||
with self.assertRaises(ValueError) as error:
|
||||
settings.model_settings()
|
||||
self.assertNotIn('private-secret', str(error.exception))
|
||||
|
||||
def test_legacy_validation_and_bounded_read(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
p = root / 'apikey.txt'
|
||||
invalid = [
|
||||
'https://user:password@model.example\nkey\nmodel',
|
||||
'https://model.example#fragment\nkey\nmodel',
|
||||
'https://model.example\nkey with spaces\nmodel',
|
||||
'https://model.example\nkey\ninvalid model',
|
||||
'x' * 16385,
|
||||
]
|
||||
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True):
|
||||
for content in invalid:
|
||||
with self.subTest(content_type=invalid.index(content)):
|
||||
p.write_text(content)
|
||||
with self.assertRaises(ValueError): settings.model_settings()
|
||||
|
||||
def test_dangling_saved_config_symlink_does_not_fall_back(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
p = root / 'config.json'
|
||||
p.symlink_to(root / 'missing.json')
|
||||
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
|
||||
with self.assertRaises(ValueError): settings.model_settings()
|
||||
legacy.assert_not_called()
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,19 @@
|
||||
import io,json,sys,threading,unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_agent_task import _model_json
|
||||
|
||||
class ThinkingTimeoutTest(unittest.TestCase):
|
||||
def test_valid_slow_local_thinking_plan_is_not_abandoned_at_thirty_seconds(self):
|
||||
calls=[]
|
||||
def provider(request,timeout):
|
||||
calls.append(timeout)
|
||||
if timeout<50:raise TimeoutError('fixture: correct answer arrives at 49 seconds')
|
||||
return io.BytesIO(json.dumps({'model':'qwen3.8-27b-local','choices':[{'finish_reason':'stop','message':{'content':'{"goal":"compare formal quality"}'}}]}).encode())
|
||||
result=_model_json(base_url='http://127.0.0.1:1/v1',api_key='fixture',model='qwen3.8-27b-local',system='fixture',payload={},cancel_event=threading.Event(),opener=provider)
|
||||
self.assertEqual(result,{'goal':'compare formal quality'})
|
||||
self.assertEqual(len(calls),1)
|
||||
from nao_runtime_policy import REQUEST_TIMEOUT_SECONDS
|
||||
self.assertLessEqual(calls[0],REQUEST_TIMEOUT_SECONDS)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,188 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_product_identification import identify_products, resolve_clarification_choice, authorized_extra_fields
|
||||
|
||||
|
||||
class ProductIdentificationTests(unittest.TestCase):
|
||||
def test_explicit_negative_dimension_does_not_return_as_positive_clarification_option(self):
|
||||
options={'products':['18X15X4+T','18X15X4+Z','18X15X4.1+Z']}
|
||||
result=identify_products('18X15X4合格率,不要包含18X15X4.1',options,respect_exclusions=True)
|
||||
self.assertEqual(result['productMatch']['status'],'ambiguous')
|
||||
self.assertEqual([x['name'] for x in result['productMatch']['candidates']],['18X15X4+T','18X15X4+Z'])
|
||||
self.assertEqual(result['explicitlyExcludedProducts'],['18X15X4.1+Z'])
|
||||
cancelled=identify_products('18X15X4合格率,取消排除18X15X4.1',options,respect_exclusions=True)
|
||||
self.assertEqual(cancelled['explicitlyExcludedProducts'],[])
|
||||
negative_only=identify_products('看公司整体合格率,排除18X15X4.1+Z',options,respect_exclusions=True)
|
||||
self.assertEqual(negative_only['productMatch']['status'],'none')
|
||||
self.assertEqual(negative_only['productMatch']['candidates'],[])
|
||||
self.assertEqual(negative_only['explicitlyExcludedProducts'],['18X15X4.1+Z'])
|
||||
named=identify_products('查看CUSTOM_SPEC',{'products':['CUSTOM_SPEC']},respect_exclusions=True)
|
||||
self.assertEqual(named['productMatch']['status'],'exact')
|
||||
|
||||
def test_round_two_axis_geometry_preserves_diameter_and_exact_suffix(self):
|
||||
options={'products':['D37.2X6.5+C','D37.2X6.51+C','37.2X6.5X2+C','18X15X4+T']}
|
||||
for text in ('D37.2x6.5','Φ37.20×6.50','Ø37.2*6.5'):
|
||||
match=identify_products(text,options)['productMatch']
|
||||
self.assertEqual(match['status'],'unique');self.assertEqual([item['name'] for item in match['candidates']],['D37.2X6.5+C'])
|
||||
self.assertEqual(identify_products('D37.2x6.5+Q',options)['productMatch']['status'],'none')
|
||||
self.assertEqual(identify_products('D18x15x4',options)['productMatch']['status'],'none')
|
||||
def setUp(self):
|
||||
self.options = {
|
||||
'products': ['18X15X4+T', '18X15X4+Z', '18X15X4.1+Z', '28X20X3'],
|
||||
'organizationPaths': [
|
||||
{'product': '18X15X4+T', 'department': '汽车', 'group': '国内'},
|
||||
{'product': '18X15X4+Z', 'department': '汽车', 'group': '海外'},
|
||||
{'product': '18X15X4.1+Z', 'department': '工业', 'group': '美洲'},
|
||||
],
|
||||
}
|
||||
|
||||
def test_dimension_shorthand_produces_candidates_without_wrong_decimal(self):
|
||||
result = identify_products('今年18x15x4合格率变化趋势', self.options)
|
||||
self.assertEqual(result['mentionedProducts'], [])
|
||||
match = result['productMatch']
|
||||
self.assertEqual(match['status'], 'ambiguous')
|
||||
self.assertEqual([item['name'] for item in match['candidates']], ['18X15X4+T', '18X15X4+Z'])
|
||||
self.assertEqual(match['candidateCount'], 2)
|
||||
self.assertNotIn('18X15X4.1+Z', [item['name'] for item in match['candidates']])
|
||||
|
||||
def test_full_name_wins_over_other_suffix_and_normalizes_multiplication(self):
|
||||
result = identify_products('今年18×15×4+t的趋势', self.options)
|
||||
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
|
||||
self.assertEqual(result['productMatch']['status'], 'exact')
|
||||
self.assertEqual([item['name'] for item in result['productMatch']['candidates']], ['18X15X4+T'])
|
||||
|
||||
def test_unique_and_decimal_equivalent_dimensions(self):
|
||||
match = identify_products('18.0*15*4.10的变化', self.options)['productMatch']
|
||||
self.assertEqual(match['status'], 'unique')
|
||||
self.assertEqual(match['candidates'][0]['name'], '18X15X4.1+Z')
|
||||
self.assertEqual(match['candidates'][0]['departments'], ['工业'])
|
||||
|
||||
def test_unknown_dimension_does_not_guess(self):
|
||||
match = identify_products('20x11x7的趋势', self.options)['productMatch']
|
||||
self.assertEqual(match['status'], 'none')
|
||||
self.assertEqual(match['candidates'], [])
|
||||
self.assertEqual(match['mentions'][0]['query'], '20x11x7')
|
||||
|
||||
def test_only_authorized_options_can_be_candidates(self):
|
||||
scoped = {'products': ['18X15X4+T'], 'organizationPaths': self.options['organizationPaths']}
|
||||
match = identify_products('18x15x4', scoped)['productMatch']
|
||||
self.assertEqual(match['status'], 'unique')
|
||||
self.assertEqual([item['name'] for item in match['candidates']], ['18X15X4+T'])
|
||||
self.assertEqual(match['candidates'][0]['groups'], ['国内'])
|
||||
|
||||
def test_multiple_specifications_stay_separate(self):
|
||||
match = identify_products('比较18×15×4+T和28x20x3', self.options)['productMatch']
|
||||
self.assertEqual(match['status'], 'multiple')
|
||||
self.assertEqual([m['status'] for m in match['mentions']], ['exact', 'exact'])
|
||||
self.assertEqual([[c['name'] for c in m['candidates']] for m in match['mentions']],
|
||||
[['18X15X4+T'], ['28X20X3']])
|
||||
|
||||
def test_candidate_cap_does_not_misrepresent_ambiguity(self):
|
||||
options = {'products': [f'18X15X4+S{i}' for i in range(30)]}
|
||||
match = identify_products('18x15x4', options)['productMatch']
|
||||
self.assertEqual(match['status'], 'ambiguous')
|
||||
self.assertEqual(match['candidateCount'], 30)
|
||||
self.assertEqual(len(match['candidates']), 20)
|
||||
self.assertTrue(match['truncated'])
|
||||
current=identify_products('18x15x4',options,respect_exclusions=True)['productMatch']
|
||||
self.assertEqual(current['candidateCount'],30)
|
||||
self.assertEqual(len(current['candidates']),20)
|
||||
self.assertTrue(current['truncated'])
|
||||
|
||||
def test_non_product_question_and_empty_catalog_are_safe(self):
|
||||
self.assertEqual(identify_products('你好', self.options)['productMatch']['status'], 'none')
|
||||
self.assertEqual(identify_products('18x15x4', {'products': None})['productMatch']['status'], 'none')
|
||||
|
||||
def pending(self):
|
||||
return {'needsClarification': True, 'productResolution': {
|
||||
'originalQuestion': '今年18x15x4合格率变化趋势',
|
||||
'productMatch': identify_products('今年18x15x4合格率变化趋势', self.options)['productMatch'],
|
||||
}}
|
||||
|
||||
def test_followup_suffix_is_resolved_only_from_pending_candidates(self):
|
||||
for reply, expected in (('选 +T', '18X15X4+T'), ('选择+z', '18X15X4+Z')):
|
||||
with self.subTest(reply=reply):
|
||||
result = resolve_clarification_choice(reply, self.options, self.pending())
|
||||
self.assertEqual(result['mentionedProducts'], [expected])
|
||||
self.assertEqual(result['productMatch']['status'], 'exact')
|
||||
self.assertEqual(result['productMatch']['previousQuestion'], '今年18x15x4合格率变化趋势')
|
||||
self.assertTrue(result['productMatch']['selectedFromClarification'])
|
||||
|
||||
def test_selection_never_searches_suffix_globally_or_uses_revoked_access(self):
|
||||
options = {'products': ['18X15X4+Z', '28X20X3+T']}
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', options, self.pending()))
|
||||
|
||||
def test_failed_selected_question_retry_recovers_only_same_authorized_choice(self):
|
||||
previous = self.pending()
|
||||
previous['needsClarification'] = False
|
||||
previous['productResolution'].update(selectedName='18X15X4+T', selectionQuestion='选 +T')
|
||||
result = resolve_clarification_choice('选+T', self.options, previous)
|
||||
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
|
||||
self.assertEqual(result['productMatch']['previousQuestion'], '今年18x15x4合格率变化趋势')
|
||||
self.assertIsNone(resolve_clarification_choice('选+Z', self.options, previous))
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', {'products':['18X15X4+Z']}, previous))
|
||||
previous['productResolution']['selectedName'] = '28X20X3'
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', self.options, previous))
|
||||
|
||||
def test_unrelated_or_multi_variant_followup_does_not_choose(self):
|
||||
for reply in ('再看整体合格率', '我想比较+T和+Z', '只查18x15x4+T九月趋势'):
|
||||
with self.subTest(reply=reply):
|
||||
self.assertIsNone(resolve_clarification_choice(reply, self.options, self.pending()))
|
||||
prior = self.pending(); prior['needsClarification'] = False
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', self.options, prior))
|
||||
|
||||
def test_complete_candidate_choice_from_ui_uses_the_same_pending_frame(self):
|
||||
for reply in ('选择18x15x4+T','18X15X4+T','选择18X15X4+T。','继续上一轮分析,保留未修改的原目标和条件。\n我的选择:18X15X4+T'):
|
||||
result=resolve_clarification_choice(reply,self.options,self.pending())
|
||||
self.assertEqual(result['mentionedProducts'],['18X15X4+T'])
|
||||
self.assertTrue(result['productMatch']['selectedFromClarification'])
|
||||
self.assertEqual(result['productMatch']['previousQuestion'],'今年18x15x4合格率变化趋势')
|
||||
|
||||
def test_truncated_or_duplicate_suffix_candidates_cannot_select(self):
|
||||
prior = self.pending(); prior['productResolution']['productMatch']['truncated'] = True
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', self.options, prior))
|
||||
options = {'products': ['18X15X4+A+T', '18X15X4+B+T']}
|
||||
prior = {'needsClarification': True, 'productResolution': {
|
||||
'originalQuestion':'今年18x15x4合格率变化趋势',
|
||||
'productMatch': identify_products('18x15x4', options)['productMatch']}}
|
||||
self.assertIsNone(resolve_clarification_choice('选+T', options, prior))
|
||||
|
||||
def test_case_colliding_catalog_names_remain_ambiguous(self):
|
||||
match = identify_products('18x15x4+t', {'products':['18X15X4+T', '18x15x4+t']})['productMatch']
|
||||
self.assertEqual(match['status'], 'ambiguous')
|
||||
self.assertEqual(match['candidateCount'], 2)
|
||||
|
||||
def test_customer_code_that_looks_like_specification_is_not_a_product(self):
|
||||
for question in ('今年客户代码18x15x4的合格率变化趋势', '查客户编码为18X15X4+T的趋势',
|
||||
'客户代码“18x15x4”今年合格率趋势', '查客户代码为「18X15X4+T」的趋势',
|
||||
'客户代码 "18x15x4" 的合格率走势'):
|
||||
with self.subTest(question=question):
|
||||
result = identify_products(question, self.options)
|
||||
self.assertEqual(result['mentionedProducts'], [])
|
||||
self.assertEqual(result['productMatch']['status'], 'none')
|
||||
|
||||
def test_customer_code_and_explicit_product_are_kept_separate(self):
|
||||
result = identify_products('客户代码18x15x4的18X15X4+T产品合格率趋势', self.options)
|
||||
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
|
||||
self.assertEqual(len(result['productMatch']['mentions']), 1)
|
||||
self.assertEqual(result['productMatch']['status'], 'exact')
|
||||
quoted = identify_products('客户代码“18x15x4”的18X15X4+T产品合格率趋势', self.options)
|
||||
self.assertEqual(quoted['mentionedProducts'], ['18X15X4+T'])
|
||||
self.assertEqual(quoted['productMatch']['status'], 'exact')
|
||||
|
||||
def test_authorized_extra_fields_are_not_inferred_or_expanded(self):
|
||||
catalog = {'kind':'analysis_catalog', 'extraFields':[
|
||||
{'key':'customerCode', 'label':'客户代码', 'blackColumn':'客户代码', 'brightColumn':'客户代码', 'enabled':True},
|
||||
{'key':'disabled', 'label':'禁用字段', 'enabled':False},
|
||||
]}
|
||||
self.assertEqual(authorized_extra_fields(catalog),
|
||||
{'extraFieldsStatus':'ready', 'extraFields':[{'key':'customerCode','label':'客户代码'}]})
|
||||
self.assertEqual(authorized_extra_fields({'kind':'analysis_catalog','extraFields':[]})['extraFields'], [])
|
||||
self.assertEqual(authorized_extra_fields({'kind':'missing'})['extraFieldsStatus'], 'unavailable')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,240 @@
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_protocol_execution import execute
|
||||
from nao_agent_task import normalize_task, bind_task
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_quality_tools import FakeApi, RELEASE
|
||||
|
||||
ACTOR='11111111-1111-4111-8111-111111111111'
|
||||
SESSION='22222222-2222-4222-8222-222222222222'
|
||||
TURN='55555555-5555-4555-8555-555555555555'
|
||||
|
||||
def task(filters=None):
|
||||
value = normalize_task(task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],trendRequired=True,
|
||||
period={'year':2026},filterUpdates=filters or {}),'今年合格率月度趋势',{'_protocol':{'version':1}})
|
||||
if value['filters'].get('columnFilters') == []: value['filters'].pop('columnFilters')
|
||||
return value
|
||||
|
||||
def ranking_task(**updates):
|
||||
plan=task_value(kind='analysis',dimensions=['product','group'],
|
||||
dimensionStages={'product':['bright','black'],'group':['bright']},stages=['black','bright'],
|
||||
fields=['totalWeight','scrapWeight'],period={'year':2026},trendRequired=False,
|
||||
visualPreference='auto',visualType='pareto')
|
||||
plan.update(updates)
|
||||
value=normalize_task(plan,
|
||||
'按产品和业务分组查看各阶段重量排名',{'_protocol':{'version':1}})
|
||||
if value['filters'].get('columnFilters') == []:value['filters'].pop('columnFilters')
|
||||
return value
|
||||
|
||||
def identity():
|
||||
return {'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN}
|
||||
|
||||
class BoundApi(FakeApi):
|
||||
def __init__(self,proof=True,cancel=None):
|
||||
super().__init__();self.proof=proof;self.cancel=cancel
|
||||
def call(self,path,payload=None,method=None):
|
||||
result=super().call(path,payload,method)
|
||||
if path.startswith('/api/metrics'):
|
||||
parsed=parse_qs(urlsplit(path).query)
|
||||
if 'columnFilters' in parsed:
|
||||
result['context']['filters']={'columnFilters':json.loads(parsed['columnFilters'][0])}
|
||||
if self.proof:result['protocolEvidence']={**identity(),'queryId':TURN,'responseHash':'a'*64}
|
||||
if self.cancel:self.cancel.set()
|
||||
return result
|
||||
|
||||
class BoundRankingApi(BoundApi):
|
||||
def __init__(self,proof=True,cancel=None):
|
||||
super().__init__(proof=proof,cancel=cancel);self.rank_query=0
|
||||
def call(self,path,payload=None,method=None):
|
||||
result=super().call(path,payload,method)
|
||||
if path.startswith('/api/metrics'):
|
||||
self.rank_query+=1
|
||||
parsed=parse_qs(urlsplit(path).query)
|
||||
view=parsed['view'][0]
|
||||
rows=[]
|
||||
for stage,total,scrap in (('black',100,10),('bright',150,12)):
|
||||
result['summary'][stage]['totalWeight']=total
|
||||
result['summary'][stage]['scrapWeight']=scrap
|
||||
for month in result['coverage']['sourceMonths']:
|
||||
month['blackRows']=10;month['brightRows']=12
|
||||
for suffix,share in (('a',.6),('b',.4)):
|
||||
row={'key':view+'-'+suffix,'label':view+' '+suffix,'coverage':{'blackRows':2,'brightRows':2}}
|
||||
for stage in ('black','bright'):
|
||||
summary=result['summary'][stage]
|
||||
row[stage]={key:summary[key]*share for key in ('totalWeight','scrapWeight')}
|
||||
rows.append(row)
|
||||
result['rows']=rows;result['context']['rowCount']=len(rows)
|
||||
if 'monthStart' in parsed:
|
||||
result['context']['monthStart']=int(parsed['monthStart'][0])
|
||||
result['context']['monthEnd']=int(parsed['monthEnd'][0])
|
||||
if self.proof:
|
||||
result['protocolEvidence']={**identity(),'queryId':f'{TURN}-{self.rank_query}',
|
||||
'responseHash':f'{self.rank_query:064x}'}
|
||||
return result
|
||||
|
||||
class ProtocolExecutionTests(unittest.TestCase):
|
||||
def test_comparison_returns_both_proofs_and_the_confirmed_comparison_context(self):
|
||||
value=task_value(kind='metrics',dimensions=[],stages=['black'],fields=['rate'],trendRequired=False,
|
||||
comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='none')
|
||||
t=normalize_task(value,'2026年9月对比8月黑片合格率',{'_protocol':{'version':1}})
|
||||
t['filters'].pop('columnFilters',None)
|
||||
class ComparisonApi(BoundApi):
|
||||
def call(inner,path,payload=None,method=None):
|
||||
response=super().call(path,payload,method)
|
||||
if path.startswith('/api/metrics'):
|
||||
month=parse_qs(urlsplit(path).query).get('month',['0'])[0]
|
||||
response['protocolEvidence']['queryId']='query-'+month
|
||||
return response
|
||||
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={'answer':'已核对两个期间','evidenceSufficient':True,'missingEvidence':[]}):
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,authority=identity(),actor_root=root,
|
||||
delegated=ComparisonApi(),connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertEqual(result['agentContext']['comparisonPeriod'],{'year':2026,'month':8})
|
||||
self.assertEqual({p['queryId'] for p in result['protocolEvidence']},{'query-9','query-8'})
|
||||
self.assertEqual(result['agentContext']['period'],{'year':2026,'month':9})
|
||||
|
||||
def exercise(self,filters=None,api=None):
|
||||
t=task(filters);request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
|
||||
event=threading.Event();api=api or BoundApi();emitted=[]
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize',return_value={'answer':'月度正式走势','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
|
||||
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
|
||||
delegated=api,connection=('offline','synthetic','model'),cancel_event=event,emit=emitted.append)
|
||||
documents=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
|
||||
return result,api,documents,synth
|
||||
|
||||
def test_real_tools_and_visuals_preserve_bound_monthly_series_and_cleared_scope(self):
|
||||
selected,api,docs,_=self.exercise({'product':'18X15X4+T'})
|
||||
self.assertEqual(selected['status'],'COMPLETED')
|
||||
self.assertEqual(selected['agentContext']['filters'],{'product':'18X15X4+T'})
|
||||
self.assertTrue(any('product=18X15X4%2BT' in path for _,path,_ in api.calls))
|
||||
self.assertEqual(len(docs),1)
|
||||
self.assertEqual(len(docs[0]['result']['trend']),12)
|
||||
self.assertEqual(docs[0]['result']['protocolEvidence']['executionId'],TURN)
|
||||
self.assertTrue(selected['visuals'])
|
||||
self.assertEqual(selected['protocolEvidence'][0]['queryId'],TURN)
|
||||
cleared,api,docs,_=self.exercise()
|
||||
self.assertEqual(cleared['agentContext']['filters'],{})
|
||||
self.assertTrue(all('product=' not in path for _,path,_ in api.calls))
|
||||
self.assertTrue(all('columnFilters=' not in path for _,path,_ in api.calls))
|
||||
self.assertEqual(docs[0]['result']['filters'],{'includeUnassigned':False})
|
||||
|
||||
def test_query_without_matching_execution_proof_never_reaches_the_model(self):
|
||||
with self.assertRaises(ValueError):self.exercise(api=BoundApi(proof=False))
|
||||
|
||||
def test_weight_ranking_reads_dimension_stage_field_cartesian_product_before_shared_finalizer(self):
|
||||
t=ranking_task();api=BoundRankingApi();request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
|
||||
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
|
||||
return_value={'answer':'已核验全量重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
|
||||
delegated=api,connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
docs=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
|
||||
docs.sort(key=lambda doc:int(doc['result']['protocolEvidence']['queryId'].rsplit('-',1)[1]))
|
||||
reads=[parse_qs(urlsplit(path).query) for _,path,_ in api.calls if path.startswith('/api/metrics')]
|
||||
self.assertEqual(len(reads),6)
|
||||
self.assertEqual([doc['result']['dimension'] for doc in docs],['product','product','product','product','group','group'])
|
||||
self.assertEqual([doc['result']['stage'] for doc in docs],['bright','bright','black','black','bright','bright'])
|
||||
self.assertEqual([doc['result']['metric'] for doc in docs],['totalWeight','scrapWeight']*3)
|
||||
self.assertEqual(len(result['protocolEvidence']),6)
|
||||
self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}),6)
|
||||
self.assertTrue(all(proof['executionId']==TURN for proof in result['protocolEvidence']))
|
||||
from nao_weight_ranking import matching_rankings,ranking_gaps
|
||||
bound_context=bind_task({'releaseId':RELEASE},t)
|
||||
self.assertEqual(result['status'],'COMPLETED', (result.get('visualState'),
|
||||
[(v.get('dimension'),v.get('stage'),v.get('metric')) for v in result.get('visuals',[])],
|
||||
result.get('evidenceCompletion'),len(matching_rankings(docs,t,bound_context)),
|
||||
ranking_gaps(docs,t,bound_context),[(d['result'].get('dimension'),d['result'].get('stage'),d['result'].get('metric'),d['result'].get('period'),d['result'].get('periodCoverage'),d['result'].get('rankingCoverage')) for d in docs]))
|
||||
self.assertTrue(result['visuals'])
|
||||
self.assertEqual(result['toolCount'],6)
|
||||
synth.assert_called_once()
|
||||
self.assertFalse(any('rowLimit' in params or 'rowOffset' in params for params in reads))
|
||||
|
||||
def test_ranking_and_trend_or_comparison_are_rejected_without_dropping_task_requirements(self):
|
||||
for updates in ({'trendRequired':True},
|
||||
{'comparisonRequired':True,'comparisonPeriod':{'year':2025}}):
|
||||
with self.subTest(updates=updates):
|
||||
# This test exercises execute's fail-closed guard directly.
|
||||
# Planner normalization now correctly rejects the same legacy
|
||||
# shape earlier, before it could be confirmed as a protocol task.
|
||||
t=ranking_task();t.update(updates);api=BoundRankingApi()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with self.assertRaisesRegex(ValueError,'不能与趋势或期间比较合并'):
|
||||
execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
|
||||
authority={**identity(),'task':t},actor_root=root,delegated=api,
|
||||
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertEqual(t['trendRequired'],updates.get('trendRequired',False))
|
||||
self.assertEqual(t.get('comparisonPeriod'),updates.get('comparisonPeriod'))
|
||||
self.assertFalse(any(path.startswith('/api/metrics') for _,path,_ in api.calls))
|
||||
|
||||
def test_ranking_with_ready_but_incomplete_visual_set_finishes_partial(self):
|
||||
t=ranking_task();api=BoundRankingApi()
|
||||
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
|
||||
return_value={'answer':'已核验重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}),\
|
||||
patch('analysis_visual_evidence.visual_result',return_value={
|
||||
'visualSchemaVersion':1,'visualState':'ready','visuals':[]}):
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
|
||||
authority={**identity(),'task':t},actor_root=root,delegated=api,
|
||||
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertEqual(result['visualState'],'ready')
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertTrue(any('重量排名图表未覆盖' in item for item in result['evidenceCompletion']['missingEvidence']))
|
||||
|
||||
def test_clarification_executes_no_tool_or_finalizer(self):
|
||||
t=task();t.update(needsClarification=True,clarification='请选完整规格')
|
||||
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize') as synth:
|
||||
api=MagicMock()
|
||||
result=execute(request={'releaseId':RELEASE,'question':t['question']},task=t,authority=identity(),actor_root=Path(folder),
|
||||
delegated=api,connection=('offline','key','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertEqual(result['status'],'NEEDS_CLARIFICATION');api.call.assert_not_called();synth.assert_not_called()
|
||||
|
||||
def test_backend_previous_task_wins_over_local_task_and_chat(self):
|
||||
previous=task({'product':'18X15X4+T'})
|
||||
planned=task();planned.update(needsClarification=True,clarification='请确认期间')
|
||||
options={'products':['18X15X4+T','18X15X4+Z'],'organizationPaths':[]}
|
||||
captured={}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
||||
local=root/'private'/'last-agent-task.json'
|
||||
local.write_text(json.dumps({'sessionId':SESSION,'releaseId':RELEASE,'task':task({'product':'WRONG'})}));local.chmod(0o600)
|
||||
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
|
||||
api=MagicMock();api.call.side_effect=lambda path,**kw: options if path.startswith('/api/options') else {'task':planned,**identity()} if path=='/api/analysis/protocol/task' else {}
|
||||
delegated=MagicMock();delegated.login.return_value=api
|
||||
gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1'
|
||||
def plan(**kw):captured.update(copy.deepcopy({k:v for k,v in kw.items() if k!='cancel_event'}));return planned
|
||||
with ExitStack() as stack:
|
||||
for name,value in {'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
|
||||
'SessionState':MagicMock(),'_input_cancel_listener':MagicMock(),'_model_settings':MagicMock(return_value=('offline','key','model',{})),
|
||||
'_model_gate_settings':MagicMock(return_value=(1,'key')),'DelegatedApi':MagicMock(return_value=delegated),'plan_agent_task':plan}.items():
|
||||
stack.enter_context(patch.object(runtime,name,value))
|
||||
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
|
||||
result=runtime.run_turn({'turnId':TURN,'actorId':ACTOR,'sessionId':SESSION,'releaseId':'33333333-3333-4333-8333-333333333333',
|
||||
'accessToken':'synthetic','question':'清除规格条件','context':{'_protocol':{'version':1,'attemptId':SESSION,'previousTask':previous},
|
||||
'_serverHistory':[{'answer':'unverified'}],'lastVisibleAnswer':'unverified'}},stdout=io.StringIO(),stdin=io.StringIO())
|
||||
self.assertEqual(result['status'],'NEEDS_CLARIFICATION')
|
||||
self.assertEqual(captured['previous'],previous)
|
||||
self.assertNotIn('_serverHistory',captured['context']);self.assertNotIn('lastVisibleAnswer',captured['context'])
|
||||
manager.ensure_instance.assert_not_called()
|
||||
self.assertEqual(json.loads(local.read_text())['task']['filters']['product'],'WRONG')
|
||||
|
||||
def test_new_task_does_not_inherit_old_filters_or_period(self):
|
||||
context=bind_task({'_protocol':{'version':1}},task({'product':'OLD','department':'OLD'}))
|
||||
t=normalize_task(task_value(newTask=True,kind='metrics',dimensions=[],period={'year':2025}),'另一个新任务',context)
|
||||
self.assertEqual(t['filters'],{'columnFilters':[]});self.assertEqual(t['period'],{'year':2025})
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,433 @@
|
||||
"""Security and stream-state tests for the delegated NAO runtime (no model calls)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from unittest.mock import patch
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
|
||||
import nao_quality_runtime as runtime
|
||||
|
||||
|
||||
class FakeResponse(io.BytesIO):
|
||||
status = 200
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.close()
|
||||
|
||||
|
||||
class FakeOpener:
|
||||
def __init__(self, body=b"{}"):
|
||||
self.body = body
|
||||
self.calls = []
|
||||
|
||||
def open(self, request, timeout=None):
|
||||
self.calls.append((request, timeout))
|
||||
return FakeResponse(self.body)
|
||||
|
||||
|
||||
class RuntimeAccessTests(unittest.TestCase):
|
||||
def test_complete_batch_ranking_progress_does_not_claim_arbitrary_truncation(self):
|
||||
text = runtime.public_tool_result({'kind':'quality_changes','completeRanking':True,
|
||||
'rows':[{'key':'甲'}], 'rowsTruncated':True})
|
||||
self.assertIn('完整授权范围排序',text)
|
||||
self.assertNotIn('不代表全量排名',text)
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-")
|
||||
self.users = Path(self.temp.name).resolve()
|
||||
self.actor = str(uuid.uuid4())
|
||||
self.release = str(uuid.uuid4())
|
||||
self.actor_root = self.users / self.actor
|
||||
self.actor_root.mkdir(mode=0o700)
|
||||
self.access = self.actor_root / "access.json"
|
||||
runtime._atomic_json(self.access, {
|
||||
"token": "delegated-secret-for-test",
|
||||
"releaseId": self.release,
|
||||
"queryUrl": runtime.RUNTIME_QUERY_URL,
|
||||
})
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def test_delegated_login_only_validates_file_and_never_calls_admin_login(self):
|
||||
class NoNetwork:
|
||||
calls = 0
|
||||
|
||||
def open(self, *_args, **_kwargs):
|
||||
self.calls += 1
|
||||
raise AssertionError("login must not call NAO or admin auth")
|
||||
|
||||
opener = NoNetwork()
|
||||
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor, opener=opener)
|
||||
self.assertIs(api.login(), api)
|
||||
self.assertEqual(opener.calls, 0)
|
||||
|
||||
def test_every_call_uses_fixed_loopback_delegate_and_pinned_release(self):
|
||||
opener = FakeOpener(b'{"items":[]}')
|
||||
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor, opener=opener).login()
|
||||
self.assertEqual(api.call("/api/releases", method="GET"), {"items": []})
|
||||
request, _ = opener.calls[0]
|
||||
body = json.loads(request.data.decode())
|
||||
self.assertEqual(request.full_url, runtime.RUNTIME_QUERY_URL)
|
||||
self.assertEqual(set(body), {"token", "method", "path", "payload"})
|
||||
self.assertEqual(body["token"], "delegated-secret-for-test")
|
||||
with self.assertRaisesRegex(ValueError, "releaseId"):
|
||||
api.call("/api/metrics?releaseId=" + str(uuid.uuid4()), method="GET")
|
||||
with self.assertRaisesRegex(ValueError, "允许范围"):
|
||||
api.call("/api/admin/config", method="GET")
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
echoed = runtime.DelegatedApi(
|
||||
self.access, runtime_root=self.users, expected_actor_id=self.actor,
|
||||
opener=FakeOpener(b'{"token":"delegated-secret-for-test","note":"echo delegated-secret-for-test"}'),
|
||||
).call("/api/releases", method="GET")
|
||||
self.assertEqual(echoed, {"note": "echo [REDACTED]"})
|
||||
|
||||
def test_scoped_query_envelope_binds_node_and_scope_without_changing_url(self):
|
||||
opener = FakeOpener(b'{"ok":true}')
|
||||
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor, opener=opener).login()
|
||||
self.assertEqual(api.call('/api/metrics?releaseId=' + self.release, method='GET',
|
||||
node_id='trend_a', scope_id='execution:trend_a:1'), {'ok': True})
|
||||
request, _ = opener.calls[0]
|
||||
wrapper = json.loads(request.data.decode())
|
||||
self.assertEqual(set(wrapper), {'token', 'method', 'path', 'payload', 'nodeId', 'scopeId'})
|
||||
self.assertEqual(wrapper['nodeId'], 'trend_a')
|
||||
self.assertEqual(wrapper['scopeId'], 'execution:trend_a:1')
|
||||
self.assertEqual(wrapper['path'], '/api/metrics?releaseId=' + self.release)
|
||||
with self.assertRaisesRegex(ValueError, '同时提供'):
|
||||
api.call('/api/metrics?releaseId=' + self.release, method='GET', node_id='trend_a')
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
|
||||
def test_batch_changes_reaches_delegate_with_exact_protocol_scope(self):
|
||||
opener=FakeOpener(b'{"ok":true}')
|
||||
api=runtime.DelegatedApi(self.access,runtime_root=self.users,
|
||||
expected_actor_id=self.actor,opener=opener).login()
|
||||
payload={'releaseId':self.release,'period':{'year':2026,'month':9},
|
||||
'comparisonPeriod':{'year':2026,'month':8},'filters':{'group':'消费电子-3C'},
|
||||
'dimension':'product','sort':'current_rate','stage':None,'limit':3,
|
||||
'fields':['rate','totalWeight','scrapWeight']}
|
||||
self.assertEqual(api.call('/api/analysis/changes',payload=payload,method='POST',
|
||||
node_id='products',scope_id='scope:products'),{'ok':True})
|
||||
request,_=opener.calls[0];wrapper=json.loads(request.data)
|
||||
self.assertEqual(wrapper['payload'],payload)
|
||||
self.assertEqual(wrapper['nodeId'],'products')
|
||||
self.assertEqual(wrapper['scopeId'],'scope:products')
|
||||
self.assertEqual(request.full_url,runtime.RUNTIME_QUERY_URL)
|
||||
|
||||
def test_malformed_access_and_non_loopback_endpoint_are_rejected(self):
|
||||
runtime._atomic_json(self.access, {"token": "x", "releaseId": self.release,
|
||||
"queryUrl": "http://example.com/api/analysis/runtime/query"})
|
||||
with self.assertRaisesRegex(ValueError, "本机接口"):
|
||||
runtime.load_access_file(self.access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor)
|
||||
runtime._atomic_json(self.access, {"token": "", "releaseId": self.release,
|
||||
"queryUrl": runtime.RUNTIME_QUERY_URL})
|
||||
with self.assertRaisesRegex(ValueError, "不完整"):
|
||||
runtime.load_access_file(self.access, runtime_root=self.users,
|
||||
expected_actor_id=self.actor)
|
||||
|
||||
def test_access_file_cannot_escape_actor_directory(self):
|
||||
outside = self.users.parent / self.actor / "access.json"
|
||||
with self.assertRaisesRegex(ValueError, "越界"):
|
||||
runtime.load_access_file(outside, runtime_root=self.users)
|
||||
|
||||
def test_access_symlinks_are_rejected(self):
|
||||
link = self.actor_root / "linked-access.json"
|
||||
link.symlink_to(self.access)
|
||||
with self.assertRaisesRegex(ValueError, "路径无效"):
|
||||
runtime.load_access_file(link, runtime_root=self.users)
|
||||
root_link = self.users.parent / (self.users.name + "-link")
|
||||
root_link.symlink_to(self.users, target_is_directory=True)
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError, "运行目录无效"):
|
||||
runtime.load_access_file(self.access, runtime_root=root_link)
|
||||
finally:
|
||||
root_link.unlink()
|
||||
|
||||
def test_session_to_native_chat_mapping_is_actor_scoped(self):
|
||||
session_id = str(uuid.uuid4())
|
||||
first = runtime.SessionState(self.actor_root, self.actor)
|
||||
other_actor = str(uuid.uuid4())
|
||||
other_root = self.users / other_actor
|
||||
other_root.mkdir(mode=0o700)
|
||||
second = runtime.SessionState(other_root, other_actor)
|
||||
first.set_chat_id(session_id, "native-chat-one")
|
||||
second.set_chat_id(session_id, "native-chat-two")
|
||||
self.assertEqual(first.get_chat_id(session_id), "native-chat-one")
|
||||
self.assertEqual(second.get_chat_id(session_id), "native-chat-two")
|
||||
self.assertNotEqual(first.path, second.path)
|
||||
|
||||
|
||||
class RuntimeStreamTests(unittest.TestCase):
|
||||
def test_accumulator_keeps_provider_reasoning_separate_from_answer(self):
|
||||
updates = []
|
||||
stream = runtime.ConversationAccumulator(updates.append)
|
||||
stream.feed({"type": "reasoning-delta", "delta": "private thought"})
|
||||
stream.feed({"type": "text-start", "id": "first"})
|
||||
stream.feed({"type": "text-delta", "id": "first", "delta": "early block"})
|
||||
stream.feed({"type": "text-start", "id": "final"})
|
||||
stream.feed({"type": "text-delta", "id": "final", "delta": "final answer"})
|
||||
self.assertEqual(stream.answer, "final answer")
|
||||
self.assertNotIn("private thought", updates)
|
||||
self.assertEqual(stream.model_reasoning, "private thought")
|
||||
|
||||
def test_provider_reasoning_is_bounded_and_never_inferred_from_answer(self):
|
||||
stream = runtime.ConversationAccumulator()
|
||||
stream.feed({"type": "text-delta", "delta": "Only a final answer"})
|
||||
self.assertEqual(stream.model_reasoning, "")
|
||||
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "x" * 199_999})
|
||||
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "yz"})
|
||||
stream.feed({"type": "reasoning-delta", "id": "r2", "delta": "more"})
|
||||
self.assertEqual(len(stream.model_reasoning.strip()), 200_000)
|
||||
self.assertTrue(stream.reasoning_truncated)
|
||||
self.assertEqual(stream.answer, "Only a final answer")
|
||||
|
||||
def test_length_and_stringified_mcp_error_are_partial(self):
|
||||
stream = runtime.ConversationAccumulator()
|
||||
stream.feed({"type": "tool-input-available", "toolName": "quality_data", "toolCallId": "q1"})
|
||||
stream.feed({"type": "tool-output-available", "toolCallId": "q1",
|
||||
"output": '{"isError":true,"content":[{"text":"forbidden"}]}'})
|
||||
stream.feed({"type": "text-delta", "id": "answer", "delta": "查询受限,说明原因"})
|
||||
stream.feed({"type": "finish", "finishReason": "stop"})
|
||||
self.assertEqual(stream.tool_calls[0]["status"], "failed")
|
||||
self.assertEqual(stream.status()[0], "PARTIAL")
|
||||
output_error = runtime.ConversationAccumulator()
|
||||
output_error.feed({"type": "tool-input-available", "toolName": "details", "toolCallId": "q2"})
|
||||
output_error.feed({"type": "tool-output-error", "toolCallId": "q2"})
|
||||
self.assertEqual(output_error.tool_calls[0]["status"], "failed")
|
||||
truncated = runtime.ConversationAccumulator()
|
||||
truncated.feed({"type": "text-delta", "id": "a", "delta": "正文未完"})
|
||||
truncated.feed({"type": "finish", "finishReason": "length"})
|
||||
self.assertEqual(truncated.status(), ("PARTIAL", "length"))
|
||||
|
||||
def test_native_stop_controller_stops_before_closing_stream_on_cancel(self):
|
||||
calls = []
|
||||
response = FakeResponse()
|
||||
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=5)
|
||||
controller.set_chat_id("native-id")
|
||||
controller.set_response(response)
|
||||
controller.start()
|
||||
controller.request_cancel("user")
|
||||
controller.close()
|
||||
self.assertEqual(calls, ["native-id"])
|
||||
self.assertTrue(response.closed)
|
||||
stream = runtime.ConversationAccumulator()
|
||||
self.assertEqual(stream.status(cancel_cause="user", stop_succeeded=True),
|
||||
("CANCELLED", "user_cancel"))
|
||||
|
||||
def test_watchdog_timeout_invokes_native_stop(self):
|
||||
calls = []
|
||||
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=0.03)
|
||||
controller.set_chat_id("timed-native-id")
|
||||
controller.start()
|
||||
deadline = time.monotonic() + 1
|
||||
while controller.thread.is_alive() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
controller.close()
|
||||
self.assertEqual(calls, ["timed-native-id"])
|
||||
self.assertEqual(controller.cause, "timeout")
|
||||
|
||||
def test_complete_evidence_can_stop_native_before_turn_timeout(self):
|
||||
calls = []
|
||||
controller = runtime.NativeStopController(
|
||||
lambda chat_id: calls.append(chat_id) or True, timeout=3,
|
||||
completion_ready=lambda: True)
|
||||
controller.set_chat_id("covered-native-id")
|
||||
controller.start()
|
||||
deadline = time.monotonic() + 2
|
||||
while controller.thread.is_alive() and time.monotonic() < deadline:
|
||||
time.sleep(0.02)
|
||||
controller.close()
|
||||
self.assertEqual(calls, ["covered-native-id"])
|
||||
self.assertEqual(controller.cause, "coverage_ready")
|
||||
stream = runtime.ConversationAccumulator()
|
||||
self.assertEqual(stream.status(cancel_cause="coverage_ready", stop_succeeded=True),
|
||||
("FAILED", "coverage_ready"))
|
||||
|
||||
def test_transport_failure_stops_native_and_falls_back_to_owned_instance(self):
|
||||
class BrokenClient:
|
||||
def open(self, *_args, **_kwargs):
|
||||
raise OSError("private transport detail must not escape")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
||||
users = Path(temp).resolve()
|
||||
actor_id = str(uuid.uuid4())
|
||||
actor_root = users / actor_id
|
||||
actor_root.mkdir(mode=0o700)
|
||||
session_id = str(uuid.uuid4())
|
||||
state = runtime.SessionState(actor_root, actor_id)
|
||||
with patch.object(runtime, "_stop_native", return_value=False) as native_stop, \
|
||||
patch.object(runtime, "_stop_instance", return_value=True) as instance_stop:
|
||||
result = runtime._sse_request(
|
||||
client=BrokenClient(), base_url="http://127.0.0.1:10000", model="test",
|
||||
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
|
||||
session_state=state, session_id=session_id, actor_root=actor_root,
|
||||
emit=lambda _event: None, timeout=5,
|
||||
)
|
||||
self.assertEqual(result[1], "FAILED")
|
||||
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
|
||||
instance_stop.assert_called_once_with(actor_root, grace_seconds=1.0)
|
||||
|
||||
def test_early_eof_without_finish_or_done_stops_native_task(self):
|
||||
class EofClient:
|
||||
def open(self, *_args, **_kwargs):
|
||||
return FakeResponse(b'data: {"type":"text-delta","id":"a","delta":"partial"}\n')
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
||||
users = Path(temp).resolve()
|
||||
actor_id = str(uuid.uuid4())
|
||||
actor_root = users / actor_id
|
||||
actor_root.mkdir(mode=0o700)
|
||||
session_id = str(uuid.uuid4())
|
||||
with patch.object(runtime, "_stop_native", return_value=True) as native_stop:
|
||||
result = runtime._sse_request(
|
||||
client=EofClient(), base_url="http://127.0.0.1:10000", model="test",
|
||||
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
|
||||
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
|
||||
actor_root=actor_root, emit=lambda _event: None, timeout=5,
|
||||
)
|
||||
self.assertEqual(result[1], "PARTIAL")
|
||||
self.assertEqual(result[2], "transport_or_stream_error")
|
||||
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
|
||||
|
||||
def test_story_export_uses_last_successful_call_and_matching_markdown(self):
|
||||
class StoryClient:
|
||||
def __init__(self):
|
||||
self.slugs = []
|
||||
|
||||
def open(self, request, timeout=None):
|
||||
encoded_input = parse_qs(urlsplit(request.full_url).query)["input"][0]
|
||||
query = json.loads(encoded_input)["json"]
|
||||
slug = query["storySlug"]
|
||||
self.slugs.append(slug)
|
||||
html = f"<html>{slug}</html>".encode()
|
||||
payload = {"result": {"data": {"json": {
|
||||
"data": base64.b64encode(html).decode("ascii")
|
||||
}}}}
|
||||
return FakeResponse(json.dumps(payload).encode())
|
||||
|
||||
client = StoryClient()
|
||||
calls = [
|
||||
{"tool": "story", "toolCallId": "create"},
|
||||
{"tool": "story", "toolCallId": "replace"},
|
||||
{"tool": "story", "toolCallId": "failed-replacement"},
|
||||
]
|
||||
outputs = {
|
||||
"create": {"success": True, "id": "old-report", "code": "# Initial revision"},
|
||||
"replace": {"success": True, "id": "final-report", "code": "# Final revision"},
|
||||
"failed-replacement": {"success": False, "id": "broken-report", "code": "# Failed"},
|
||||
}
|
||||
markdown, html = runtime._story_files(client, "http://127.0.0.1:10000", "chat", calls, outputs)
|
||||
self.assertEqual(markdown, "# Final revision")
|
||||
self.assertEqual(html, "<html>final-report</html>")
|
||||
self.assertEqual(client.slugs, ["final-report"])
|
||||
|
||||
def test_sse_uses_idle_timeout_and_warm_session_configuration_is_reused(self):
|
||||
class CaptureClient:
|
||||
def __init__(self):
|
||||
self.timeout = None
|
||||
|
||||
def open(self, request, timeout=None):
|
||||
self.timeout = timeout
|
||||
return FakeResponse(b'data: {"type":"reasoning-delta","id":"r","delta":"provider reasoning"}\n'
|
||||
b'data: {"type":"reasoning-end","id":"r"}\n'
|
||||
b'data: {"type":"text-delta","id":"a","delta":"answer"}\n'
|
||||
b'data: {"type":"finish","finishReason":"stop"}\n'
|
||||
b'data: [DONE]\n')
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
||||
users = Path(temp).resolve()
|
||||
actor_id = str(uuid.uuid4())
|
||||
session_id = str(uuid.uuid4())
|
||||
actor_root = users / actor_id
|
||||
actor_root.mkdir(mode=0o700)
|
||||
project = actor_root / "project"
|
||||
project.mkdir()
|
||||
current_session = actor_root / "private" / "current-session.json"
|
||||
runtime._atomic_json(current_session, {"actorId": actor_id, "sessionId": session_id})
|
||||
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=True,
|
||||
refresh_mcp=True, backup_schema=False)
|
||||
mcp_path = project / "agent" / "mcps" / "mcp.json"
|
||||
original_config = mcp_path.read_bytes()
|
||||
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=False,
|
||||
refresh_mcp=False, backup_schema=False)
|
||||
self.assertEqual(mcp_path.read_bytes(), original_config)
|
||||
|
||||
client = CaptureClient()
|
||||
events = []
|
||||
result = runtime._sse_request(
|
||||
client=client, base_url="http://127.0.0.1:10000", model="test-model",
|
||||
question="q", release_id=str(uuid.uuid4()), context=None, chat_id=None,
|
||||
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
|
||||
actor_root=actor_root, emit=events.append, timeout=5,
|
||||
)
|
||||
self.assertEqual(client.timeout, runtime.SSE_IDLE_TIMEOUT_SECONDS)
|
||||
self.assertEqual(result[1], "COMPLETED")
|
||||
self.assertEqual(result[0].answer, "answer")
|
||||
self.assertFalse(any("modelReasoning" in event for event in events))
|
||||
self.assertTrue(any(event.get("timeline") for event in events))
|
||||
|
||||
next_session = str(uuid.uuid4())
|
||||
runtime._write_project(actor_root, next_session, "test-model", refresh_templates=False,
|
||||
refresh_mcp=True, backup_schema=False)
|
||||
updated = json.loads(mcp_path.read_text(encoding="utf-8"))
|
||||
server_env = updated["mcpServers"]["quality"]["env"]
|
||||
self.assertEqual(server_env["QUALITY_AGENT_SESSION_ID"], next_session)
|
||||
self.assertEqual(json.loads(current_session.read_text())["sessionId"], next_session)
|
||||
|
||||
|
||||
class RuntimeManagerTests(unittest.TestCase):
|
||||
def test_ensure_instance_executes_project_setup_and_registry_path(self):
|
||||
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
||||
users = Path(temp).resolve()
|
||||
manager = runtime.RuntimeManager(users_root=users)
|
||||
actor_id = str(uuid.uuid4())
|
||||
session_id = str(uuid.uuid4())
|
||||
actor_root, actor_lock = manager.acquire_actor(actor_id)
|
||||
fake_port = 49123
|
||||
|
||||
def fake_start(root, model_base, model_key, model_name, fingerprint):
|
||||
self.assertEqual(root, actor_root)
|
||||
runtime._atomic_json(root / "instance.json", {
|
||||
"actorId": actor_id, "serverPort": fake_port,
|
||||
"fingerprint": fingerprint, "processes": [],
|
||||
})
|
||||
|
||||
try:
|
||||
with patch.object(runtime, "_model_settings",
|
||||
return_value=("https://model.invalid/v1", "test-key", "test-model")), \
|
||||
patch.object(runtime, "_config_fingerprint", return_value="test-fingerprint"), \
|
||||
patch.object(manager, "_start", side_effect=fake_start) as start:
|
||||
config = manager.ensure_instance(actor_root, session_id)
|
||||
self.assertEqual(config, ("https://model.invalid/v1", "test-key", "test-model",
|
||||
f"http://127.0.0.1:{fake_port}"))
|
||||
start.assert_called_once()
|
||||
project = actor_root / "project"
|
||||
self.assertTrue((project / "RULES.md").is_file())
|
||||
self.assertTrue((project / "nao_config.yaml").is_file())
|
||||
mcp_config = json.loads((project / "agent" / "mcps" / "mcp.json").read_text())
|
||||
self.assertEqual(mcp_config["mcpServers"]["quality"]["env"]["QUALITY_AGENT_SESSION_ID"],
|
||||
session_id)
|
||||
self.assertEqual(runtime.RuntimeManager._current_session(actor_root), session_id)
|
||||
finally:
|
||||
manager.release_actor(actor_lock)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,731 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from dbgpt_trial_data import encoded_selection
|
||||
from nao_quality_mcp import MCPServer, serve
|
||||
from nao_quality_tools import (
|
||||
QualityToolError,
|
||||
QualityTools,
|
||||
normalize_filters,
|
||||
symmetric_stage_decomposition,
|
||||
tool_definition,
|
||||
)
|
||||
|
||||
|
||||
RELEASE = "release-a"
|
||||
|
||||
|
||||
def snapshot_period(year: int, month: int | None, view: str, params: dict, rows: list[dict] | None = None) -> dict:
|
||||
months = [month] if month is not None else list(range(1, 13))
|
||||
rate_by_month = {1: (0.94, 0.96), 2: (0.95, 0.97), 3: (0.96, 0.98)}
|
||||
black_rate, bright_rate = rate_by_month.get(month or 2, (0.95, 0.97))
|
||||
black_scrap = 11.0 + (month or 2)
|
||||
bright_scrap = 7.0 + (month or 2)
|
||||
summary = {
|
||||
"key": "overall", "label": "总体", "compositeRate": black_rate * bright_rate,
|
||||
"black": {"rate": black_rate, "qualifiedWeight": 1000.0, "scrapWeight": black_scrap,
|
||||
"totalWeight": 1000.0 + black_scrap,
|
||||
"defects": [{"name": "责废", "weight": black_scrap / 2, "rate": 0.5},
|
||||
{"name": "料废", "weight": black_scrap / 3, "rate": 1 / 3},
|
||||
{"name": "外废", "weight": black_scrap / 6, "rate": 1 / 6}]},
|
||||
"bright": {"rate": bright_rate, "qualifiedWeight": 900.0, "scrapWeight": bright_scrap,
|
||||
"totalWeight": 900.0 + bright_scrap,
|
||||
"defects": [{"name": "镀废", "weight": bright_scrap / 4, "rate": 0.25},
|
||||
{"name": "材料废", "weight": bright_scrap / 4, "rate": 0.25},
|
||||
{"name": "加工废", "weight": bright_scrap / 4, "rate": 0.25},
|
||||
{"name": "扩散废", "weight": bright_scrap / 4, "rate": 0.25}]},
|
||||
"coverage": {"blackRows": 10, "brightRows": 12},
|
||||
}
|
||||
source_months = [{"year": year, "month": current, "blackRows": 10, "brightRows": 12} for current in months]
|
||||
context = {"year": year, "month": month, "quarter": None, "view": view,
|
||||
"department": params.get("department"), "group": params.get("group"),
|
||||
"product": params.get("product"), "projectGroup": params.get("projectGroup"),
|
||||
"includeUnassigned": params.get("includeUnassigned", False),
|
||||
"exclusions": json.loads(params["exclusions"]) if "exclusions" in params else {}}
|
||||
if rows is None:
|
||||
rows = [summary] if view != "overall" else []
|
||||
row_count = len(rows)
|
||||
if "rowLimit" in params:
|
||||
start = int(params.get("rowOffset", 0))
|
||||
rows = rows[start:start + params["rowLimit"]]
|
||||
context["rowCount"] = 1 if view != "overall" else len(rows)
|
||||
context["rowCount"] = row_count
|
||||
trend = [{"period": f"{year}-{current:02d}", "periodLabel": f"{year}-{current:02d}", **summary} for current in months]
|
||||
return {
|
||||
"release": {"id": params.get("releaseId", RELEASE), "name": "发布版本 A", "status": "PUBLISHED"},
|
||||
"context": context, "summary": summary, "rows": rows, "trend": trend,
|
||||
"defects": {"black": summary["black"]["defects"], "bright": summary["bright"]["defects"]},
|
||||
"coverage": {"rawRows": len(months) * 22, "includedRows": len(months) * 22,
|
||||
"excludedBlankRows": 0, "unmappedRows": 0, "projectGroupUnconfiguredRows": 0,
|
||||
"missingStages": [], "sourceMonths": source_months},
|
||||
}
|
||||
|
||||
|
||||
class FakeApi:
|
||||
def __init__(self, *, bad_coverage_month: int | None = None, zero_denominator_month: int | None = None,
|
||||
products: dict[int, list[dict]] | None = None) -> None:
|
||||
self.calls: list[tuple[str, str, dict | None]] = []
|
||||
self.bad_coverage_month = bad_coverage_month
|
||||
self.zero_denominator_month = zero_denominator_month
|
||||
self.products = products or {}
|
||||
self.current_release_id = RELEASE
|
||||
|
||||
def login(self):
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _product_row(key: str, product: str, black_scrap: float, bright_scrap: float,
|
||||
black_rate: float = 0.95, bright_rate: float = 0.97) -> dict:
|
||||
return {"key": key, "product": product, "label": product, "compositeRate": black_rate * bright_rate,
|
||||
"black": {"rate": black_rate, "qualifiedWeight": 100.0, "scrapWeight": black_scrap,
|
||||
"totalWeight": 100.0 + black_scrap, "defects": [{"name": "责废", "weight": black_scrap}]},
|
||||
"bright": {"rate": bright_rate, "qualifiedWeight": 100.0, "scrapWeight": bright_scrap,
|
||||
"totalWeight": 100.0 + bright_scrap, "defects": [{"name": "镀废", "weight": bright_scrap}]},
|
||||
"coverage": {"blackRows": 4, "brightRows": 5}}
|
||||
|
||||
def call(self, path: str, payload=None, method=None):
|
||||
self.calls.append((method or "GET", path, payload))
|
||||
parsed = urlsplit(path)
|
||||
params = {key: values[-1] for key, values in parse_qs(parsed.query).items()}
|
||||
for key in ("year", "month", "quarter", "rowLimit", "rowOffset", "limit"):
|
||||
if key in params:
|
||||
params[key] = int(params[key])
|
||||
if "includeUnassigned" in params:
|
||||
params["includeUnassigned"] = params["includeUnassigned"].lower() == "true"
|
||||
if parsed.path == "/api/releases":
|
||||
return {"currentReleaseId": self.current_release_id, "items": [
|
||||
{"id": RELEASE, "name": "发布版本 A", "status": "PUBLISHED"},
|
||||
{"id": "candidate", "name": "候选", "status": "CANDIDATE"},
|
||||
{"id": "release-b", "name": "发布版本 B", "status": "PUBLISHED"},
|
||||
]}
|
||||
if parsed.path == "/api/analysis/tools/catalog":
|
||||
return {"kind": "analysis_catalog", "visibleWeightFields": ["black_scrap_weight"],
|
||||
"visibleDefectFields": [{"code": "black_responsibility_scrap", "name": "责废"}],
|
||||
"dimensions": [{"key": "product"}], "topDimensions": ["category", "product"],
|
||||
"grouping": ["overall", "product"], "scope": "当前授权范围", "limitations": []}
|
||||
if parsed.path == "/api/options":
|
||||
return {"years": [2025, 2026], "departments": [], "groups": [], "products": [], "projectGroups": []}
|
||||
if parsed.path == "/api/metrics":
|
||||
year = int(params["year"])
|
||||
month = int(params["month"]) if "month" in params else None
|
||||
view = params.get("view", "overall")
|
||||
rows = self.products.get(month or 0, []) if view == "product" else None
|
||||
result = snapshot_period(year, month, view, params, rows)
|
||||
if self.bad_coverage_month == month:
|
||||
result["coverage"]["sourceMonths"][0]["brightRows"] = 0
|
||||
result["coverage"]["missingStages"] = [{"year": year, "month": month, "stage": "bright"}]
|
||||
if self.zero_denominator_month == month:
|
||||
result["summary"]["bright"]["qualifiedWeight"] = 0
|
||||
result["summary"]["bright"]["scrapWeight"] = 0
|
||||
result["summary"]["bright"]["totalWeight"] = 0
|
||||
result["summary"]["bright"]["rate"] = 0
|
||||
result["summary"]["compositeRate"] = 0
|
||||
return result
|
||||
if parsed.path == "/api/analytics/top":
|
||||
result_query = {key: value for key, value in params.items()
|
||||
if key in {"releaseId", "stage", "dimension", "category", "year", "month", "quarter",
|
||||
"department", "group", "product", "projectGroup", "includeUnassigned", "limit"}}
|
||||
return {"releaseId": params.get("releaseId"), "stage": params.get("stage"),
|
||||
"dimension": params.get("dimension"), "category": params.get("category", ""),
|
||||
"denominator": 10, "shareBasis": "当前筛选下分组对象重量合计;不表示总体贡献",
|
||||
"rankingCoverage": {"complete": False}, "coverage": {"rawRows": 22},
|
||||
"rows": [{"rank": 1, "key": "规格 A", "label": "规格 A", "sourceCategory": "责废",
|
||||
"weight": 10, "denominator": 10, "share": 1, "coverage": {"rows": 2}}],
|
||||
"cumulativeThresholdApplied": False, "resultId": "top-id", "resultQuery": result_query}
|
||||
if parsed.path == "/api/analysis/tools/details":
|
||||
assert method == "POST"
|
||||
return {"kind": "analysis_details", "rows": [{"product": "规格 A"}], "rowCount": 1,
|
||||
"limit": payload["limit"], "sampled": True, "truncated": True,
|
||||
"scopeExplanation": "有限样本不代表全部明细。",
|
||||
"range": {**{key: payload.get(key) for key in ("releaseId", "stage", "year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned")}, "exclusions": payload.get("exclusions", {})}}
|
||||
if parsed.path == "/api/analysis/defects":
|
||||
assert method == "POST"
|
||||
if payload.get("mode") == "catalog":
|
||||
return {"kind": "fine_defect_catalog", "releaseId": payload["releaseId"],
|
||||
"release": {"id": payload["releaseId"], "status": "PUBLISHED"},
|
||||
"context": {"stage": payload.get("stage"), "mode": "catalog"},
|
||||
"grain": "source_defect_weight", "stages": [payload.get("stage")] if payload.get("stage") else ["black", "bright"],
|
||||
"categories": [{"stage": "black", "category": "责废", "available": True,
|
||||
"formalComponent": "black_responsibility_scrap"}],
|
||||
"fields": [{"stage": "black", "key": "责废尺寸重量", "label": "责废尺寸重量",
|
||||
"category": "责废", "sourceField": "责废尺寸重量", "available": True}],
|
||||
"scanned": False, "limitations": ["固定源字段目录"]}
|
||||
year, month = payload["year"], payload.get("month", 1)
|
||||
if payload.get("dimension") == "product":
|
||||
rows = [{"key": "product-a", "label": "规格 A", "category": payload.get("category") or "责废",
|
||||
"defect": payload.get("defect"), "weight": 13 if month == 3 else 10, "sourceRows": 8, "presentRows": 8,
|
||||
"missingRows": 0, "invalidRows": 0, "negativeRows": 0}]
|
||||
else:
|
||||
by_month = {
|
||||
1: [
|
||||
{"key": "责废尺寸重量", "label": "责废尺寸重量", "category": "责废", "weight": 10,
|
||||
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
{"key": "料废尺寸重量", "label": "料废尺寸重量", "category": "料废", "weight": 4,
|
||||
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
{"key": "责废缺角重量", "label": "责废缺角重量", "category": "责废", "weight": 2,
|
||||
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
],
|
||||
3: [
|
||||
{"key": "责废尺寸重量", "label": "责废尺寸重量", "category": "责废", "weight": 13,
|
||||
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
{"key": "料废尺寸重量", "label": "料废尺寸重量", "category": "料废", "weight": 5,
|
||||
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
{"key": "责废刀丝重量", "label": "责废刀丝重量", "category": "责废", "weight": 1,
|
||||
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
|
||||
],
|
||||
}
|
||||
rows = by_month.get(month, by_month[1])
|
||||
if payload.get("defect") and payload.get("dimension") != "product":
|
||||
rows = [row for row in rows if row["key"] == payload["defect"]]
|
||||
if payload.get("category"):
|
||||
rows = [row for row in rows if row["category"] == payload["category"]]
|
||||
context = {key: payload.get(key) for key in ("year", "month", "quarter", "monthStart", "monthEnd", "department", "group", "product",
|
||||
"projectGroup", "includeUnassigned", "stage", "dimension", "defect", "category")}
|
||||
context.update(extraFilters=payload.get("extraFilters", {}),
|
||||
exclusions=payload.get("exclusions", {}),
|
||||
columnFilters=payload.get("columnFilters", []),
|
||||
filters={"columnFilters": payload.get("columnFilters", [])})
|
||||
return {"kind": "fine_defect_top", "releaseId": payload["releaseId"],
|
||||
"release": {"id": payload["releaseId"], "status": "PUBLISHED"},
|
||||
"context": {**context, "mode": "top"}, "grain": "source_defect_weight",
|
||||
"rows": rows[:payload["limit"]], "totalGroups": len(rows), "rowsTruncated": len(rows) > payload["limit"],
|
||||
"sourceRows": 8,
|
||||
"coverage": {"available": True, "sourceRows": 8, "hasValidWeights": True,
|
||||
"categories": [{"category": "责废", "isReconciled": True}]},
|
||||
"categoryReconciliation": [{"category": "责废", "isReconciled": True}],
|
||||
"sourceFields": [{"key": row["key"], "label": row["label"], "category": row["category"],
|
||||
"sourceField": row["key"]} for row in rows],
|
||||
"notice": "来源字段聚合,不跨字段合并。"}
|
||||
raise AssertionError(f"Unexpected API path: {path}")
|
||||
|
||||
|
||||
class QualityToolsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.api = FakeApi()
|
||||
self.tools = QualityTools(api_factory=lambda: self.api, evidence_dir=self.temp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def test_schema_has_one_tool_and_modes(self):
|
||||
definition = tool_definition()
|
||||
self.assertEqual(definition["name"], "quality_data")
|
||||
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
|
||||
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
|
||||
self.assertIn("defect", definition["inputSchema"]["properties"])
|
||||
self.assertFalse(definition["inputSchema"]["additionalProperties"])
|
||||
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
|
||||
|
||||
def test_scope_wire_encoding_and_filter_conflict(self):
|
||||
filters, scope = normalize_filters("auto_combined", {}, scope_was_explicit=True)
|
||||
self.assertEqual(filters["department"], '新能源汽车磁钢')
|
||||
self.assertNotIn('group', filters)
|
||||
self.assertEqual(scope, "auto_combined")
|
||||
with self.assertRaisesRegex(QualityToolError, "不能同时指定"):
|
||||
normalize_filters("auto_domestic", {"group": "汽车交通-国内"}, scope_was_explicit=True)
|
||||
explicit_group, _ = normalize_filters(None, {"group": "研发组"}, scope_was_explicit=False)
|
||||
self.assertEqual(explicit_group["group"], "研发组")
|
||||
|
||||
def test_exclusions_reach_metrics_and_details_and_are_checked(self):
|
||||
exclusions = {"product": ["规格+T", "__UNASSIGNED__"]}
|
||||
filters = {"exclusions": exclusions}
|
||||
result = self.tools.quality_data({"mode": "compare", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "comparisonPeriod": {"year": 2026, "month": 1}, "filters": filters})
|
||||
metric_calls = [parse_qs(urlsplit(path).query) for _, path, _ in self.api.calls if urlsplit(path).path == "/api/metrics"]
|
||||
self.assertEqual(2, len(metric_calls))
|
||||
self.assertTrue(all(json.loads(call["exclusions"][0]) == exclusions for call in metric_calls))
|
||||
self.tools.quality_data({"mode": "details", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}, "stage": "black", "filters": filters})
|
||||
detail = next(payload for _, path, payload in self.api.calls if path == "/api/analysis/tools/details")
|
||||
self.assertEqual(exclusions, detail["exclusions"])
|
||||
snapshot = snapshot_period(2026, 3, "overall", {"releaseId": RELEASE})
|
||||
with self.assertRaisesRegex(QualityToolError, "未确认排除"):
|
||||
self.tools._validate_source_snapshot(snapshot, RELEASE, {"year": 2026, "month": 3}, {**filters, "includeUnassigned": False}, "overall")
|
||||
|
||||
def test_details_rejects_missing_exclusion_echo(self):
|
||||
class MissingExclusionEchoApi(FakeApi):
|
||||
def call(self, path, payload=None, method=None):
|
||||
response = super().call(path, payload, method)
|
||||
if path == "/api/analysis/tools/details":
|
||||
response["range"].pop("exclusions", None)
|
||||
return response
|
||||
api = MissingExclusionEchoApi()
|
||||
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
|
||||
with self.assertRaisesRegex(QualityToolError, "未确认排除"):
|
||||
tools.quality_data({"mode": "details", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
|
||||
"stage": "black", "filters": {"exclusions": {"product": ["P"]}}})
|
||||
|
||||
def test_exclusions_are_validated_unsupported_modes_reject_and_defects_forwards_them(self):
|
||||
for exclusions in ({"product": []}, {"unknown": ["x"]}, {"product": [None]}, None):
|
||||
with self.subTest(exclusions=exclusions), self.assertRaises(QualityToolError):
|
||||
normalize_filters(None, {"exclusions": exclusions}, scope_was_explicit=False)
|
||||
with self.assertRaisesRegex(QualityToolError, "暂不支持排除条件"):
|
||||
self.tools.quality_data({"mode": "top", "releaseId": RELEASE, "period": {"year": 2026},
|
||||
"filters": {"exclusions": {"product": ["P"]}}, "stage": "black", "dimension": "product"})
|
||||
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "filters": {"exclusions": {"product": ["P"]}},
|
||||
"stage": "black", "dimension": "defect", "limit": 10})
|
||||
request = next(payload for method, path, payload in reversed(self.api.calls)
|
||||
if method == "POST" and urlsplit(path).path == "/api/analysis/defects")
|
||||
self.assertEqual(request["exclusions"], {"product": ["P"]})
|
||||
self.assertEqual(result["filters"]["exclusions"], {"product": ["P"]})
|
||||
self.assertFalse(any(urlsplit(path).path == "/api/analytics/top" for _, path, _ in self.api.calls))
|
||||
|
||||
def test_symmetric_decomposition_is_exact_in_percentage_points(self):
|
||||
result = symmetric_stage_decomposition(
|
||||
0.96, 0.98, 0.94, 0.96, 0.96 * 0.98 - 0.94 * 0.96,
|
||||
black_current_total_weight=100, bright_current_total_weight=90,
|
||||
black_comparison_total_weight=80, bright_comparison_total_weight=70)
|
||||
self.assertEqual(result["status"], "available")
|
||||
self.assertAlmostEqual(float(result["blackPP"] + result["brightPP"]),
|
||||
100 * (0.96 * 0.98 - 0.94 * 0.96), places=10)
|
||||
|
||||
def test_missing_or_invalid_stage_rate_blocks_decomposition(self):
|
||||
for invalid in (None, "bad", -0.1, 1.01):
|
||||
result = symmetric_stage_decomposition(invalid, 0.98, 0.94, 0.96, 0.02,
|
||||
black_current_total_weight=100, bright_current_total_weight=100,
|
||||
black_comparison_total_weight=100, bright_comparison_total_weight=100)
|
||||
self.assertEqual(result["status"], "unavailable")
|
||||
self.assertIsNone(result["blackPP"])
|
||||
|
||||
def test_symmetric_decomposition_refuses_zero_stage_denominator(self):
|
||||
result = symmetric_stage_decomposition(
|
||||
0.0, 0.98, 0.94, 0.96, 0.02,
|
||||
black_current_total_weight=0, bright_current_total_weight=100,
|
||||
black_comparison_total_weight=100, bright_comparison_total_weight=100)
|
||||
self.assertEqual(result["status"], "unavailable")
|
||||
self.assertIsNone(result["blackPP"])
|
||||
|
||||
def test_compare_pins_same_release_and_reuses_filters(self):
|
||||
result = self.tools.quality_data({
|
||||
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
"filters": {"department": "汽车交通", "group": "汽车交通-国内", "product": "规格 A",
|
||||
"projectGroup": "项目甲", "includeUnassigned": False},
|
||||
})
|
||||
self.assertEqual(result["releaseId"], RELEASE)
|
||||
self.assertTrue(result["delta"]["available"])
|
||||
self.assertEqual(result["stageDecomposition"]["status"], "available")
|
||||
self.assertNotIn("blackPP", result["stageDecomposition"])
|
||||
self.assertAlmostEqual(result["stageRateChangesPP"]["black"], 2.0)
|
||||
self.assertAlmostEqual(result["stageRateChangesPP"]["bright"], 2.0)
|
||||
self.assertNotAlmostEqual(result["stageDecomposition"]["blackContributionToCompositePP"], 2.0)
|
||||
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
|
||||
self.assertEqual(len(metric_calls), 2)
|
||||
for _, path, _ in metric_calls:
|
||||
query = parse_qs(urlsplit(path).query)
|
||||
self.assertEqual(query["releaseId"], [RELEASE])
|
||||
self.assertEqual(query["department"], ["新能源汽车磁钢"])
|
||||
self.assertEqual(query["group"], ["汽车交通-国内"])
|
||||
self.assertEqual(query["product"], ["规格 A"])
|
||||
self.assertEqual(query["projectGroup"], ["项目甲"])
|
||||
self.assertTrue(result["evidencePath"].endswith(result["evidenceId"] + ".json"))
|
||||
self.assertRegex(result["evidenceRef"], r"^Q[0-9A-F]{8}$")
|
||||
evidence = json.loads(Path(result["evidencePath"]).read_text())
|
||||
self.assertEqual(evidence["evidenceRef"], result["evidenceRef"])
|
||||
|
||||
def test_compare_single_stage_coverage_keeps_composite_but_not_missing_stage_delta(self):
|
||||
self.api.bad_coverage_month = 3
|
||||
result = self.tools.quality_data({
|
||||
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
})
|
||||
self.assertTrue(result["delta"]["available"])
|
||||
self.assertIsNotNone(result["delta"]["percentagePoints"])
|
||||
self.assertIsNone(result["stageRateChangesPP"]["bright"])
|
||||
self.assertEqual(result["stageDecomposition"]["status"], "unavailable")
|
||||
|
||||
def test_compare_zero_stage_denominator_uses_backend_composite_without_stage_fallback(self):
|
||||
self.api.zero_denominator_month = 3
|
||||
result = self.tools.quality_data({
|
||||
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
})
|
||||
self.assertTrue(result["delta"]["available"])
|
||||
self.assertIsNotNone(result["delta"]["percentagePoints"])
|
||||
self.assertIsNone(result["stageRateChangesPP"]["bright"])
|
||||
self.assertEqual(result["stageDecomposition"]["status"], "unavailable")
|
||||
|
||||
def test_data_modes_require_release_and_catalog_refreshes_current(self):
|
||||
with self.assertRaisesRegex(QualityToolError, "显式传回"):
|
||||
self.tools.quality_data({"mode": "query", "period": {"year": 2026, "month": 3}})
|
||||
first = self.tools.quality_data({"mode": "catalog"})
|
||||
self.api.current_release_id = "release-b"
|
||||
second = self.tools.quality_data({"mode": "catalog"})
|
||||
self.assertEqual(first["releaseId"], RELEASE)
|
||||
self.assertEqual(second["releaseId"], "release-b")
|
||||
explicit = self.tools.quality_data({"mode": "query", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 1}})
|
||||
self.assertEqual(explicit["releaseId"], RELEASE)
|
||||
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
|
||||
ids = [parse_qs(urlsplit(call[1]).query)["releaseId"][0] for call in metric_calls]
|
||||
self.assertEqual(ids, [RELEASE, "release-b", RELEASE])
|
||||
|
||||
def test_scope_is_returned_and_metrics_cache_does_not_cross_calls(self):
|
||||
args = {"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
|
||||
"scope": "auto_combined"}
|
||||
first = self.tools.quality_data(args)
|
||||
second = self.tools.quality_data(args)
|
||||
expected = encoded_selection(["汽车交通-国内", "汽车交通-海外"])
|
||||
self.assertEqual(first["selectedScope"], "auto_combined")
|
||||
self.assertEqual(first["filters"]["department"], '新能源汽车磁钢')
|
||||
self.assertEqual(second["selectedScope"], "auto_combined")
|
||||
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
|
||||
self.assertEqual(len(metric_calls), 2)
|
||||
|
||||
def test_candidate_release_is_rejected(self):
|
||||
with self.assertRaisesRegex(QualityToolError, "不是已发布版本"):
|
||||
self.tools.quality_data({"mode": "query", "releaseId": "candidate", "period": {"year": 2026, "month": 1}})
|
||||
|
||||
def test_top_product_category_preserves_basis_and_coverage(self):
|
||||
result = self.tools.quality_data({"mode": "top", "period": {"year": 2026, "month": 3},
|
||||
"releaseId": RELEASE, "stage": "black", "dimension": "product", "category": "黑片责废"})
|
||||
self.assertEqual(result["category"], "责废")
|
||||
self.assertIn("不表示总体贡献", result["shareBasis"])
|
||||
self.assertIn("coverage", result)
|
||||
self.assertIn("亮片类别份额不必等同阶段正式报废重量", result["meaning"])
|
||||
call = next(item for item in self.api.calls if urlsplit(item[1]).path == "/api/analytics/top")
|
||||
self.assertEqual(parse_qs(urlsplit(call[1]).query)["category"], ["责废"])
|
||||
|
||||
def test_catalog_describes_fine_defect_endpoint_without_calling_it(self):
|
||||
result = self.tools.quality_data({"mode": "catalog"})
|
||||
capability = result["fineDefects"]
|
||||
self.assertEqual(capability["endpoint"], "POST /api/analysis/defects")
|
||||
self.assertIn("defect", capability["dimensions"])
|
||||
self.assertFalse(capability["scanned"])
|
||||
self.assertEqual(capability["fields"][0]["key"], "责废尺寸重量")
|
||||
catalog_call = next(call for call in self.api.calls if urlsplit(call[1]).path == "/api/analysis/defects")
|
||||
self.assertEqual(catalog_call[2]["mode"], "catalog")
|
||||
|
||||
def test_defects_query_preserves_source_field_keys_and_filters(self):
|
||||
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "scope": "auto_domestic",
|
||||
"filters": {"department": "汽车交通", "product": "规格 A"},
|
||||
"stage": "black", "dimension": "defect", "category": "黑片责废",
|
||||
"limit": 2})
|
||||
self.assertEqual(result["grain"], "source_defect_weight")
|
||||
self.assertEqual(result["category"], "责废")
|
||||
self.assertEqual(result["selectedScope"], "auto_domestic")
|
||||
self.assertEqual(result["rows"][0]["key"], "责废尺寸重量")
|
||||
self.assertEqual(result["rowsReturned"], 2)
|
||||
self.assertIn("source_defect_weight", result["grain"])
|
||||
call = next(item for item in self.api.calls if urlsplit(item[1]).path == "/api/analysis/defects")
|
||||
self.assertEqual(call[0], "POST")
|
||||
self.assertEqual(call[2]["releaseId"], RELEASE)
|
||||
self.assertEqual(call[2]["group"], "汽车交通-国内")
|
||||
self.assertEqual(call[2]["department"], "新能源汽车磁钢")
|
||||
self.assertEqual(call[2]["category"], "责废")
|
||||
|
||||
def test_defects_accepts_java_normalized_scope_sets_but_rejects_added_values(self):
|
||||
class NormalizedScopeApi(FakeApi):
|
||||
def call(inner, path, payload=None, method=None):
|
||||
response = super().call(path, payload, method)
|
||||
if urlsplit(path).path == "/api/analysis/defects" and payload.get("mode") == "top":
|
||||
for key in ("department", "group", "product", "projectGroup"):
|
||||
value = response["context"].get(key)
|
||||
response["context"][key] = [] if value is None else [value]
|
||||
return response
|
||||
|
||||
api = NormalizedScopeApi()
|
||||
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
|
||||
# Java echoes the normalized empty sets for unset scope filters.
|
||||
empty_result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "stage": "black", "dimension": "defect"})
|
||||
self.assertTrue(empty_result["coverage"]["available"])
|
||||
# A selected scalar and its singleton collection have the same scope.
|
||||
selected_result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"filters": {"department": "部门A", "group": "组A", "product": "规格A",
|
||||
"projectGroup": "项目A"},
|
||||
"stage": "black", "dimension": "defect"})
|
||||
self.assertTrue(selected_result["coverage"]["available"])
|
||||
|
||||
original_call = api.call
|
||||
def added_scope_value(path, payload=None, method=None):
|
||||
response = original_call(path, payload, method)
|
||||
if urlsplit(path).path == "/api/analysis/defects" and payload.get("mode") == "top":
|
||||
response["context"]["group"] = [payload["group"], "未请求组"]
|
||||
return response
|
||||
api.call = added_scope_value
|
||||
with self.assertRaisesRegex(QualityToolError, "未确认 group"):
|
||||
tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "filters": {"group": "组A"},
|
||||
"stage": "black", "dimension": "defect"})
|
||||
|
||||
def test_defects_preserves_month_range_exclusions_and_authorized_column_filters(self):
|
||||
class AuthorizedColumnApi(FakeApi):
|
||||
def call(inner, path, payload=None, method=None):
|
||||
response = super().call(path, payload, method)
|
||||
if path == "/api/analysis/tools/catalog":
|
||||
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
|
||||
"type": "source_value", "stages": ["black"],
|
||||
"columns": {"black": "customer_code"}, "operators": ["in"]}]
|
||||
return response
|
||||
api = AuthorizedColumnApi()
|
||||
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
|
||||
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]}
|
||||
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
|
||||
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
|
||||
"stage": "black", "dimension": "defect", "limit": 100})
|
||||
request = next(payload for method, path, payload in reversed(api.calls)
|
||||
if method == "POST" and urlsplit(path).path == "/api/analysis/defects")
|
||||
self.assertEqual((request["monthStart"], request["monthEnd"]), (3, 9))
|
||||
self.assertEqual(request["exclusions"], {"product": ["规格甲"]})
|
||||
self.assertEqual(request["columnFilters"], [rule])
|
||||
self.assertEqual(result["period"], {"year": 2026, "monthStart": 3, "monthEnd": 9})
|
||||
self.assertEqual(result["filters"]["columnFilters"], [rule])
|
||||
|
||||
def test_reported_stage_defect_calls_default_missing_dimension_without_changing_scope(self):
|
||||
for stage in ("black", "bright"):
|
||||
with self.subTest(stage=stage):
|
||||
arguments = {"mode": "defects", "releaseId": RELEASE, "stage": stage,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
"filters": {"product": "规格 A"}, "limit": 20}
|
||||
result = self.tools.quality_data(arguments)
|
||||
self.assertEqual(result["dimension"], "defect")
|
||||
self.assertEqual(result["stage"], stage)
|
||||
self.assertEqual(result["filters"]["product"], "规格 A")
|
||||
self.assertEqual(result["period"], arguments["period"])
|
||||
self.assertEqual(result["comparisonPeriod"], arguments["comparisonPeriod"])
|
||||
self.assertNotIn("dimension", arguments)
|
||||
|
||||
def test_missing_defect_dimension_is_not_guessed_for_specific_field_queries(self):
|
||||
with self.assertRaisesRegex(QualityToolError, "dimension"):
|
||||
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"stage": "black", "defect": "责废尺寸重量"})
|
||||
|
||||
def test_defects_compare_full_outer_pairs_without_zero_filling(self):
|
||||
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
"stage": "black", "dimension": "defect", "limit": 10})
|
||||
rows = {row["key"]: row for row in result["pairedRows"]}
|
||||
self.assertEqual(result["pairedTotalRows"], 4)
|
||||
self.assertAlmostEqual(rows["责废尺寸重量"]["deltaWeight"], 3)
|
||||
self.assertEqual(rows["责废刀丝重量"]["changeType"], "new")
|
||||
self.assertFalse(rows["责废刀丝重量"]["deltaAvailable"])
|
||||
self.assertIsNone(rows["责废刀丝重量"]["deltaWeight"])
|
||||
self.assertEqual(rows["责废缺角重量"]["changeType"], "exit")
|
||||
self.assertIsNone(rows["责废缺角重量"]["deltaWeight"])
|
||||
calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/analysis/defects"]
|
||||
self.assertEqual([call[2]["limit"] for call in calls], [100, 100])
|
||||
|
||||
def test_single_defect_top_fetches_bounded_candidates_before_presentation_limit(self):
|
||||
result=self.tools.quality_data({'mode':'defects','releaseId':RELEASE,'period':{'year':2026,'month':3},
|
||||
'stage':'black','dimension':'defect','limit':1})
|
||||
calls=[call for call in self.api.calls if urlsplit(call[1]).path=='/api/analysis/defects']
|
||||
self.assertEqual(calls[-1][2]['limit'],100)
|
||||
self.assertEqual(len(result['rows']),1);self.assertEqual(result['totalRows'],3)
|
||||
self.assertTrue(result['rankingComplete']);self.assertTrue(result['truncated'])
|
||||
self.assertEqual(result['rows'][0]['weight'],13)
|
||||
|
||||
def test_defects_product_ranking_requires_concrete_defect(self):
|
||||
with self.assertRaisesRegex(QualityToolError, "必须指定 defect"):
|
||||
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"stage": "black", "dimension": "product"})
|
||||
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"stage": "black", "dimension": "product",
|
||||
"defect": "责废尺寸重量"})
|
||||
self.assertEqual(result["rows"][0]["label"], "规格 A")
|
||||
self.assertEqual(result["defect"], "责废尺寸重量")
|
||||
|
||||
def test_defect_delta_ranking_precedes_limit_and_keeps_negative_order(self):
|
||||
def snapshot(values):
|
||||
return {"available": True, "rowsTruncated": False,
|
||||
"rows": [{"key": key, "weight": weight} for key, weight in values.items()]}
|
||||
current = snapshot({"a": 8, "b": 4, "龘": 30, "missing": None})
|
||||
previous = snapshot({"a": 10, "b": 5, "龘": 10, "missing": 2})
|
||||
paired = QualityTools._pair_defect_snapshots(current, previous, "defect", "scrap_delta")
|
||||
self.assertEqual([row["key"] for row in paired["rows"]], ["龘", "b", "a", "missing"])
|
||||
self.assertEqual(paired["rows"][0]["deltaWeight"], 20)
|
||||
|
||||
def test_defects_rejects_duplicate_keys_or_unconfirmed_context(self):
|
||||
original_call = self.api.call
|
||||
|
||||
def duplicate_call(path, payload=None, method=None):
|
||||
result = original_call(path, payload, method)
|
||||
if urlsplit(path).path == "/api/analysis/defects":
|
||||
result["rows"].append(dict(result["rows"][0]))
|
||||
return result
|
||||
|
||||
self.api.call = duplicate_call
|
||||
with self.assertRaisesRegex(QualityToolError, "重复 key"):
|
||||
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3},
|
||||
"stage": "black", "dimension": "defect"})
|
||||
|
||||
self.api.call = original_call
|
||||
def wrong_context(path, payload=None, method=None):
|
||||
result = original_call(path, payload, method)
|
||||
if urlsplit(path).path == "/api/analysis/defects":
|
||||
result["context"]["product"] = "另一个规格"
|
||||
return result
|
||||
self.api.call = wrong_context
|
||||
with self.assertRaisesRegex(QualityToolError, "未确认 product"):
|
||||
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "month": 3}, "filters": {"product": "规格 A"},
|
||||
"stage": "black", "dimension": "defect"})
|
||||
|
||||
def test_full_outer_change_marks_new_and_exit_and_does_not_fill_rates(self):
|
||||
self.api.products = {
|
||||
1: [FakeApi._product_row("a", "规格 A", 10, 5), FakeApi._product_row("b", "规格 B", 4, 2)],
|
||||
3: [FakeApi._product_row("a", "规格 A", 13, 4), FakeApi._product_row("c", "规格 C", 2, 1)],
|
||||
}
|
||||
result = self.tools.quality_data({"mode": "changes", "period": {"year": 2026, "month": 3},
|
||||
"releaseId": RELEASE,
|
||||
"comparisonPeriod": {"year": 2026, "month": 1},
|
||||
"dimension": "product", "limit": 10})
|
||||
by_key = {row["key"]: row for row in result["rows"]}
|
||||
self.assertEqual(result["totalRows"], 3)
|
||||
self.assertEqual(by_key["c"]["changeType"], "new")
|
||||
self.assertEqual(by_key["c"]["comparison"]["scrapWeight"], 0)
|
||||
self.assertIsNone(by_key["c"]["comparison"]["rate"])
|
||||
self.assertEqual(by_key["b"]["changeType"], "exit")
|
||||
self.assertEqual(by_key["b"]["current"]["scrapWeight"], 0)
|
||||
self.assertIsNone(by_key["b"]["current"]["rate"])
|
||||
self.assertIn("black + bright 两阶段", result["weightBasis"])
|
||||
self.assertNotIn("totalWeight", by_key["c"]["current"])
|
||||
self.assertIn("blackInputWeight", by_key["c"]["current"])
|
||||
self.assertIn("brightInputWeight", by_key["c"]["current"])
|
||||
self.assertNotIn("crossStageWeightSum", by_key["c"]["current"])
|
||||
self.assertNotIn("totalWeightDelta", by_key["c"])
|
||||
self.assertEqual([row["rank"] for row in result["rows"]], [1, 2, 3])
|
||||
self.assertIn("不等于独立投入", by_key["c"]["volumeNote"])
|
||||
self.assertIn("正式 compositeRate", result["rateBasis"])
|
||||
signals = result["prioritySignals"]
|
||||
self.assertEqual(signals["scrapIncrease"][0]["label"], "规格 C")
|
||||
self.assertIn("blackInputWeight", signals["scrapIncrease"][0]["current"])
|
||||
self.assertIn("brightInputWeight", signals["scrapIncrease"][0]["comparison"])
|
||||
self.assertIn("不是统一贡献分数", signals["basis"])
|
||||
|
||||
def test_duplicate_and_missing_row_keys_are_rejected(self):
|
||||
with self.assertRaisesRegex(QualityToolError, "重复 key"):
|
||||
QualityTools._index_rows({"rows": [{"key": "x"}, {"key": "x"}]}, "product", "current")
|
||||
with self.assertRaisesRegex(QualityToolError, "空 key"):
|
||||
QualityTools._index_rows({"rows": [{"key": None}]}, "product", "current")
|
||||
|
||||
def test_category_absence_is_null_and_present_row_missing_stage_is_null(self):
|
||||
row = {"key": "x", "black": {"scrapWeight": None, "defects": []}, "bright": {"scrapWeight": 5}}
|
||||
self.assertIsNone(QualityTools._row_scrap(row, "black", "责废"))
|
||||
self.assertIsNone(QualityTools._row_scrap({"key": "x", "black": None, "bright": {"scrapWeight": 5}}, None, None))
|
||||
self.assertEqual(QualityTools._row_scrap(None, None, None), 0)
|
||||
|
||||
def test_changes_rate_requires_row_level_stage_coverage(self):
|
||||
row = {"compositeRate": 0.95,
|
||||
"black": {"rate": 0.96, "totalWeight": 4, "qualifiedWeight": 3},
|
||||
"bright": {"rate": 0.99, "totalWeight": 0, "qualifiedWeight": 0},
|
||||
"coverage": {"blackRows": 3, "brightRows": 0}}
|
||||
self.assertIsNone(QualityTools._row_rate(row, None))
|
||||
self.assertAlmostEqual(float(QualityTools._row_rate(row, "black")), 0.96)
|
||||
self.assertIsNone(QualityTools._row_rate(row, "bright"))
|
||||
self.assertIsNone(QualityTools._row_rate({**row, "coverage": None}, "black"))
|
||||
positive = {**row, "bright": {"rate": 0.99, "totalWeight": 5, "qualifiedWeight": 4},
|
||||
"coverage": {"blackRows": 3, "brightRows": 2}}
|
||||
self.assertAlmostEqual(float(QualityTools._row_rate(positive, None)), 0.95)
|
||||
|
||||
def test_details_always_labels_bounded_sample(self):
|
||||
result = self.tools.quality_data({"mode": "details", "period": {"year": 2026, "month": 3},
|
||||
"releaseId": RELEASE,
|
||||
"stage": "black", "limit": 5})
|
||||
self.assertTrue(result["sampled"])
|
||||
self.assertTrue(result["truncated"])
|
||||
self.assertIn("不能当总体排名", result["notice"])
|
||||
|
||||
def test_catalog_returns_concise_periods_and_release_binding(self):
|
||||
result = self.tools.quality_data({"mode": "catalog"})
|
||||
self.assertEqual(result["releaseId"], RELEASE)
|
||||
self.assertEqual(result["periods"]["years"], [2025, 2026])
|
||||
self.assertEqual(result["periods"]["availableMonthsInLatestYear"], list(range(1, 13)))
|
||||
|
||||
def test_empty_coverage_is_an_error_not_success(self):
|
||||
original_call = self.api.call
|
||||
|
||||
def empty_call(path, payload=None, method=None):
|
||||
parsed = urlsplit(path)
|
||||
if parsed.path == "/api/metrics":
|
||||
result = original_call(path, payload, method)
|
||||
result["coverage"]["rawRows"] = 0
|
||||
result["coverage"]["includedRows"] = 0
|
||||
return result
|
||||
return original_call(path, payload, method)
|
||||
|
||||
self.api.call = empty_call
|
||||
with self.assertRaisesRegex(QualityToolError, "2026年3月.*没有可用的已发布来源数据"):
|
||||
self.tools.quality_data({"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}})
|
||||
|
||||
def test_custom_formula_metadata_survives_query_and_both_comparison_sides(self):
|
||||
original=self.api.call
|
||||
rules={'combined':'configured_custom_formula','supportsStageContribution':False}
|
||||
descriptions={'rate':'正式自定义综合率'}
|
||||
def with_rules(path,payload=None,method=None):
|
||||
result=original(path,payload,method)
|
||||
if urlsplit(path).path == '/api/metrics':
|
||||
result['context'].update(calculationRules=rules,metricFormulaDescriptions=descriptions)
|
||||
return result
|
||||
self.api.call=with_rules
|
||||
queried=self.tools.quality_data({'mode':'query','releaseId':RELEASE,'period':{'year':2026,'month':3}})
|
||||
self.assertEqual(queried['calculationRules'],rules)
|
||||
compared=self.tools.quality_data({'mode':'compare','releaseId':RELEASE,
|
||||
'period':{'year':2026,'month':3},'comparisonPeriod':{'year':2026,'month':2}})
|
||||
for side in ('current','comparison'):
|
||||
self.assertEqual(compared[side]['calculationRules'],rules)
|
||||
self.assertEqual(compared[side]['metricFormulaDescriptions'],descriptions)
|
||||
|
||||
def test_mcp_json_rpc_initialize_list_ping_and_notification(self):
|
||||
server = MCPServer(tools=self.tools)
|
||||
initialized = server.handle({"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {"protocolVersion": "2024-11-05"}})
|
||||
self.assertEqual(initialized["result"]["serverInfo"]["name"], "quality")
|
||||
listed = server.handle({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
|
||||
self.assertEqual([tool["name"] for tool in listed["result"]["tools"]], ["quality_data"])
|
||||
self.assertEqual(server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"}), None)
|
||||
self.assertEqual(server.handle({"jsonrpc": "2.0", "id": 3, "method": "ping"})["result"], {})
|
||||
|
||||
def test_mcp_initialize_negotiates_newer_client_protocol(self):
|
||||
server = MCPServer(tools=self.tools)
|
||||
response = server.handle({"jsonrpc": "2.0", "id": "new-client", "method": "initialize",
|
||||
"params": {"protocolVersion": "2025-11-25"}})
|
||||
self.assertEqual(response["result"]["protocolVersion"], "2025-03-26")
|
||||
|
||||
def test_mcp_tools_call_and_newline_stdio(self):
|
||||
server = MCPServer(tools=self.tools, logger=logging.getLogger("test.nao_quality_mcp"))
|
||||
call = server.handle({"jsonrpc": "2.0", "id": 8, "method": "tools/call",
|
||||
"params": {"name": "quality_data", "arguments": {
|
||||
"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}}}})
|
||||
self.assertEqual(call["result"]["structuredContent"]["releaseId"], RELEASE)
|
||||
self.assertFalse(call["result"]["isError"])
|
||||
|
||||
request_lines = [
|
||||
{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2024-11-05"}},
|
||||
{"jsonrpc": "2.0", "method": "notifications/initialized"},
|
||||
{"jsonrpc": "2.0", "id": 2, "method": "tools/list"},
|
||||
{"jsonrpc": "2.0", "id": 3, "method": "ping"},
|
||||
]
|
||||
source = io.StringIO("\n".join(json.dumps(item) for item in request_lines) + "\n")
|
||||
output = io.StringIO()
|
||||
from unittest.mock import patch
|
||||
with patch("nao_quality_mcp._configure_logger", return_value=logging.getLogger("test.nao_quality_mcp")):
|
||||
serve(source, output, tools=self.tools)
|
||||
responses = [json.loads(line) for line in output.getvalue().splitlines()]
|
||||
self.assertEqual([item["id"] for item in responses], [1, 2, 3])
|
||||
self.assertEqual(responses[1]["result"]["tools"][0]["name"], "quality_data")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,133 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_agent_task import TaskUnavailable, normalize_task
|
||||
from nao_query_plan import normalize_query_plan, scope_documents
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
|
||||
def ranking_to_trends_plan():
|
||||
return [
|
||||
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 5,
|
||||
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
|
||||
'visualType': 'pareto', 'visualPreference': 'auto'},
|
||||
{'nodeId': 'top_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
|
||||
'stages': ['black'], 'selector': {'kind': 'ranking_top', 'sourceNodeId': 'rank',
|
||||
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
|
||||
'visualType': 'line', 'visualPreference': 'auto'},
|
||||
]
|
||||
|
||||
|
||||
class QueryPlanNormalizationTests(unittest.TestCase):
|
||||
def normalize(self, value, context=None):
|
||||
return normalize_task(value, '按黑片投入重量排名后查看前两项报废率趋势',
|
||||
{'_protocol': {'version': 1}, **(context or {})})
|
||||
|
||||
def test_rank_then_top_trend_preserves_root_union_and_filters(self):
|
||||
value = task_value(kind='analysis', queryPlan=ranking_to_trends_plan(),
|
||||
dimensions=['product'], dimensionStages={'product': ['black']},
|
||||
stages=['black'], fields=['scrapRate', 'totalWeight'],
|
||||
trendRequired=True, comparisonRequired=False, period={'year': 2026},
|
||||
filterUpdates={'department': '部门甲'})
|
||||
root = self.normalize(value)
|
||||
self.assertEqual(root['filters'], {'department': '部门甲', 'columnFilters': []})
|
||||
self.assertEqual(root['dimensions'], ['product'])
|
||||
self.assertEqual(root['fields'], ['scrapRate', 'totalWeight'])
|
||||
self.assertEqual(root['stages'], ['black'])
|
||||
self.assertTrue(root['trendRequired'])
|
||||
self.assertEqual(root['periodCoverage'], 'available')
|
||||
self.assertEqual(root['comparisonRequired'], False)
|
||||
self.assertEqual(root['dimensionStages'], {'product': ['black']})
|
||||
self.assertEqual(root['queryPlan'][1]['selector']['count'], 2)
|
||||
self.assertNotIn('period', root['queryPlan'][1])
|
||||
self.assertNotIn('filters', root['queryPlan'][1])
|
||||
|
||||
def test_true_root_trend_and_existing_comparison_cannot_be_silently_dropped(self):
|
||||
value = task_value(kind='metrics', queryPlan=[
|
||||
{'nodeId': 'one', 'purpose': 'metrics', 'fields': ['rate'], 'stages': ['black'],
|
||||
'selector': {'kind': 'base'}, 'visualPreference': 'none'}],
|
||||
dimensions=[], dimensionStages={}, stages=['black'], fields=['rate'],
|
||||
trendRequired=True, period={'year': 2026, 'month': 9})
|
||||
with self.assertRaisesRegex(TaskUnavailable, 'trendRequired'):
|
||||
self.normalize(value)
|
||||
value['trendRequired'] = False
|
||||
value['comparisonPeriod'] = {'year': 2026, 'month': 8}
|
||||
with self.assertRaisesRegex(TaskUnavailable, 'comparisonPeriod'):
|
||||
self.normalize(value)
|
||||
|
||||
def test_two_month_root_period_is_valid_for_explicit_line_trend(self):
|
||||
node = {'nodeId': 'short_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
|
||||
'stages': ['black'], 'selector': {'kind': 'base'},
|
||||
'visualType': 'line', 'visualPreference': 'auto'}
|
||||
value = task_value(kind='metrics', queryPlan=[node], dimensions=[],
|
||||
dimensionStages={}, stages=['black'], fields=['scrapRate'],
|
||||
trendRequired=True, comparisonRequired=False,
|
||||
period={'year': 2026, 'monthStart': 7, 'monthEnd': 8})
|
||||
root = self.normalize(value)
|
||||
self.assertEqual(root['queryPlan'][0]['visualType'], 'line')
|
||||
self.assertEqual(root['period']['monthStart'], 7)
|
||||
self.assertEqual(root['period']['monthEnd'], 8)
|
||||
|
||||
def test_previous_ranking_requires_trusted_preview_and_does_not_restore_old_filters(self):
|
||||
artifact = {'queryId': 'trusted-query', 'dimension': 'product', 'stage': 'black',
|
||||
'metric': 'totalWeight', 'rows': [{'rank': 1, 'name': '旧规格A'},
|
||||
{'rank': 2, 'name': '旧规格B'}],
|
||||
'period': {'year': 2025}, 'filters': {'department': '旧部门'}}
|
||||
context = {'_protocol': {'version': 1, 'availableRankings': [artifact]}}
|
||||
node = {'nodeId': 'old_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
|
||||
'stages': ['black'], 'selector': {'kind': 'previous_ranking',
|
||||
'queryId': 'trusted-query', 'stage': 'black', 'metric': 'totalWeight', 'ranks': [1, 2]},
|
||||
'visualPreference': 'auto', 'visualType': 'line'}
|
||||
value = task_value(kind='metrics', queryPlan=[node], dimensions=['product'],
|
||||
dimensionStages={}, stages=['black'], fields=['scrapRate'], trendRequired=True,
|
||||
comparisonRequired=False, period={'year': 2026}, filterUpdates={'department': '本轮部门'})
|
||||
root = self.normalize(value, context)
|
||||
self.assertEqual(root['filters'], {'department': '本轮部门', 'columnFilters': []})
|
||||
self.assertEqual(root['queryPlan'][0]['selector']['queryId'], 'trusted-query')
|
||||
value['queryPlan'][0]['selector']['queryId'] = 'answer-text-id'
|
||||
with self.assertRaisesRegex(TaskUnavailable, 'previous_ranking'):
|
||||
self.normalize(value, context)
|
||||
|
||||
def test_catalog_group_members_must_match_authorized_directory_and_root_department(self):
|
||||
catalog = {'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
|
||||
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}
|
||||
node = {'nodeId': 'compare_groups', 'purpose': 'metrics', 'fields': ['rate'],
|
||||
'stages': ['black'], 'selector': {'kind': 'catalog_members', 'dimension': 'group',
|
||||
'names': ['部门甲-国内', '部门甲-海外']}, 'comparisonRequired': True,
|
||||
'visualPreference': 'none'}
|
||||
value = task_value(kind='metrics', queryPlan=[node], dimensions=['group'],
|
||||
dimensionStages={}, stages=['black'], fields=['rate'], trendRequired=False,
|
||||
comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2026, 'month': 8}, filterUpdates={'department': '部门甲'})
|
||||
root = self.normalize(value, {'_businessCatalog': catalog})
|
||||
self.assertEqual(root['queryPlan'][0]['selector']['names'], catalog['groups'])
|
||||
self.assertEqual(root['filters']['department'], '部门甲')
|
||||
value['queryPlan'][0]['selector']['names'] = ['其他部门-国内']
|
||||
with self.assertRaisesRegex(TaskUnavailable, '目录中'):
|
||||
self.normalize(value, {'_businessCatalog': catalog})
|
||||
|
||||
def test_exact_scope_document_selection_rejects_wrong_node_stage_and_period(self):
|
||||
scope = {'nodeId': 'rank', 'scopeId': 'scope-rank', 'purpose': 'ranking',
|
||||
'task': {'kind': 'analysis', 'period': {'year': 2026}, 'filters': {},
|
||||
'dimensions': ['product'], 'fields': ['totalWeight'], 'stages': ['black'],
|
||||
'dimensionStages': {'product': ['black']}}}
|
||||
proof = {'nodeId': 'rank', 'scopeId': 'scope-rank', 'queryId': 'q1'}
|
||||
good = {'releaseId': 'release', 'mode': 'rank', 'result': {
|
||||
'releaseId': 'release', 'period': {'year': 2026}, 'filters': {},
|
||||
'protocolEvidence': proof, 'kind': 'quality_weight_ranking', 'dimension': 'product',
|
||||
'stage': 'black', 'metric': 'totalWeight'}}
|
||||
self.assertEqual(scope_documents([good], scope, release_id='release'), [good])
|
||||
forged = copy.deepcopy(good); forged['result']['protocolEvidence']['scopeId'] = 'other'
|
||||
wrong_stage = copy.deepcopy(good); wrong_stage['result']['stage'] = 'bright'
|
||||
wrong_period = copy.deepcopy(good); wrong_period['result']['period'] = {'year': 2025}
|
||||
self.assertEqual(scope_documents([forged, wrong_stage, wrong_period], scope,
|
||||
release_id='release'), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,283 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_agent_task import normalize_task
|
||||
from nao_protocol_execution import execute
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
|
||||
|
||||
|
||||
class ComparisonPlanApi(BoundApi):
|
||||
def __init__(self, scope_tasks):
|
||||
super().__init__()
|
||||
self.scope_tasks = scope_tasks
|
||||
self.envelopes = []
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'version': 1, 'scopes': [
|
||||
{'nodeId': 'groups_compare', 'scopeId': 'scope-' + key,
|
||||
'label': key, 'purpose': 'metrics', 'task': copy.deepcopy(task)}
|
||||
for key, task in self.scope_tasks.items()], 'pending': [], 'gaps': []}
|
||||
response = super().call(path, payload, method)
|
||||
if path.startswith('/api/metrics'):
|
||||
proof = response.get('protocolEvidence')
|
||||
if isinstance(proof, dict):
|
||||
proof['nodeId'] = node_id
|
||||
proof['scopeId'] = scope_id
|
||||
proof['queryId'] = f'{TURN}-{len(self.envelopes)}'
|
||||
proof['responseHash'] = f'{len(self.envelopes):064x}'
|
||||
return response
|
||||
|
||||
|
||||
class QueryPlanComparisonExecutionTests(unittest.TestCase):
|
||||
def test_missing_prior_year_composite_sources_finish_partial_with_root_frame_and_proofs(self):
|
||||
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate'], 'stages': [],
|
||||
'selector': {'kind': 'catalog_members', 'dimension': 'group',
|
||||
'names': ['部门甲-国内', '部门甲-海外']},
|
||||
'visualPreference': 'none', 'visualType': None, 'comparisonRequired': True}
|
||||
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
|
||||
dimensionStages={}, stages=[], fields=['rate'], trendRequired=False,
|
||||
comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='none',
|
||||
filterUpdates={'department': '部门甲'})
|
||||
context = {'_protocol': {'version': 1}, '_businessCatalog': {
|
||||
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
|
||||
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
|
||||
task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月的综合合格率', context)
|
||||
child_tasks = {}
|
||||
for name in rule['selector']['names']:
|
||||
child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'],
|
||||
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
|
||||
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
|
||||
'fields': ['rate'], 'stages': [], 'trendRequired': False, 'comparisonRequired': True,
|
||||
'periodCoverage': task['periodCoverage'], 'visualPreference': 'none', 'visualType': None,
|
||||
'freshQuery': True, 'historyMode': 'none'}
|
||||
|
||||
class MissingPriorYearApi(ComparisonPlanApi):
|
||||
def __init__(self, scope_tasks):
|
||||
super().__init__(scope_tasks)
|
||||
self.actual_metric_responses = []
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
if path.startswith('/api/metrics') and 'year=2025' in path:
|
||||
response['coverage'].update(rawRows=0, includedRows=0,
|
||||
sourceMonths=[], missingStages=['black', 'bright'])
|
||||
response['summary']['compositeRate'] = None
|
||||
if path.startswith('/api/metrics'):
|
||||
self.actual_metric_responses.append(copy.deepcopy(response))
|
||||
return response
|
||||
|
||||
api = MissingPriorYearApi(child_tasks)
|
||||
api.task = task
|
||||
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
|
||||
events = []
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer._open_model', side_effect=AssertionError(
|
||||
'缺少正式来源时不应调用模型补造事实')) as open_model:
|
||||
result = execute(request=request, task=task, authority={**identity(), 'task': task},
|
||||
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
|
||||
cancel_event=threading.Event(), emit=events.append)
|
||||
self.assertFalse(list((root / 'evidence').glob('nao-quality-*.json')))
|
||||
private_completion = json.loads((root / 'private' / 'protocol-completion.json').read_text())
|
||||
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(result['agentContext'], {'releaseId': RELEASE, 'filters': task['filters'],
|
||||
'period': task['period'], 'comparisonPeriod': task['comparisonPeriod']})
|
||||
self.assertEqual(len(result['protocolEvidence']), 4, result.get('evidenceCompletion'))
|
||||
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'})
|
||||
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
|
||||
{'scope-部门甲-国内', 'scope-部门甲-海外'})
|
||||
self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}), 4)
|
||||
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertTrue(all('2025' in gap for gap in result['evidenceCompletion']['missingEvidence']),
|
||||
result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertFalse(result['scopedReport']['facts'])
|
||||
self.assertNotIn('100%', result['answer'])
|
||||
self.assertNotIn('100.00%', result['answer'])
|
||||
self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics')
|
||||
and 'year=2025' in item['path']]), 2)
|
||||
self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics')]), 4)
|
||||
missing_responses = [response for response in api.actual_metric_responses
|
||||
if response.get('context', {}).get('year') == 2025]
|
||||
self.assertEqual(len(missing_responses), 2)
|
||||
self.assertTrue(all(response['coverage'].get('rawRows') == 0
|
||||
and response['coverage'].get('includedRows') == 0
|
||||
and response['coverage'].get('sourceMonths') == []
|
||||
for response in missing_responses))
|
||||
from types import SimpleNamespace
|
||||
from nao_protocol_execution import _verified_empty_comparison_receipts
|
||||
domestic_responses = [response for response in api.actual_metric_responses
|
||||
if response.get('context', {}).get('group') == '部门甲-国内']
|
||||
def receipt_observations(responses):
|
||||
return [{'release': copy.deepcopy(response.get('release')),
|
||||
'context': copy.deepcopy(response.get('context')),
|
||||
'filters': copy.deepcopy(response.get('filters')),
|
||||
'coverage': copy.deepcopy(response.get('coverage')),
|
||||
'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')),
|
||||
'hasSummary': isinstance(response.get('summary'), dict)}
|
||||
for response in responses]
|
||||
scope = {'nodeId': 'groups_compare', 'scopeId': 'scope-部门甲-国内',
|
||||
'label': '部门甲-国内'}
|
||||
receipts = _verified_empty_comparison_receipts(SimpleNamespace(
|
||||
metric_responses=receipt_observations(domestic_responses)), scope=scope,
|
||||
task=child_tasks['部门甲-国内'], identity=identity(), release_id=RELEASE)
|
||||
self.assertIsNotNone(receipts)
|
||||
for mutate in (
|
||||
lambda response: response['context'].pop('year', None),
|
||||
lambda response: response['coverage'].__setitem__('rawRows', -1),
|
||||
lambda response: response['coverage'].__setitem__('includedRows', 2),
|
||||
lambda response: response['coverage'].pop('includedRows', None),
|
||||
lambda response: response['coverage'].pop('rawRows', None),
|
||||
lambda response: response['protocolEvidence'].update(scopeId='forged-scope'),
|
||||
lambda response: response['protocolEvidence'].update(executionId='forged-execution')):
|
||||
invalid = copy.deepcopy(domestic_responses)
|
||||
mutate(invalid[1])
|
||||
self.assertIsNone(_verified_empty_comparison_receipts(SimpleNamespace(
|
||||
metric_responses=receipt_observations(invalid)), scope=scope, task=child_tasks['部门甲-国内'],
|
||||
identity=identity(), release_id=RELEASE))
|
||||
self.assertTrue(any('正式查询 4/40' in event.get('message', '') for event in events))
|
||||
diagnostics = private_completion['queryDiagnostics']
|
||||
self.assertEqual(len(diagnostics), 4)
|
||||
self.assertEqual(sum(row['rawRows'] == 0 for row in diagnostics), 2)
|
||||
self.assertEqual(sum(row['includedRows'] == 0 for row in diagnostics), 2)
|
||||
self.assertTrue(all(row['nodeId'] == 'groups_compare' and row['queryId'] for row in diagnostics))
|
||||
open_model.assert_not_called()
|
||||
|
||||
def test_one_empty_group_keeps_other_group_charts_ready_while_turn_is_partial(self):
|
||||
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics',
|
||||
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'],
|
||||
'selector': {'kind': 'catalog_members', 'dimension': 'group',
|
||||
'names': ['部门甲-国内', '部门甲-海外']},
|
||||
'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True}
|
||||
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
|
||||
dimensionStages={}, stages=['black'], fields=rule['fields'], trendRequired=False,
|
||||
comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='required',
|
||||
visualType='column', filterUpdates={'department': '部门甲'})
|
||||
context = {'_protocol': {'version': 1}, '_businessCatalog': {
|
||||
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
|
||||
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
|
||||
task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月黑片合格率和重量', context)
|
||||
child_tasks = {name: {'version': 1, 'kind': 'metrics', 'question': task['question'],
|
||||
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
|
||||
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
|
||||
'fields': list(rule['fields']), 'stages': ['black'], 'trendRequired': False,
|
||||
'comparisonRequired': True, 'periodCoverage': task['periodCoverage'],
|
||||
'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'}
|
||||
for name in rule['selector']['names']}
|
||||
|
||||
class MissingOverseasApi(ComparisonPlanApi):
|
||||
def __init__(self, scope_tasks):
|
||||
super().__init__(scope_tasks)
|
||||
self.actual_metric_responses = []
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
if (path.startswith('/api/metrics')
|
||||
and response.get('context', {}).get('group') == '部门甲-海外'
|
||||
and response.get('context', {}).get('year') == 2025):
|
||||
response['coverage'].update(rawRows=0, includedRows=0,
|
||||
sourceMonths=[], missingStages=['black', 'bright'])
|
||||
response['summary']['compositeRate'] = None
|
||||
if path.startswith('/api/metrics'):
|
||||
self.actual_metric_responses.append(copy.deepcopy(response))
|
||||
return response
|
||||
|
||||
api = MissingOverseasApi(child_tasks)
|
||||
api.task = task
|
||||
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize', return_value={
|
||||
'answer': '国内对照数据有效,海外缺少去年来源。',
|
||||
'evidenceSufficient': True, 'missingEvidence': [], 'usage': {}}):
|
||||
result = execute(request=request, task=task, authority={**identity(), 'task': task},
|
||||
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
|
||||
cancel_event=threading.Event(), emit=lambda _: None)
|
||||
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
self.assertTrue(result['visuals'])
|
||||
self.assertEqual({chart.get('scope', {}).get('group') for chart in result['visuals']},
|
||||
{'部门甲-国内'})
|
||||
self.assertEqual({chart['metric'] for chart in result['visuals']},
|
||||
{'black_rate', 'black_total_weight', 'black_scrap_weight'})
|
||||
self.assertTrue(any('部门甲-海外' in gap and '2025' in gap
|
||||
for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
self.assertEqual(len(result['protocolEvidence']), 4)
|
||||
self.assertEqual(len(result['queryEvidenceRefs']), 1)
|
||||
|
||||
def test_catalog_group_compare_uses_current_periods_without_mutating_root_filters(self):
|
||||
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate', 'totalWeight', 'scrapWeight'],
|
||||
'stages': ['black'], 'selector': {'kind': 'catalog_members',
|
||||
'dimension': 'group', 'names': ['部门甲-国内', '部门甲-海外']},
|
||||
'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True}
|
||||
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
|
||||
dimensionStages={}, stages=['black'], fields=['rate', 'totalWeight', 'scrapWeight'], trendRequired=False,
|
||||
comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2026, 'month': 8}, filterUpdates={'department': '部门甲'})
|
||||
context = {'_protocol': {'version': 1}, '_businessCatalog': {
|
||||
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
|
||||
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
|
||||
task = normalize_task(value, '比较部门甲的国内和海外两期黑片合格率', context)
|
||||
child_tasks = {}
|
||||
for name in rule['selector']['names']:
|
||||
child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'],
|
||||
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
|
||||
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
|
||||
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'], 'trendRequired': False,
|
||||
'comparisonRequired': True, 'periodCoverage': task['periodCoverage'],
|
||||
'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'}
|
||||
api = ComparisonPlanApi(child_tasks)
|
||||
api.task = task
|
||||
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize', return_value={
|
||||
'answer': '已核对国内和海外两期指标', 'evidenceSufficient': True,
|
||||
'missingEvidence': [], 'usage': {}}) as synth:
|
||||
result = execute(request=request, task=task, authority={**identity(), 'task': task},
|
||||
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
|
||||
cancel_event=threading.Event(), emit=lambda _: None)
|
||||
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertEqual(len(result['protocolEvidence']), 4)
|
||||
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'})
|
||||
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
|
||||
{'scope-部门甲-国内', 'scope-部门甲-海外'})
|
||||
metric_calls = [call for call in api.envelopes if call['path'].startswith('/api/metrics')]
|
||||
self.assertTrue(metric_calls)
|
||||
self.assertTrue(all(call['nodeId'] == 'groups_compare' and call['scopeId'] for call in metric_calls))
|
||||
self.assertEqual(result['agentContext']['filters'], task['filters'])
|
||||
self.assertNotIn('group', result['agentContext']['filters'])
|
||||
self.assertEqual(result['agentContext']['comparisonPeriod'], task['comparisonPeriod'])
|
||||
chart_keys = {'id', 'type', 'title', 'subtitle', 'unit', 'categories', 'series',
|
||||
'evidenceRefs', 'requiredFields', 'scope', 'period', 'comparisonPeriod',
|
||||
'stage', 'metric', 'coverage', 'note', 'originTurnId', 'sectionKey', 'dimension'}
|
||||
self.assertTrue(result['visuals'])
|
||||
self.assertTrue(all(set(chart) <= chart_keys for chart in result['visuals']))
|
||||
self.assertTrue(all(not {'protocolNodeId', 'protocolScopeId', 'protocolScopeLabel'} & set(chart)
|
||||
for chart in result['visuals']))
|
||||
for group in ('部门甲-国内', '部门甲-海外'):
|
||||
scoped_charts = [chart for chart in result['visuals']
|
||||
if chart.get('scope', {}).get('group') == group]
|
||||
self.assertEqual({chart['metric'] for chart in scoped_charts},
|
||||
{'black_rate', 'black_total_weight', 'black_scrap_weight'})
|
||||
self.assertTrue(all(chart['type'] == 'column' for chart in scoped_charts))
|
||||
synth.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,194 @@
|
||||
import copy
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_agent_task import TaskUnavailable, normalize_task, plan_task
|
||||
from nao_query_contract import bind_query
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
|
||||
def defect_catalog():
|
||||
fields = [
|
||||
{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
|
||||
'sourceField': '责废尺寸重量', 'category': '责废',
|
||||
'grain': 'source_defect_weight', 'available': True},
|
||||
{'stage': 'black', 'key': '料废端面重量', 'label': '料废端面重量',
|
||||
'sourceField': '料废端面重量', 'category': '料废',
|
||||
'grain': 'source_defect_weight', 'available': True},
|
||||
{'stage': 'bright', 'key': '加工缺口重量', 'label': '加工缺口重量',
|
||||
'sourceField': '加工缺口重量', 'category': '加工废',
|
||||
'grain': 'source_defect_weight', 'available': True},
|
||||
{'stage': 'bright', 'key': '加工数量', 'label': '加工数量',
|
||||
'sourceField': '加工数量', 'category': '加工废',
|
||||
'grain': 'source_defect_count', 'available': True, 'countOnly': True},
|
||||
]
|
||||
categories = [
|
||||
{'stage': 'black', 'category': '责废', 'available': True},
|
||||
{'stage': 'black', 'category': '料废', 'available': True},
|
||||
{'stage': 'bright', 'category': '加工废', 'available': True},
|
||||
]
|
||||
return {'fineDefects': {'fields': fields, 'categories': categories}}
|
||||
|
||||
|
||||
def defect_node(*, dimension='defect', stages=None, selector=None, defect=None,
|
||||
category=None, node_id='black_defects', comparison=False):
|
||||
node = {'nodeId': node_id, 'purpose': 'defects', 'dimension': dimension,
|
||||
'fields': ['scrapWeight'], 'stages': stages or ['black'],
|
||||
'selector': selector or {'kind': 'base'}, 'visualType': 'bar',
|
||||
'visualPreference': 'auto', 'comparisonRequired': comparison}
|
||||
if defect is not None:
|
||||
node['defect'] = defect
|
||||
if category is not None:
|
||||
node['category'] = category
|
||||
return node
|
||||
|
||||
|
||||
class QueryPlanDefectNormalizationTests(unittest.TestCase):
|
||||
def normalize(self, value, question='查看来源具体缺陷重量', context=None):
|
||||
supplied = context or {}
|
||||
business = {**defect_catalog(), **supplied.get('_businessCatalog', {})}
|
||||
return normalize_task(value, question, {'_protocol': {'version': 1},
|
||||
**supplied, '_businessCatalog': business})
|
||||
|
||||
def test_defect_node_derives_root_output_dimensions_stages_and_fields(self):
|
||||
value = task_value(kind='analysis', queryPlan=[defect_node()],
|
||||
dimensions=['defect'], dimensionStages={'defect': ['black']},
|
||||
stages=['black'], fields=['scrapWeight'], trendRequired=False,
|
||||
comparisonRequired=False, period={'year': 2026, 'month': 9})
|
||||
task = self.normalize(value)
|
||||
self.assertEqual(task['dimensions'], ['defect'])
|
||||
self.assertEqual(task['dimensionStages'], {'defect': ['black']})
|
||||
self.assertEqual(task['stages'], ['black'])
|
||||
self.assertEqual(task['fields'], ['scrapWeight'])
|
||||
self.assertEqual(task['queryPlan'][0]['visualType'], 'bar')
|
||||
self.assertEqual(task['periodCoverage'], 'available')
|
||||
|
||||
def test_product_output_adds_product_and_defect_but_selector_group_is_not_output_dimension(self):
|
||||
selector = {'kind': 'catalog_members', 'dimension': 'group',
|
||||
'names': ['部门甲-国内', '部门甲-海外']}
|
||||
node = defect_node(dimension='product', defect='责废尺寸重量', category='责废', selector=selector)
|
||||
value = task_value(kind='analysis', queryPlan=[node], dimensions=['product', 'group', 'defect'],
|
||||
dimensionStages={'product': ['black'], 'group': [], 'defect': ['black']},
|
||||
stages=['black'], fields=['scrapWeight'], trendRequired=False,
|
||||
comparisonRequired=False, period={'year': 2026, 'month': 9})
|
||||
context = {'_businessCatalog': {'groups': ['部门甲-国内', '部门甲-海外']}}
|
||||
task = self.normalize(value, context=context)
|
||||
self.assertEqual(task['dimensions'], ['product', 'group', 'defect'])
|
||||
self.assertEqual(task['dimensionStages'], {'product': ['black'], 'group': [], 'defect': ['black']})
|
||||
self.assertEqual(task['queryPlan'][0]['selector'], selector)
|
||||
|
||||
def test_defect_output_can_follow_a_trusted_product_ranking_selector(self):
|
||||
nodes = [
|
||||
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 10,
|
||||
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
|
||||
'visualType': 'pareto', 'visualPreference': 'auto', 'comparisonRequired': False},
|
||||
defect_node(dimension='product', defect='责废尺寸重量',
|
||||
selector={'kind': 'ranking_top', 'sourceNodeId': 'rank',
|
||||
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
|
||||
node_id='top_defects'),
|
||||
]
|
||||
value = task_value(kind='analysis', queryPlan=nodes, dimensions=['product', 'defect'],
|
||||
dimensionStages={'product': ['black'], 'defect': ['black']}, stages=['black'],
|
||||
fields=['totalWeight', 'scrapWeight'], trendRequired=False, comparisonRequired=False,
|
||||
period={'year': 2026, 'month': 9})
|
||||
task = self.normalize(value)
|
||||
self.assertEqual(task['dimensions'], ['product', 'defect'])
|
||||
self.assertEqual(task['queryPlan'][1]['selector']['kind'], 'ranking_top')
|
||||
self.assertEqual(task['queryPlan'][1]['defect'], '责废尺寸重量')
|
||||
|
||||
def test_category_or_defect_must_be_authorized_for_every_stage_and_not_count_only(self):
|
||||
cases = [
|
||||
defect_node(stages=['black', 'bright'], category='责废'),
|
||||
defect_node(stages=['bright'], defect='加工数量'),
|
||||
defect_node(stages=['black'], defect='责废尺寸重量', category='料废'),
|
||||
]
|
||||
for node in cases:
|
||||
with self.subTest(node=node):
|
||||
dims = ['defect']
|
||||
dimension_stages = {'defect': list(node['stages'])}
|
||||
value = task_value(kind='analysis', queryPlan=[node], dimensions=dims,
|
||||
dimensionStages=dimension_stages, stages=list(node['stages']),
|
||||
fields=['scrapWeight'], trendRequired=False, comparisonRequired=False,
|
||||
period={'year': 2026, 'month': 9})
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
self.normalize(value)
|
||||
|
||||
def test_auto_plan_is_limited_to_unqualified_all_detail_defect_analysis(self):
|
||||
value = task_value(kind='analysis', dimensions=['defect'], dimensionStages={'defect': ['black']},
|
||||
stages=['black'], fields=['scrapWeight'], trendRequired=False,
|
||||
comparisonRequired=False, period={'year': 2026, 'month': 9})
|
||||
task = self.normalize(value, question='查看2026年9月黑片具体缺陷重量排名')
|
||||
self.assertEqual(len(task['queryPlan']), 1)
|
||||
self.assertEqual(task['queryPlan'][0]['purpose'], 'defects')
|
||||
self.assertEqual(task['queryPlan'][0]['dimension'], 'defect')
|
||||
self.assertEqual(task['queryPlan'][0]['limit'], 100)
|
||||
self.assertEqual(task['queryPlan'][0]['visualType'], 'bar')
|
||||
|
||||
def test_named_category_or_mixed_goal_cannot_be_silently_expanded(self):
|
||||
value = task_value(kind='analysis', dimensions=['defect'], dimensionStages={'defect': ['black']},
|
||||
stages=['black'], fields=['scrapWeight'], trendRequired=False,
|
||||
comparisonRequired=False, period={'year': 2026, 'month': 9})
|
||||
with self.assertRaises(TaskUnavailable) as raised:
|
||||
self.normalize(value, question='只查看黑片责废的具体缺陷重量')
|
||||
self.assertEqual(raised.exception.field, 'queryPlan')
|
||||
self.assertEqual(raised.exception.expected, 'bounded defects node plan')
|
||||
|
||||
def test_comparison_requires_root_period_and_true_node_union(self):
|
||||
node = defect_node(category='责废', comparison=True)
|
||||
value = task_value(kind='analysis', queryPlan=[node], dimensions=['defect'],
|
||||
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
|
||||
trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2025, 'month': 9})
|
||||
task = self.normalize(value)
|
||||
self.assertTrue(task['queryPlan'][0]['comparisonRequired'])
|
||||
self.assertEqual(task['comparisonPeriod'], {'year': 2025, 'month': 9})
|
||||
value['comparisonRequired'] = False
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
self.normalize(value)
|
||||
|
||||
def test_defect_query_contract_binds_comparison_frame_and_column_filters(self):
|
||||
task = normalize_task(task_value(kind='analysis', queryPlan=[defect_node(comparison=True)],
|
||||
dimensions=['defect'], dimensionStages={'defect': ['black']}, stages=['black'],
|
||||
fields=['scrapWeight'], trendRequired=False, comparisonRequired=True,
|
||||
period={'year': 2026, 'monthStart': 3, 'monthEnd': 9},
|
||||
comparisonPeriod={'year': 2025, 'monthStart': 3, 'monthEnd': 9}),
|
||||
'比较两期黑片来源具体缺陷重量', {'_protocol': {'version': 1},
|
||||
'_businessCatalog': defect_catalog()})
|
||||
rule = {'field': 'customerCode', 'stages': ['black'], 'operator': 'in', 'values': ['C-17']}
|
||||
task['filters']['columnFilters'] = [rule]
|
||||
bound = bind_query({'mode': 'defects', 'releaseId': 'release-defects',
|
||||
'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
|
||||
'filters': {}, 'columnFilters': [rule], 'stage': 'black', 'dimension': 'defect',
|
||||
'limit': 100}, {'task': task, 'releaseId': 'release-defects', 'exactScope': True,
|
||||
'nodeId': 'black_defects', 'scopeId': 'server-issued-scope'})
|
||||
self.assertEqual(bound['period'], task['period'])
|
||||
self.assertEqual(bound['comparisonPeriod'], task['comparisonPeriod'])
|
||||
self.assertEqual(bound['columnFilters'], [rule])
|
||||
with self.assertRaisesRegex(ValueError, '期间偏离'):
|
||||
bind_query({**bound, 'comparisonPeriod': {'year': 2024, 'month': 9}},
|
||||
{'task': task, 'releaseId': 'release-defects', 'exactScope': True,
|
||||
'nodeId': 'black_defects', 'scopeId': 'server-issued-scope'})
|
||||
|
||||
def test_planning_payload_keeps_authorized_fine_defect_catalog(self):
|
||||
from unittest.mock import patch
|
||||
plan = task_value(kind='analysis', queryPlan=[defect_node()], dimensions=['defect'],
|
||||
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
|
||||
trendRequired=False, comparisonRequired=False, period={'year': 2026, 'month': 9})
|
||||
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False}
|
||||
context = {'_protocol': {'version': 1}, '_businessCatalog': defect_catalog()}
|
||||
with patch('nao_agent_task._model_json', side_effect=[plan, review]) as model_call:
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='比较黑片具体缺陷重量', context=context, cancel_event=threading.Event())
|
||||
payload = model_call.call_args_list[0].kwargs['payload']
|
||||
self.assertEqual(payload['businessCatalog']['fineDefects'], defect_catalog()['fineDefects'])
|
||||
self.assertEqual(planned['queryPlan'][0]['purpose'], 'defects')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,162 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_agent_task import normalize_task
|
||||
from nao_protocol_execution import execute
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_protocol_execution import RELEASE, TURN, identity
|
||||
from test_nao_query_plan_defects import defect_catalog, defect_node
|
||||
from test_nao_quality_tools import FakeApi
|
||||
|
||||
|
||||
SOURCE_FIELD = '责废尺寸重量'
|
||||
|
||||
|
||||
def java_category_reconciliation(fields, source_rows):
|
||||
expected_counts = {'责废': 7, '料废': 5}
|
||||
component_codes = {'责废': 'black_responsibility_scrap', '料废': 'black_material_scrap'}
|
||||
by_category = {}
|
||||
for field in fields:
|
||||
by_category.setdefault(field['category'], []).append(field)
|
||||
result = []
|
||||
for category, selected in by_category.items():
|
||||
components = [{'key': field['key'], 'label': field['label'], 'weight': 4.0,
|
||||
'sourceRows': source_rows, 'presentRows': source_rows,
|
||||
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}
|
||||
for field in selected]
|
||||
expected = expected_counts[category]
|
||||
result.append({'category': category, 'sourceRows': source_rows, 'components': components,
|
||||
'subitemWeightSum': 4.0 * len(selected), 'validSubitemCount': len(selected),
|
||||
'subitemFieldCount': expected, 'coverage': {'valid': len(selected), 'total': expected},
|
||||
'formalComponent': component_codes[category], 'formalComponentSum': 9.0,
|
||||
'formalComponentCoverage': {'sourceRows': source_rows, 'presentRows': source_rows,
|
||||
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
|
||||
'diff': 4.0 * len(selected) - 9.0, 'isReconciled': False,
|
||||
'reconciliationStatus': 'INCOMPLETE',
|
||||
'reconciliationReasons': ['部分原始分项字段受授权限制,不能完整回加']})
|
||||
return result
|
||||
|
||||
|
||||
class DefectProtocolApi(FakeApi):
|
||||
def __init__(self, *, bad_proof=False, unavailable=False):
|
||||
super().__init__()
|
||||
self.root = None
|
||||
self.node = None
|
||||
self.bad_proof = bad_proof
|
||||
self.unavailable = unavailable
|
||||
self.envelopes = []
|
||||
self.reads = []
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
child = copy.deepcopy(self.root)
|
||||
child.pop('queryPlan', None)
|
||||
child['defectQuery'] = {'dimension': self.node['dimension'], 'limit': self.node['limit']}
|
||||
scope = {'nodeId': self.node['nodeId'], 'scopeId': 'black-defects-scope',
|
||||
'label': '黑片具体缺陷', 'purpose': 'defects', 'task': child}
|
||||
return {'version': 1, 'scopes': [scope], 'pending': [], 'gaps': []}
|
||||
if path == '/api/analysis/defects':
|
||||
self.reads.append(copy.deepcopy(payload))
|
||||
year = payload['year']
|
||||
rows = [
|
||||
{'key': SOURCE_FIELD, 'label': SOURCE_FIELD, 'category': '责废',
|
||||
'weight': 8.0 if year == 2026 else 5.0, 'sourceRows': 3,
|
||||
'presentRows': 3, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
|
||||
{'key': '料废尺寸重量', 'label': '料废尺寸重量', 'category': '料废',
|
||||
'weight': 4.0 if year == 2026 else 3.0, 'sourceRows': 3,
|
||||
'presentRows': 3, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
|
||||
]
|
||||
fields = [{'key': row['key'], 'label': row['label'], 'sourceField': row['key'],
|
||||
'category': row['category']} for row in rows]
|
||||
context = {key: payload.get(key) for key in (
|
||||
'year', 'month', 'quarter', 'monthStart', 'monthEnd', 'department', 'group',
|
||||
'product', 'projectGroup', 'includeUnassigned', 'stage', 'dimension', 'defect', 'category')}
|
||||
context['extraFilters'] = payload.get('extraFilters', {})
|
||||
context['exclusions'] = payload.get('exclusions', {})
|
||||
context['filters'] = {'columnFilters': payload.get('columnFilters', [])}
|
||||
available = not self.unavailable
|
||||
proof = {**identity(), 'queryId': f'defect-{year}', 'responseHash': f'{year:064x}',
|
||||
'nodeId': 'forged' if self.bad_proof else node_id, 'scopeId': scope_id}
|
||||
return {'kind': 'fine_defect_top', 'releaseId': payload['releaseId'],
|
||||
'release': {'id': payload['releaseId'], 'status': 'PUBLISHED'},
|
||||
'context': {**context, 'mode': 'top'}, 'grain': 'source_defect_weight',
|
||||
'rows': rows if available else [], 'totalGroups': len(rows) if available else 0,
|
||||
'rowsTruncated': False, 'sourceRows': 3,
|
||||
'coverage': {'available': available, 'sourceRows': 3,
|
||||
'hasValidWeights': available},
|
||||
'categoryReconciliation': java_category_reconciliation(fields, 3) if available else [],
|
||||
'sourceFields': fields if available else [], 'notice': '来源细项重量。',
|
||||
'protocolEvidence': proof}
|
||||
return super().call(path, payload, method)
|
||||
|
||||
|
||||
class QueryPlanDefectExecutionTests(unittest.TestCase):
|
||||
def task(self):
|
||||
node = defect_node(comparison=True, node_id='faults')
|
||||
value = task_value(kind='analysis', queryPlan=[node], dimensions=['defect'],
|
||||
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
|
||||
trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9},
|
||||
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='auto', visualType='bar')
|
||||
task = normalize_task(value, '比较2026年9月和2025年9月黑片来源具体缺陷重量',
|
||||
{'_protocol': {'version': 1}, '_businessCatalog': defect_catalog()})
|
||||
if task['filters'].get('columnFilters') == []:
|
||||
task['filters'].pop('columnFilters')
|
||||
return task
|
||||
|
||||
def run_query(self, api):
|
||||
task = self.task()
|
||||
api.root, api.node = task, task['queryPlan'][0]
|
||||
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
(root / 'evidence').mkdir()
|
||||
(root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize', return_value={
|
||||
'answer': '已核对两个期间的来源缺陷重量。', 'evidenceSufficient': True,
|
||||
'missingEvidence': [], 'usage': {}}):
|
||||
result = execute(request=request, task=task,
|
||||
authority={**identity(), 'task': task}, actor_root=root, delegated=api,
|
||||
connection=('offline', 'synthetic', 'model'), cancel_event=threading.Event(),
|
||||
emit=lambda _: None)
|
||||
documents = [json.loads(path.read_text()) for path in (root / 'evidence').glob('nao-quality-*.json')]
|
||||
return result, documents
|
||||
|
||||
def test_comparison_defect_node_uses_two_bound_requests_and_completes(self):
|
||||
api = DefectProtocolApi()
|
||||
result, documents = self.run_query(api)
|
||||
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertEqual(result['stopReason'], 'evidence_finalized')
|
||||
self.assertEqual(result['completionMethod'], 'bounded_scoped_query_plan')
|
||||
self.assertEqual([item['year'] for item in api.reads], [2026, 2025])
|
||||
self.assertTrue(all(item['limit'] == 100 for item in api.reads))
|
||||
self.assertEqual(len(documents), 1)
|
||||
self.assertEqual(len(documents[0]['result']['protocolEvidence']), 2)
|
||||
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'faults'})
|
||||
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']}, {'black-defects-scope'})
|
||||
self.assertEqual([item['nodeId'] for item in api.envelopes
|
||||
if item['path'] == '/api/analysis/defects'], ['faults', 'faults'])
|
||||
self.assertEqual([item['scopeId'] for item in api.envelopes
|
||||
if item['path'] == '/api/analysis/defects'],
|
||||
['black-defects-scope', 'black-defects-scope'])
|
||||
|
||||
def test_unavailable_detail_or_forged_scope_receipt_stays_partial(self):
|
||||
for api in (DefectProtocolApi(unavailable=True), DefectProtocolApi(bad_proof=True)):
|
||||
with self.subTest(api=api):
|
||||
result, _ = self.run_query(api)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,360 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_defect_report import protocol_defect_report
|
||||
from nao_evidence_finalizer import coverage_sufficient_scoped
|
||||
from nao_query_plan import scope_documents
|
||||
from analysis_visual_evidence import protocol_visual_result
|
||||
|
||||
|
||||
RELEASE = 'release-defects'
|
||||
QUESTION = '查看黑片来源具体缺陷重量'
|
||||
FIELD = '责废尺寸重量'
|
||||
|
||||
|
||||
def source_field(key=FIELD, category='责废'):
|
||||
return {'key': key, 'label': key, 'sourceField': key, 'category': category}
|
||||
|
||||
|
||||
def java_category_reconciliation(fields, *, source_rows, expected_fields=None,
|
||||
reconciled=True, diff=0.0, reasons=None,
|
||||
formal_component='black_responsibility_scrap', field_weights=None,
|
||||
status=None):
|
||||
expected_fields = len(fields) if expected_fields is None else expected_fields
|
||||
field_weights = field_weights or {field['key']: 4.5 for field in fields}
|
||||
components = [{'key': field['key'], 'label': field['label'], 'weight': field_weights[field['key']],
|
||||
'presentRows': source_rows, 'missingRows': 0, 'invalidRows': 0,
|
||||
'negativeRows': 0, 'sourceRows': source_rows} for field in fields]
|
||||
formal_source_rows = source_rows
|
||||
return {'category': fields[0]['category'], 'sourceRows': source_rows,
|
||||
'components': components, 'subitemWeightSum': sum(field_weights.values()),
|
||||
'validSubitemCount': len(fields), 'subitemFieldCount': expected_fields,
|
||||
'coverage': {'valid': len(fields), 'total': expected_fields},
|
||||
'formalComponent': formal_component,
|
||||
'formalComponentSum': sum(field_weights.values()) - diff,
|
||||
'formalComponentCoverage': {'presentRows': formal_source_rows, 'missingRows': 0,
|
||||
'invalidRows': 0, 'negativeRows': 0, 'sourceRows': formal_source_rows},
|
||||
'diff': diff, 'isReconciled': reconciled,
|
||||
'reconciliationStatus': status or ('RECONCILED' if reconciled else 'DIFF'),
|
||||
'reconciliationReasons': list(reasons or [])}
|
||||
|
||||
|
||||
def defect_result(*, dimension='defect', comparison=False, stage='black', filters=None,
|
||||
rows=None, ranking_complete=True, fields=None):
|
||||
fields = fields or [source_field()]
|
||||
if rows is None:
|
||||
if dimension == 'product':
|
||||
rows = [{'key': '规格A', 'label': '规格A', 'category': fields[0]['category'],
|
||||
'weight': 4.5, 'defect': fields[0]['key'], 'sourceRows': 4,
|
||||
'presentRows': 4, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}]
|
||||
else:
|
||||
rows = [{'key': field['key'], 'label': field['label'], 'category': field['category'],
|
||||
'weight': 4.5, 'defect': field['key'], 'sourceRows': 4,
|
||||
'presentRows': 4, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}
|
||||
for field in fields]
|
||||
current_reconciliation = java_category_reconciliation(fields, source_rows=4)
|
||||
result = {
|
||||
'kind': 'quality_fine_defects', 'releaseId': RELEASE, 'dimension': dimension,
|
||||
'defect': fields[0]['key'] if dimension == 'product' else None, 'category': None,
|
||||
'grain': 'source_defect_weight', 'stage': stage,
|
||||
'period': {'year': 2026, 'month': 9}, 'filters': filters or {},
|
||||
'coverage': {'available': True, 'hasValidWeights': True, 'sourceRows': 4},
|
||||
'sourceRows': 4, 'sourceFields': fields,
|
||||
'categoryReconciliation': [current_reconciliation],
|
||||
'rows': rows, 'totalRows': len(rows), 'rowsReturned': len(rows),
|
||||
'truncated': False, 'rankingComplete': ranking_complete,
|
||||
'protocolEvidence': [{'nodeId': 'faults', 'scopeId': 'fault-scope', 'queryId': 'q-black'}],
|
||||
}
|
||||
if comparison:
|
||||
comparison_reconciliation = java_category_reconciliation(fields, source_rows=3)
|
||||
result.update(comparisonPeriod={'year': 2025, 'month': 9},
|
||||
comparisonCoverage={'available': True, 'hasValidWeights': True, 'sourceRows': 3},
|
||||
comparisonSourceFields=fields,
|
||||
comparisonCategoryReconciliation=[comparison_reconciliation],
|
||||
pairedRows=[{'key': FIELD,
|
||||
'current': {'key': FIELD, 'label': FIELD, 'category': '责废', 'weight': 4.5, 'defect': FIELD},
|
||||
'comparison': {'key': FIELD, 'label': FIELD, 'category': '责废', 'weight': 3.0, 'defect': FIELD},
|
||||
'present': {'current': True, 'comparison': True}, 'deltaWeight': 1.5,
|
||||
'deltaAvailable': True}],
|
||||
pairedTotalRows=1, pairedTruncated=False, pairingComplete=True,
|
||||
pairedRankingComplete=True, sort='scrap_delta')
|
||||
result['protocolEvidence'].append({'nodeId': 'faults', 'scopeId': 'fault-scope', 'queryId': 'q-old'})
|
||||
return result
|
||||
|
||||
|
||||
def document(result, *, mode='defects'):
|
||||
return {'tool': 'quality_data', 'mode': mode, 'releaseId': RELEASE,
|
||||
'evidenceRef': 'QABCDEF01', 'result': result}
|
||||
|
||||
|
||||
def child_task(*, dimension='defect', comparison=False, stages=None, visual_type='bar'):
|
||||
output_dimensions = ['defect'] if dimension == 'defect' else ['product', 'defect']
|
||||
stages = stages or ['black']
|
||||
task = {'version': 1, 'kind': 'analysis', 'question': QUESTION, 'goal': QUESTION,
|
||||
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2025, 'month': 9} if comparison else None,
|
||||
'comparisonRequired': comparison, 'trendRequired': False,
|
||||
'dimensions': output_dimensions, 'dimensionStages': {key: list(stages) for key in output_dimensions},
|
||||
'fields': ['scrapWeight'], 'stages': list(stages), 'filters': {},
|
||||
'defectQuery': {'dimension': dimension, 'limit': 100,
|
||||
**({'defect': FIELD} if dimension == 'product' else {})},
|
||||
'visualPreference': 'auto', 'visualType': visual_type,
|
||||
'periodCoverage': 'available', 'freshQuery': True, 'historyMode': 'none'}
|
||||
return task
|
||||
|
||||
|
||||
def scope(task):
|
||||
return {'nodeId': 'faults', 'scopeId': 'fault-scope', 'label': '黑片具体缺陷',
|
||||
'purpose': 'defects', 'task': task}
|
||||
|
||||
|
||||
def compact_comparison_result(result):
|
||||
"""Mirror the lossless comparison shape used by evidence_summaries."""
|
||||
compact = copy.deepcopy(result)
|
||||
compact['sourceFields'] = copy.deepcopy(compact['sourceFields'])
|
||||
compact['comparisonSourceFields'] = copy.deepcopy(compact['comparisonSourceFields'])
|
||||
for fields in (compact['sourceFields'], compact['comparisonSourceFields']):
|
||||
for field in fields:
|
||||
field.pop('label', None)
|
||||
field.pop('sourceField', None)
|
||||
columns = ['field', 'category', 'currentWeight', 'comparisonWeight', 'deltaWeight',
|
||||
'currentPresent', 'comparisonPresent', 'current_sourceRows', 'current_presentRows',
|
||||
'current_missingRows', 'current_invalidRows', 'current_negativeRows',
|
||||
'comparison_sourceRows', 'comparison_presentRows', 'comparison_missingRows',
|
||||
'comparison_invalidRows', 'comparison_negativeRows', 'deltaAvailable',
|
||||
'deltaUnavailableReason']
|
||||
values = []
|
||||
for pair in compact.pop('pairedRows'):
|
||||
current, comparison = pair['current'], pair['comparison']
|
||||
values.append([pair['key'], current['category'], current['weight'], comparison['weight'],
|
||||
pair['deltaWeight'], pair['present']['current'], pair['present']['comparison'],
|
||||
*[current.get(key) for key in ('sourceRows', 'presentRows', 'missingRows', 'invalidRows', 'negativeRows')],
|
||||
*[comparison.get(key) for key in ('sourceRows', 'presentRows', 'missingRows', 'invalidRows', 'negativeRows')],
|
||||
pair['deltaAvailable'], pair.get('deltaUnavailableReason')])
|
||||
compact['defectColumns'] = columns
|
||||
compact['defectValues'] = values
|
||||
compact.pop('rows', None)
|
||||
return compact
|
||||
|
||||
|
||||
def compact_category_components(result):
|
||||
compact = copy.deepcopy(result)
|
||||
for name in ('categoryReconciliation', 'comparisonCategoryReconciliation'):
|
||||
for category in compact.get(name, []):
|
||||
components = category.pop('components', None)
|
||||
if not components:
|
||||
continue
|
||||
columns = list(dict.fromkeys(key for row in components for key in row))
|
||||
category['componentColumns'] = columns
|
||||
category['componentValues'] = [[row.get(key) for key in columns] for row in components]
|
||||
return compact
|
||||
|
||||
|
||||
class QueryPlanDefectReportTests(unittest.TestCase):
|
||||
def test_single_period_report_requires_authorized_rows_and_reconciles_top_denominator(self):
|
||||
task = child_task()
|
||||
doc = document(defect_result())
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('Top 1 项 / 全范围 1 个分组', '\n'.join(report['sections']))
|
||||
self.assertIn('不等同于正式报废重量或总体贡献', report['policy'])
|
||||
self.assertEqual(len(report['requiredFactIds']), 2)
|
||||
self.assertFalse(protocol_defect_report([document(defect_result(ranking_complete=False))],
|
||||
task, RELEASE)['coverageSufficient'])
|
||||
|
||||
def test_comparison_requires_both_complete_periods_and_source_category_reconciliation(self):
|
||||
task = child_task(comparison=True)
|
||||
doc = document(defect_result(comparison=True))
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('2025年9月', '\n'.join(report['sections']))
|
||||
bad = copy.deepcopy(doc)
|
||||
bad['result']['comparisonCategoryReconciliation'][0]['isReconciled'] = False
|
||||
report = protocol_defect_report([bad], task, RELEASE)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(report['missingEvidence'])
|
||||
|
||||
def test_product_single_field_accepts_java_partial_category_reconciliation(self):
|
||||
field = source_field('镀废缺角重量', '镀废')
|
||||
task = child_task(dimension='product', stages=['bright'])
|
||||
task['defectQuery']['defect'] = field['key']
|
||||
result = defect_result(dimension='product', stage='bright', fields=[field], rows=[
|
||||
{'key': '规格A', 'label': '规格A', 'category': '镀废', 'weight': 35.055,
|
||||
'defect': field['key'], 'sourceRows': 165, 'presentRows': 165,
|
||||
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}])
|
||||
result['sourceRows'] = 165
|
||||
result['coverage']['sourceRows'] = 165
|
||||
partial = java_category_reconciliation([field], source_rows=165,
|
||||
expected_fields=5, reconciled=False, diff=-127.553,
|
||||
formal_component='bright_plating_scrap', field_weights={field['key']: 35.055},
|
||||
status='INCOMPLETE',
|
||||
reasons=['存在缺失、非法或负值,覆盖不完整',
|
||||
'来源分项与正式分类数值不一致;需核对分类转换或单位口径',
|
||||
'当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
|
||||
doc = document(result)
|
||||
doc['result']['categoryReconciliation'] = [partial]
|
||||
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('不等同于正式报废重量或总体贡献', report['policy'])
|
||||
self.assertTrue(any('不与正式大类整体回加' in fact['text'] for fact in report['facts']))
|
||||
self.assertFalse(any('相差' in fact['text'] for fact in report['facts']))
|
||||
|
||||
compact = document(compact_category_components(doc['result']))
|
||||
compact_report = protocol_defect_report([compact], task, RELEASE)
|
||||
self.assertTrue(compact_report['coverageSufficient'], compact_report['missingEvidence'])
|
||||
|
||||
def test_comparison_single_field_java_partial_reconciliation_survives_columnar_summary(self):
|
||||
task = child_task(dimension='product', comparison=True)
|
||||
result = defect_result(dimension='product', comparison=True)
|
||||
partial_current = java_category_reconciliation([source_field()], source_rows=4,
|
||||
expected_fields=7, reconciled=False,
|
||||
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
|
||||
partial_previous = java_category_reconciliation([source_field()], source_rows=3,
|
||||
expected_fields=7, reconciled=False,
|
||||
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
|
||||
result['categoryReconciliation'] = [partial_current]
|
||||
result['comparisonCategoryReconciliation'] = [partial_previous]
|
||||
compact = compact_comparison_result(result)
|
||||
doc = document(compact_category_components(compact))
|
||||
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('2025年9月', '\n'.join(report['sections']))
|
||||
|
||||
def test_partial_category_still_requires_complete_valid_selected_field_rows(self):
|
||||
task = child_task(dimension='product')
|
||||
doc = document(defect_result(dimension='product'))
|
||||
partial = java_category_reconciliation([source_field()], source_rows=4,
|
||||
expected_fields=7, reconciled=False,
|
||||
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
|
||||
partial['components'][0].update(presentRows=3, missingRows=1)
|
||||
doc['result']['categoryReconciliation'] = [partial]
|
||||
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(report['missingEvidence'])
|
||||
|
||||
invalid_weight = document(defect_result(dimension='product'))
|
||||
invalid_weight['result']['categoryReconciliation'] = [java_category_reconciliation(
|
||||
[source_field()], source_rows=4, expected_fields=7, reconciled=False,
|
||||
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])]
|
||||
invalid_weight['result']['categoryReconciliation'][0]['components'][0]['weight'] = -0.01
|
||||
invalid_report = protocol_defect_report([invalid_weight], task, RELEASE)
|
||||
self.assertFalse(invalid_report['coverageSufficient'])
|
||||
|
||||
def test_full_category_diff_is_reported_without_claiming_kpi_reconciliation(self):
|
||||
field_names = ['责废尺寸重量', '责废缺角重量', '责废垂直度重量', '责废刀丝重量',
|
||||
'责废烂毛重量', '责废开裂重量', '责废其他重量']
|
||||
fields = [source_field(key) for key in field_names]
|
||||
task = child_task()
|
||||
task['defectQuery'] = {'dimension': 'defect', 'limit': 100}
|
||||
doc = document(defect_result(fields=fields))
|
||||
diff = 1.25
|
||||
reconciliation = java_category_reconciliation(fields, source_rows=4,
|
||||
reconciled=False, diff=diff,
|
||||
reasons=['来源分项与正式分类数值不一致;需核对分类转换或单位口径'])
|
||||
reconciliation['reconciliationStatus'] = 'DIFF'
|
||||
doc['result']['categoryReconciliation'] = [reconciliation]
|
||||
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
combined = '\n'.join(report['sections'] + [fact['text'] for fact in report['facts']])
|
||||
self.assertIn('相差 +1.25 kg', combined)
|
||||
self.assertIn('不代表正式KPI贡献或已回加一致', combined)
|
||||
|
||||
def test_reconciled_flag_with_mismatched_full_category_diff_is_rejected(self):
|
||||
field_names = ['责废尺寸重量', '责废缺角重量', '责废垂直度重量', '责废刀丝重量',
|
||||
'责废烂毛重量', '责废开裂重量', '责废其他重量']
|
||||
fields = [source_field(key) for key in field_names]
|
||||
task = child_task()
|
||||
task['defectQuery'] = {'dimension': 'defect', 'limit': 100}
|
||||
doc = document(defect_result(fields=fields))
|
||||
reconciliation = java_category_reconciliation(fields, source_rows=4,
|
||||
reconciled=True, diff=1.25)
|
||||
doc['result']['categoryReconciliation'] = [reconciliation]
|
||||
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
|
||||
def test_columnar_comparison_summary_is_restored_for_report_scope_and_chart_gates(self):
|
||||
task = child_task(comparison=True)
|
||||
compact = compact_comparison_result(defect_result(comparison=True))
|
||||
doc = document(compact)
|
||||
report = protocol_defect_report([doc], task, RELEASE)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('2025年9月', '\n'.join(report['sections']))
|
||||
selected = scope_documents([doc], scope(task), release_id=RELEASE)
|
||||
self.assertEqual(len(selected), 1)
|
||||
visual = protocol_visual_result([doc], [scope(task)], QUESTION, 'turn-defect-compare',
|
||||
release_id=RELEASE)
|
||||
self.assertEqual(visual['visualState'], 'ready', visual['missingEvidence'])
|
||||
self.assertEqual(visual['visuals'][0]['metric'], 'source_defect_delta_weight')
|
||||
|
||||
def test_malformed_columnar_comparison_is_rejected_by_report_and_scope(self):
|
||||
task = child_task(comparison=True)
|
||||
compact = compact_comparison_result(defect_result(comparison=True))
|
||||
compact['defectValues'][0].pop()
|
||||
doc = document(compact)
|
||||
self.assertFalse(protocol_defect_report([doc], task, RELEASE)['coverageSufficient'])
|
||||
self.assertEqual(scope_documents([doc], scope(task), release_id=RELEASE), [])
|
||||
|
||||
def test_missing_required_stage_is_partial_and_does_not_create_zero_rows(self):
|
||||
task = child_task(stages=['black', 'bright'])
|
||||
report = protocol_defect_report([document(defect_result())], task, RELEASE)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(any('亮片' in gap for gap in report['missingEvidence']))
|
||||
self.assertNotIn('0 kg', '\n'.join(report['sections']))
|
||||
|
||||
def test_scoped_coverage_reuses_defect_specific_gate_and_exact_proofs(self):
|
||||
task = child_task()
|
||||
doc = document(defect_result())
|
||||
self.assertEqual(scope_documents([doc], scope(task), release_id=RELEASE), [doc])
|
||||
self.assertTrue(coverage_sufficient_scoped([doc], [scope(task)], QUESTION, RELEASE))
|
||||
bad = copy.deepcopy(doc)
|
||||
bad['result']['protocolEvidence'][0]['scopeId'] = 'another-scope'
|
||||
self.assertEqual(scope_documents([bad], scope(task), release_id=RELEASE), [])
|
||||
self.assertFalse(coverage_sufficient_scoped([bad], [scope(task)], QUESTION, RELEASE))
|
||||
|
||||
|
||||
class QueryPlanDefectVisualTests(unittest.TestCase):
|
||||
def run_visual(self, visual_type):
|
||||
task = child_task(dimension='product', visual_type=visual_type)
|
||||
result = defect_result(dimension='product', rows=[
|
||||
{'key': '规格A', 'label': '规格A', 'category': '责废', 'defect': FIELD,
|
||||
'weight': 4.5, 'sourceRows': 4, 'presentRows': 4, 'missingRows': 0,
|
||||
'invalidRows': 0, 'negativeRows': 0}
|
||||
])
|
||||
doc = document(result)
|
||||
return protocol_visual_result([doc], [scope(task)], QUESTION, 'turn-defect',
|
||||
release_id=RELEASE)
|
||||
|
||||
def test_product_defect_weight_chart_is_bound_to_exact_field_and_dimension(self):
|
||||
result = self.run_visual('bar')
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
self.assertEqual(result['missingEvidence'], [])
|
||||
chart = result['visuals'][0]
|
||||
self.assertEqual(chart['dimension'], 'product')
|
||||
self.assertEqual(chart['metric'], 'source_defect_weight')
|
||||
self.assertEqual(chart['requiredFields'], [FIELD])
|
||||
self.assertEqual(chart['scope']['defect'], FIELD)
|
||||
self.assertIn('不代表正式报废大类或总体贡献', chart['note'])
|
||||
|
||||
def test_unsupported_explicit_pareto_is_partial_not_silently_changed_to_bar(self):
|
||||
result = self.run_visual('pareto')
|
||||
self.assertEqual(result['visuals'], [])
|
||||
self.assertEqual(result['visualState'], 'unavailable')
|
||||
self.assertTrue(any('图' in gap for gap in result['missingEvidence']))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,179 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
||||
str(Path(__file__).resolve().parent)]
|
||||
|
||||
from nao_agent_task import normalize_task
|
||||
from nao_protocol_execution import execute
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_protocol_execution import BoundRankingApi, RELEASE, TURN, identity
|
||||
|
||||
|
||||
def _scope_task(root, *, purpose, fields, stages, selector_kind, product=None,
|
||||
dimension=None, visual_type=None, dimension_stages=None):
|
||||
filters = copy.deepcopy(root['filters'])
|
||||
if product is not None:
|
||||
filters['product'] = product
|
||||
task = {'version': 1, 'kind': 'analysis' if purpose == 'ranking' else 'metrics',
|
||||
'question': root['question'], 'goal': root['goal'],
|
||||
'period': copy.deepcopy(root['period']), 'comparisonPeriod': None,
|
||||
'filters': filters, 'dimensions': [dimension] if purpose == 'ranking' else [],
|
||||
'dimensionStages': ({dimension: list(stages)} if purpose == 'ranking' else {}),
|
||||
'fields': list(fields), 'stages': list(stages),
|
||||
'trendRequired': purpose == 'trend', 'comparisonRequired': False,
|
||||
'visualPreference': 'auto', 'visualType': visual_type,
|
||||
'periodCoverage': root.get('periodCoverage', 'available'),
|
||||
'freshQuery': True, 'historyMode': 'none'}
|
||||
return task
|
||||
|
||||
|
||||
class QueryPlanApi(BoundRankingApi):
|
||||
def __init__(self, *, bad_node_proof=False, missing_second_child=False):
|
||||
super().__init__()
|
||||
self.rank_done = False
|
||||
self.envelopes = []
|
||||
self.bad_node_proof = bad_node_proof
|
||||
self.missing_second_child = missing_second_child
|
||||
self.root = None
|
||||
self.plan = None
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
rank_node = next(node for node in self.plan if node['nodeId'] == 'rank')
|
||||
trend_node = next(node for node in self.plan if node['nodeId'] == 'top_trend')
|
||||
rank_scope = {'nodeId': 'rank', 'scopeId': 'rank-scope', 'label': '黑片投入重量排名',
|
||||
'purpose': 'ranking', 'limit': 5,
|
||||
'task': _scope_task(self.root, purpose='ranking', fields=rank_node['fields'],
|
||||
stages=rank_node['stages'], selector_kind='base', dimension='product',
|
||||
visual_type='pareto')}
|
||||
scopes = [rank_scope]
|
||||
pending = [{'nodeId': 'top_trend', 'reason': '等待正式排名结果'}]
|
||||
if self.rank_done:
|
||||
scopes = [rank_scope]
|
||||
for suffix, product in [('a', 'product a'), ('b', 'product b')]:
|
||||
scopes.append({'nodeId': 'top_trend', 'scopeId': 'trend-' + suffix,
|
||||
'label': product, 'purpose': 'trend',
|
||||
'task': _scope_task(self.root, purpose='trend', fields=trend_node['fields'],
|
||||
stages=trend_node['stages'], selector_kind='ranking_top', product=product,
|
||||
visual_type='line')})
|
||||
if self.missing_second_child:
|
||||
scopes.pop()
|
||||
pending = []
|
||||
else:
|
||||
pending = []
|
||||
return {'version': 1, 'scopes': scopes, 'pending': pending, 'gaps': []}
|
||||
|
||||
response = super().call(path, payload, method)
|
||||
if path.startswith('/api/metrics'):
|
||||
if isinstance(response.get('summary'), dict):
|
||||
response['summary'].setdefault('black', {})['scrapRate'] = 0.05
|
||||
for month in response.get('trend', []):
|
||||
if isinstance(month, dict):
|
||||
month.setdefault('black', {})['scrapRate'] = 0.05
|
||||
parsed = parse_qs(urlsplit(path).query)
|
||||
proof = response.get('protocolEvidence')
|
||||
if isinstance(proof, dict):
|
||||
proof['nodeId'] = 'untrusted' if self.bad_node_proof else node_id
|
||||
proof['scopeId'] = scope_id
|
||||
proof['queryId'] = proof['queryId'] + '-' + str(len(self.envelopes))
|
||||
proof['responseHash'] = f'{len(self.envelopes):064x}'
|
||||
if parsed.get('view') == ['product']:
|
||||
self.rank_done = True
|
||||
return response
|
||||
|
||||
|
||||
class QueryPlanExecutionTests(unittest.TestCase):
|
||||
def task(self):
|
||||
plan = [
|
||||
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 5,
|
||||
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
|
||||
'visualType': 'pareto', 'visualPreference': 'auto'},
|
||||
{'nodeId': 'top_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
|
||||
'stages': ['black'], 'selector': {'kind': 'ranking_top', 'sourceNodeId': 'rank',
|
||||
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
|
||||
'visualType': 'line', 'visualPreference': 'auto'},
|
||||
]
|
||||
value = task_value(kind='analysis', queryPlan=plan, dimensions=['product'],
|
||||
dimensionStages={'product': ['black']}, stages=['black'],
|
||||
fields=['totalWeight', 'scrapRate'], trendRequired=True,
|
||||
comparisonRequired=False, period={'year': 2026}, filterUpdates={'department': '部门甲'})
|
||||
return normalize_task(value, '按黑片投入重量排名后查看前两项报废率趋势',
|
||||
{'_protocol': {'version': 1}})
|
||||
|
||||
def run_query(self, api, *, final_answer='已核对排名和趋势'):
|
||||
task = self.task()
|
||||
api.root = task
|
||||
api.plan = task['queryPlan']
|
||||
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize', return_value={
|
||||
'answer': final_answer, 'evidenceSufficient': True,
|
||||
'missingEvidence': [], 'usage': {}}) as synth:
|
||||
result = execute(request=request, task=task,
|
||||
authority={**identity(), 'task': task}, actor_root=root, delegated=api,
|
||||
connection=('offline', 'synthetic', 'model'),
|
||||
cancel_event=threading.Event(), emit=lambda _: None)
|
||||
documents = [json.loads(path.read_text()) for path in (root / 'evidence').glob('nao-quality-*.json')]
|
||||
return result, documents, synth
|
||||
|
||||
def test_rank_then_two_server_resolved_trends_use_scoped_envelopes_and_one_finalizer(self):
|
||||
api = QueryPlanApi()
|
||||
result, documents, synth = self.run_query(api)
|
||||
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertEqual(result['stopReason'], 'evidence_finalized')
|
||||
self.assertEqual(result['completionMethod'], 'bounded_scoped_query_plan')
|
||||
self.assertEqual(result['finishReason'], 'stop')
|
||||
self.assertEqual({doc['mode'] for doc in documents}, {'rank', 'query'})
|
||||
self.assertEqual(len(documents), 3)
|
||||
self.assertEqual(len(result['protocolEvidence']), 3)
|
||||
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'rank', 'top_trend'})
|
||||
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
|
||||
{'rank-scope', 'trend-a', 'trend-b'})
|
||||
expected_agent_context = {'releaseId': RELEASE,
|
||||
'filters': api.root['filters'],
|
||||
'period': api.root['period']}
|
||||
if api.root.get('comparisonPeriod') is not None:
|
||||
expected_agent_context['comparisonPeriod'] = api.root['comparisonPeriod']
|
||||
self.assertEqual(result['agentContext'], expected_agent_context)
|
||||
self.assertNotIn('_protocolScopes', result['agentContext'])
|
||||
self.assertNotIn('product', result['agentContext']['filters'])
|
||||
self.assertTrue(all(call['nodeId'] is not None and call['scopeId'] is not None
|
||||
for call in api.envelopes if call['path'] != '/api/analysis/protocol/queries'))
|
||||
self.assertEqual(len([call for call in api.envelopes
|
||||
if call['path'] == '/api/analysis/protocol/queries']), 3)
|
||||
synth.assert_called_once()
|
||||
context = synth.call_args.kwargs['context']
|
||||
self.assertEqual(context['filters'], api.root['filters'])
|
||||
self.assertEqual(len(context['_protocolScopes']['scopes']), 3)
|
||||
|
||||
def test_forged_scope_proof_and_missing_resolved_child_finish_partial(self):
|
||||
for api in (QueryPlanApi(bad_node_proof=True), QueryPlanApi(missing_second_child=True)):
|
||||
with self.subTest(api=api):
|
||||
result, documents, _ = self.run_query(api)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
def test_requested_scope_visual_gap_finishes_partial_even_when_text_evidence_is_complete(self):
|
||||
api = QueryPlanApi()
|
||||
with patch('analysis_visual_evidence.protocol_visual_result', return_value={
|
||||
'visualSchemaVersion': 1, 'visualState': 'ready', 'visuals': [],
|
||||
'missingEvidence': ['节点趋势图未覆盖。']}):
|
||||
result, _, _ = self.run_query(api)
|
||||
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'], True)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertIn('节点趋势图未覆盖。', result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,473 @@
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from nao_agent_task import TaskUnavailable, normalize_task
|
||||
from nao_protocol_execution import execute
|
||||
from nao_query_contract import bind_query
|
||||
from nao_query_plan import normalize_query_plan
|
||||
from nao_quality_tools import QualityToolError, QualityTools
|
||||
from nao_evidence_finalizer import _resolve_completed_protocol_raw_gaps
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
|
||||
|
||||
|
||||
class RawPlanTests(unittest.TestCase):
|
||||
def root(self, **overrides):
|
||||
value = {'period':{'year':2026,'monthStart':1,'monthEnd':2},
|
||||
'comparisonPeriod':None,'filters':{'department':'部门甲'},
|
||||
'fields':[],'dimensions':[],'stages':[],
|
||||
'trendRequired':False,'comparisonRequired':False,
|
||||
'verifySource':True,'fullDetails':True,'periodCoverage':'available'}
|
||||
value.update(overrides)
|
||||
return value
|
||||
|
||||
def plan(self):
|
||||
return [
|
||||
{'nodeId':'source','purpose':'source','fields':[],'stages':['black'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
{'nodeId':'records','purpose':'records','fields':[],'stages':['black'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
]
|
||||
|
||||
def test_raw_node_union_sets_root_stage_and_flags_without_kpi_fields(self):
|
||||
root = self.root()
|
||||
result = normalize_query_plan(self.plan(), root_task=root, root_value=self.root())
|
||||
self.assertEqual([node['purpose'] for node in result], ['source','records'])
|
||||
self.assertEqual(root['fields'], [])
|
||||
self.assertEqual(root['stages'], ['black'])
|
||||
self.assertTrue(root['verifySource'])
|
||||
self.assertTrue(root['fullDetails'])
|
||||
self.assertEqual(root['dimensions'], [])
|
||||
|
||||
def test_raw_flags_cannot_be_omitted_or_downgraded(self):
|
||||
cases = [
|
||||
(self.root(verifySource=False), self.plan()),
|
||||
(self.root(fullDetails=False), self.plan()),
|
||||
(self.root(fullDetails=True), self.plan()[:1]),
|
||||
]
|
||||
for root, plan in cases:
|
||||
with self.subTest(root=root, plan=plan), self.assertRaises(ValueError):
|
||||
normalize_query_plan(plan, root_task=copy.deepcopy(root), root_value=root)
|
||||
|
||||
def test_raw_nodes_require_empty_fields_and_no_visuals(self):
|
||||
for update in ({'fields':['rate']}, {'visualType':'bar'}, {'visualPreference':'auto'},
|
||||
{'stages':[]}):
|
||||
raw = self.plan()[:1]
|
||||
raw[0].update(update)
|
||||
with self.subTest(update=update), self.assertRaises(ValueError):
|
||||
normalize_query_plan(raw, root_task=self.root(fullDetails=False),
|
||||
root_value=self.root(fullDetails=False))
|
||||
|
||||
def test_source_fields_are_stage_specific_authorized_stable_keys(self):
|
||||
catalog = {'sourceFields':[
|
||||
{'key':'order_no','type':'source_value','stages':['black'],
|
||||
'columns':{'black':'委托单号'}},
|
||||
{'key':'batch_code','type':'source_value','stages':['bright'],
|
||||
'columns':{'bright':'批次号'}},
|
||||
]}
|
||||
plan = self.plan()[:1]
|
||||
plan[0]['sourceFields'] = {'black':['order_no']}
|
||||
root = self.root(fullDetails=False)
|
||||
result = normalize_query_plan(plan, root_task=root, root_value=root,
|
||||
context={'_businessCatalog':catalog})
|
||||
self.assertEqual(result[0]['sourceFields'], {'black':['order_no']})
|
||||
for selected in ({'black':['missing']}, {'bright':['batch_code']},
|
||||
{'black':['order_no','order_no']}):
|
||||
bad = copy.deepcopy(plan)
|
||||
bad[0]['sourceFields'] = selected
|
||||
with self.subTest(selected=selected), self.assertRaises(ValueError):
|
||||
normalize_query_plan(bad, root_task=self.root(fullDetails=False),
|
||||
root_value=self.root(fullDetails=False),
|
||||
context={'_businessCatalog':catalog})
|
||||
|
||||
def test_legacy_pure_raw_kinds_synthesize_query_plan_and_keep_full_goal(self):
|
||||
source = task_value(kind='source_verify',dimensions=[],fields=[],stages=['black'],
|
||||
verifySource=True,fullDetails=False,period={'year':2026})
|
||||
records = task_value(kind='details',dimensions=[],fields=[],stages=['bright'],
|
||||
verifySource=False,fullDetails=True,period={'year':2026})
|
||||
for value, purpose in ((source,'source'),(records,'records')):
|
||||
task = normalize_task(value,'完整核验当前原始数据',
|
||||
{'_protocol':{'version':1}})
|
||||
self.assertEqual(len(task['queryPlan']),1)
|
||||
self.assertEqual(task['queryPlan'][0]['purpose'],purpose)
|
||||
self.assertEqual(task['queryPlan'][0]['fields'],[])
|
||||
self.assertEqual(task['queryPlan'][0]['stages'],[value['stages'][0]])
|
||||
self.assertEqual(task['verifySource'],purpose=='source')
|
||||
self.assertEqual(task['fullDetails'],purpose=='records')
|
||||
|
||||
def test_legacy_metrics_with_explicit_raw_requirements_keeps_metric_and_scope(self):
|
||||
value=task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],
|
||||
verifySource=True,fullDetails=True,period={'year':2026},
|
||||
filterUpdates={'department':'部门甲'},trendRequired=False,
|
||||
comparisonRequired=False)
|
||||
task=normalize_task(value,'在部门甲范围分析综合合格率并核对原表、读取所有记录',
|
||||
{'_protocol':{'version':1}})
|
||||
by_purpose={node['purpose']:node for node in task['queryPlan']}
|
||||
self.assertEqual(by_purpose['metrics']['fields'],['rate'])
|
||||
self.assertEqual(by_purpose['metrics']['stages'],[])
|
||||
self.assertEqual(by_purpose['source']['fields'],[])
|
||||
self.assertEqual(by_purpose['records']['fields'],[])
|
||||
self.assertEqual(task['fields'],['rate'])
|
||||
self.assertEqual(task['filters']['department'],'部门甲')
|
||||
self.assertTrue(task['verifySource'])
|
||||
self.assertTrue(task['fullDetails'])
|
||||
|
||||
def test_scoped_investigate_binds_month_range_columns_exclusions_and_stage(self):
|
||||
columns = [{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}]
|
||||
task = {'filters':{'department':'部门甲','exclusions':{'product':['P-2']},
|
||||
'columnFilters':columns},
|
||||
'dimensions':[],'stages':['black'],'dimensionStages':{},
|
||||
'period':{'year':2026,'monthStart':1,'monthEnd':2},
|
||||
'comparisonPeriod':None}
|
||||
args = {'mode':'investigate','action':'rows','releaseId':RELEASE,
|
||||
'period':copy.deepcopy(task['period']),
|
||||
'filters':{'department':'部门甲','exclusions':{'product':['P-2']}},
|
||||
'columnFilters':copy.deepcopy(columns),'stage':'black',
|
||||
'limit':100,'offset':0,'recordFilter':'all'}
|
||||
bound = bind_query(args, {'task':task,'releaseId':RELEASE,'exactScope':True})
|
||||
self.assertEqual(bound['period'],task['period'])
|
||||
self.assertEqual(bound['filters']['exclusions'],task['filters']['exclusions'])
|
||||
self.assertEqual(bound['columnFilters'],columns)
|
||||
self.assertEqual(bound['stage'],'black')
|
||||
|
||||
|
||||
class InvestigationApi:
|
||||
def __init__(self, *, total_rows=205, missing_proof=False, wrong_context_key=None):
|
||||
self.total_rows = total_rows
|
||||
self.missing_proof = missing_proof
|
||||
self.wrong_context_key = wrong_context_key
|
||||
self.calls = []
|
||||
self.query_count = 0
|
||||
self.scopes = []
|
||||
|
||||
@staticmethod
|
||||
def source_fields():
|
||||
return [{'key':'customer_code','type':'source_value','stages':['black'],
|
||||
'columns':{'black':'客户代码'},'operators':['in']}]
|
||||
|
||||
def add_scope(self, node_id, scope_id, purpose, root):
|
||||
child = copy.deepcopy(root)
|
||||
child.pop('queryPlan',None)
|
||||
child['kind'] = 'source_verify' if purpose == 'source' else 'details'
|
||||
child['fields'] = []
|
||||
child['dimensions'] = []
|
||||
child['dimensionStages'] = {}
|
||||
child['stages'] = ['black']
|
||||
child['trendRequired'] = False
|
||||
child['comparisonRequired'] = False
|
||||
child['visualType'] = None
|
||||
child['visualPreference'] = 'none'
|
||||
child['verifySource'] = purpose == 'source'
|
||||
child['fullDetails'] = purpose == 'records'
|
||||
child['investigationFields'] = {'black':['委托单号','客户代码']}
|
||||
self.scopes.append({'nodeId':node_id,'scopeId':scope_id,'label':scope_id,
|
||||
'purpose':purpose,'task':child})
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
self.calls.append({'path':path,'payload':copy.deepcopy(payload),
|
||||
'method':method,'nodeId':node_id,'scopeId':scope_id})
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'version':1,'scopes':copy.deepcopy(self.scopes),'pending':[],'gaps':[]}
|
||||
if path == '/api/releases':
|
||||
return {'items':[{'id':RELEASE,'name':'Release','status':'PUBLISHED'}]}
|
||||
if path == '/api/analysis/tools/catalog':
|
||||
return {'kind':'analysis_catalog','sourceFields':self.source_fields()}
|
||||
if path != '/api/analysis/investigate':
|
||||
raise AssertionError(path)
|
||||
self.query_count += 1
|
||||
expected_period = {key:payload[key] for key in
|
||||
('year','month','quarter','monthStart','monthEnd') if key in payload}
|
||||
context = {**expected_period,'releaseId':RELEASE,'stage':payload['stage'],
|
||||
'category':None,'sourceRevisionId':None,'includeUnassigned':False,
|
||||
'department':payload.get('department'),'group':payload.get('group'),
|
||||
'product':payload.get('product'),'projectGroup':payload.get('projectGroup'),
|
||||
'extraFilters':payload.get('extraFilters',{}),
|
||||
'columnFilters':payload.get('columnFilters',[]),
|
||||
'exclusions':payload.get('exclusions',{})}
|
||||
if self.wrong_context_key:
|
||||
context[self.wrong_context_key] = 'OUT_OF_SCOPE_SENTINEL'
|
||||
proof = {'queryId':f'raw-{self.query_count}','attemptId':identity()['attemptId'],
|
||||
'sessionId':identity()['sessionId'],'turnId':identity()['turnId'],
|
||||
'revision':identity()['revision'],'executionId':identity()['executionId'],
|
||||
'responseHash':f'{self.query_count:064x}','nodeId':node_id,'scopeId':scope_id}
|
||||
action = payload['action']
|
||||
response = {'kind':'quality_investigation','action':action,'releaseId':RELEASE,
|
||||
'context':context,'totalRows':self.total_rows,'aggregateComplete':True,
|
||||
'categories':[],'lineage':[]}
|
||||
if action == 'source':
|
||||
response['sourceVerification'] = {'expectedRows':self.total_rows,
|
||||
'complete':self.total_rows > 0,'verifiedRows':self.total_rows,
|
||||
'missingRows':0,'mismatchRows':0,'allSelectedValuesMatchStored':self.total_rows > 0,
|
||||
'files':[]}
|
||||
else:
|
||||
offset = payload.get('offset',0)
|
||||
limit = payload['limit']
|
||||
count = min(limit,max(0,self.total_rows-offset))
|
||||
rows = [{'rowNumber':offset+index+1,'product':'规格A',
|
||||
'values':{'委托单号':f'ORDER-{offset+index+1}',
|
||||
'客户代码':'PRIVATE_RAW_VALUE'}}
|
||||
for index in range(count)]
|
||||
response.update(rows=rows,offset=offset,returnedRows=count,
|
||||
matchingRows=self.total_rows,hasMore=offset+count<self.total_rows,
|
||||
nextOffset=offset+count if offset+count<self.total_rows else None)
|
||||
if not self.missing_proof:
|
||||
response['protocolEvidence'] = proof
|
||||
return response
|
||||
|
||||
|
||||
class RawPlanExecutionTests(unittest.TestCase):
|
||||
def root_task(self, *, purposes=('source','records'), with_column_filters=True):
|
||||
nodes = []
|
||||
for purpose in purposes:
|
||||
nodes.append({'nodeId':purpose,'purpose':purpose,'fields':[],'stages':['black'],
|
||||
'selector':{'kind':'base'},'visualType':None,
|
||||
'visualPreference':'none','comparisonRequired':False})
|
||||
columns=[{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}]
|
||||
value = task_value(kind='analysis',dimensions=[],stages=['black'],fields=[],
|
||||
verifySource='source' in purposes,fullDetails='records' in purposes,
|
||||
period={'year':2026,'monthStart':1,'monthEnd':2},queryPlan=nodes,
|
||||
**({'columnFilters':columns} if with_column_filters else {}),
|
||||
filterUpdates={'exclusions':{'product':['P-2']}},
|
||||
comparisonRequired=False,trendRequired=False)
|
||||
return normalize_task(value,'核验原表并读取完整记录',
|
||||
{'_protocol':{'version':1},
|
||||
'_businessCatalog':{'sourceFields':InvestigationApi.source_fields()}})
|
||||
|
||||
def run_query(self, api, purposes=('source','records'), *, with_column_filters=True,
|
||||
return_completion=False):
|
||||
task = self.root_task(purposes=purposes,with_column_filters=with_column_filters)
|
||||
if not with_column_filters:
|
||||
task['filters'].pop('columnFilters',None)
|
||||
for purpose in purposes:
|
||||
api.add_scope(purpose,purpose+'-scope',purpose,task)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize') as synth:
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
|
||||
'question':task['question']},task=task,
|
||||
authority={**identity(),'task':task},actor_root=root,delegated=api,
|
||||
connection=('http://offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
documents=[json.loads(path.read_text()) for path in (root/'evidence').glob('nao-quality-*.json')]
|
||||
completion=json.loads((root/'private'/'protocol-completion.json').read_text())
|
||||
return (result,documents,synth,completion) if return_completion else (result,documents,synth)
|
||||
|
||||
def test_raw_nodes_use_server_columns_envelopes_and_read_all_stable_pages(self):
|
||||
api=InvestigationApi(total_rows=205)
|
||||
result,documents,synth=self.run_query(api)
|
||||
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertEqual(result['stopReason'],'evidence_finalized')
|
||||
self.assertEqual(result['completionMethod'],'bounded_scoped_query_plan')
|
||||
self.assertEqual(result['finishReason'],'stop')
|
||||
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertEqual(len(result['protocolEvidence']),4)
|
||||
self.assertFalse(synth.called)
|
||||
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
|
||||
self.assertEqual(len(reads),4)
|
||||
self.assertEqual({(call['nodeId'],call['scopeId']) for call in reads},
|
||||
{('source','source-scope'),('records','records-scope')})
|
||||
self.assertEqual([call['payload']['offset'] for call in reads if call['payload']['action']=='rows'],
|
||||
[0,100,200])
|
||||
self.assertTrue(all(call['payload']['includeUnassigned'] is False for call in reads))
|
||||
self.assertTrue(all(call['payload']['limit']==100 for call in reads))
|
||||
self.assertTrue(all(call['payload']['fields']==['委托单号','客户代码'] for call in reads))
|
||||
self.assertTrue(all(call['payload']['columnFilters']==[
|
||||
{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}
|
||||
] for call in reads))
|
||||
self.assertTrue(all(call['payload']['exclusions']=={'product':['P-2']} for call in reads))
|
||||
self.assertFalse(any(call['payload'].get('recordFilter') for call in reads
|
||||
if call['payload']['action']=='source'))
|
||||
self.assertFalse(any('PRIVATE_RAW_VALUE' in json.dumps(value,ensure_ascii=False)
|
||||
for value in (result['answer'],result['scopeDiagnostics'])))
|
||||
self.assertNotIn('recordList',result)
|
||||
self.assertEqual({doc['mode'] for doc in documents},{'investigate'})
|
||||
|
||||
def test_unselected_column_filters_accept_canonical_empty_context_echo(self):
|
||||
api=InvestigationApi(total_rows=2)
|
||||
result,_,_=self.run_query(api,with_column_filters=False)
|
||||
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
|
||||
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
|
||||
self.assertTrue(reads)
|
||||
self.assertTrue(all('columnFilters' not in call['payload'] for call in reads))
|
||||
|
||||
def test_failed_raw_read_keeps_private_safe_error_diagnostic(self):
|
||||
api=InvestigationApi(total_rows=2,wrong_context_key='product')
|
||||
result,_,_,completion=self.run_query(api,return_completion=True)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertEqual(result['stopReason'],'evidence_incomplete')
|
||||
self.assertEqual(result['completionMethod'],'bounded_scoped_query_plan')
|
||||
self.assertEqual(result['finishReason'],'stop')
|
||||
diagnostic=next(item for item in completion['queryDiagnostics']
|
||||
if item.get('nodeId')=='records')
|
||||
self.assertEqual(diagnostic['errorType'],'QualityToolError')
|
||||
self.assertIn('未确认product范围',diagnostic['errorSummary'])
|
||||
self.assertNotIn('P-2',diagnostic['errorSummary'])
|
||||
self.assertNotIn('C01',diagnostic['errorSummary'])
|
||||
self.assertNotIn('OUT_OF_SCOPE_SENTINEL',diagnostic['errorSummary'])
|
||||
self.assertNotIn('QualityToolError',result['answer'])
|
||||
self.assertNotIn('OUT_OF_SCOPE_SENTINEL',result['answer'])
|
||||
|
||||
def test_records_stop_at_global_4000_row_bound_and_finish_partial(self):
|
||||
api=InvestigationApi(total_rows=4500)
|
||||
result,documents,_=self.run_query(api,purposes=('records',))
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'],False)
|
||||
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
|
||||
self.assertEqual(len(reads),40)
|
||||
self.assertEqual(sum(call['payload'].get('limit',0) for call in reads
|
||||
if call['payload']['action']=='rows'),4000)
|
||||
self.assertTrue(any('4000条' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
|
||||
def test_missing_raw_protocol_proof_is_partial_and_never_promoted_to_complete(self):
|
||||
api=InvestigationApi(total_rows=2,missing_proof=True)
|
||||
result,documents,_=self.run_query(api)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertFalse(result['protocolEvidence'])
|
||||
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
def test_only_completed_raw_requirement_gaps_are_resolved(self):
|
||||
raw_gaps=[
|
||||
'该范围内逐条原始记录(客户代码、委托订单号、加工单位,不去重)未返回可用事实',
|
||||
'上述字段与归档原始Excel的一致性核验结果未返回可用事实',
|
||||
'亮片报废率仍缺少正式来源月份',
|
||||
]
|
||||
complete={'records':{'requiredChecks':1,'verifiedChecks':1,'complete':True},
|
||||
'source':{'requiredChecks':1,'verifiedChecks':1,'complete':True}}
|
||||
retained,resolved=_resolve_completed_protocol_raw_gaps(raw_gaps,complete)
|
||||
self.assertEqual(retained,[raw_gaps[2]])
|
||||
self.assertEqual(resolved,raw_gaps[:2])
|
||||
incomplete={'records':complete['records'],
|
||||
'source':{'requiredChecks':1,'verifiedChecks':0,'complete':False}}
|
||||
retained,resolved=_resolve_completed_protocol_raw_gaps(raw_gaps,incomplete)
|
||||
self.assertEqual(retained,raw_gaps[1:])
|
||||
self.assertEqual(resolved,[raw_gaps[0]])
|
||||
|
||||
|
||||
class MixedRawMetricsApi(BoundApi):
|
||||
def __init__(self, root, plan):
|
||||
super().__init__()
|
||||
self.plan=plan
|
||||
self.raw_api=InvestigationApi(total_rows=165)
|
||||
self.scopes=[]
|
||||
for node in plan:
|
||||
purpose=node['purpose']
|
||||
child=copy.deepcopy(root)
|
||||
child.pop('queryPlan',None)
|
||||
child['kind']={'source':'source_verify','records':'details'}.get(purpose,'metrics')
|
||||
child['fields']=copy.deepcopy(node['fields'])
|
||||
child['dimensions']=[]
|
||||
child['dimensionStages']={}
|
||||
child['stages']=copy.deepcopy(node['stages'])
|
||||
child['trendRequired']=purpose=='trend'
|
||||
child['comparisonRequired']=node['comparisonRequired']
|
||||
child['visualType']=None
|
||||
child['visualPreference']='none'
|
||||
child['verifySource']=purpose=='source'
|
||||
child['fullDetails']=purpose=='records'
|
||||
if purpose in ('source','records'):
|
||||
child['investigationFields']={stage:['客户代码','委托订单号','加工单位']
|
||||
for stage in node['stages']}
|
||||
self.scopes.append({'nodeId':node['nodeId'],'scopeId':purpose+'-scope',
|
||||
'label':purpose,'purpose':purpose,'task':child})
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
if path=='/api/analysis/protocol/queries':
|
||||
return {'version':1,'scopes':copy.deepcopy(self.scopes),'pending':[],'gaps':[]}
|
||||
if path=='/api/analysis/investigate':
|
||||
return self.raw_api.call(path,payload,method,node_id=node_id,scope_id=scope_id)
|
||||
result=super().call(path,payload,method)
|
||||
if path.startswith('/api/metrics'):
|
||||
result['summary']['bright']['scrapRate']=0.081
|
||||
for month in result.get('coverage',{}).get('sourceMonths',[]):
|
||||
month['blackRows']=10
|
||||
month['brightRows']=12
|
||||
proof=result.get('protocolEvidence')
|
||||
if isinstance(proof,dict):
|
||||
proof.update(nodeId=node_id,scopeId=scope_id,queryId='mixed-kpi',responseHash='b'*64)
|
||||
return result
|
||||
|
||||
|
||||
class MixedRawPlanExecutionTests(unittest.TestCase):
|
||||
def test_mixed_plan_synthesizes_only_kpi_evidence_and_platform_reports_raw_completion(self):
|
||||
plan=[
|
||||
{'nodeId':'records','purpose':'records','fields':[],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
{'nodeId':'verify','purpose':'source','fields':[],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
{'nodeId':'metric','purpose':'metrics','fields':['scrapRate'],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
]
|
||||
value=task_value(kind='details',dimensions=[],stages=['bright'],fields=['scrapRate'],
|
||||
verifySource=True,fullDetails=True,period={'year':2026,'month':9},queryPlan=plan,
|
||||
visualPreference='none',visualType=None,comparisonRequired=False,trendRequired=False)
|
||||
task=normalize_task(value,'查询2026年9月亮片报废率,并读取该范围逐条原始记录及核验归档原件',
|
||||
{'_protocol':{'version':1}})
|
||||
api=MixedRawMetricsApi(task,task['queryPlan'])
|
||||
for node in task['queryPlan']:
|
||||
self.assertIn(node['purpose'],{'records','source','metrics'})
|
||||
raw_model_gaps=[
|
||||
'该范围内逐条原始记录(客户代码、委托订单号、加工单位,不去重)未返回可用事实',
|
||||
'上述字段与归档原始Excel的一致性核验结果未返回可用事实',
|
||||
]
|
||||
model_requests=[]
|
||||
def model_opener(request,timeout):
|
||||
body=json.loads(request.data.decode())
|
||||
model_requests.append(body)
|
||||
selector=json.loads(body['messages'][1]['content'])
|
||||
fact_id=selector['facts'][0]['id']
|
||||
completion={'model':'model','usage':{},'choices':[{'finish_reason':'stop','message':{
|
||||
'content':json.dumps({'reportTitle':'2026年9月亮片报废率',
|
||||
'answerBlocks':[{'type':'paragraph','factIds':[fact_id]}],
|
||||
'evidenceSufficient':False,'missingEvidence':raw_model_gaps},ensure_ascii=False)}}]}
|
||||
return io.BytesIO(json.dumps(completion,ensure_ascii=False).encode())
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer._open_model',side_effect=model_opener):
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
|
||||
'question':task['question']},task=task,
|
||||
authority={**identity(),'task':task},actor_root=root,delegated=api,
|
||||
connection=('http://offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
docs=[json.loads(path.read_text()) for path in (root/'evidence').glob('nao-quality-*.json')]
|
||||
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertTrue(task['verifySource'])
|
||||
self.assertTrue(task['fullDetails'])
|
||||
self.assertEqual(len(result['protocolEvidence']),4)
|
||||
self.assertEqual(result['evidenceCompletion']['missingEvidence'],[])
|
||||
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
|
||||
self.assertEqual(len(model_requests),1)
|
||||
selector=json.loads(model_requests[0]['messages'][1]['content'])
|
||||
self.assertEqual([item['kind'] for item in selector['coverage']],['quality_metrics'])
|
||||
raw_summary=selector['protocolRawCoverage']
|
||||
self.assertTrue(selector['agentTask']['verifySource'])
|
||||
self.assertTrue(selector['agentTask']['fullDetails'])
|
||||
self.assertEqual(raw_summary,{
|
||||
'source':{'requiredChecks':1,'verifiedChecks':1,'complete':True},
|
||||
'records':{'requiredChecks':1,'verifiedChecks':1,'complete':True}})
|
||||
self.assertIn('source/records节点由平台独立执行',model_requests[0]['messages'][0]['content'])
|
||||
self.assertNotIn('PRIVATE_RAW_VALUE',model_requests[0]['messages'][1]['content'])
|
||||
self.assertEqual({document['mode'] for document in docs},{'query','investigate'})
|
||||
self.assertIn('已按稳定分页读全165条',result['answer'])
|
||||
self.assertIn('原表值与存储记录一致',result['answer'])
|
||||
self.assertNotIn('PRIVATE_RAW_VALUE',result['answer'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Read recovery retries only the failed request, within the original scope and budget."""
|
||||
import copy
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)]
|
||||
import nao_protocol_execution as execution
|
||||
import test_nao_query_plan_execution as fixtures
|
||||
from nao_quality_runtime import RuntimeErrorSafe
|
||||
|
||||
|
||||
class Delegate:
|
||||
def __init__(self, outcomes):
|
||||
self.outcomes = list(outcomes)
|
||||
self.requests = []
|
||||
|
||||
def call(self, path, payload=None, method=None, **scope):
|
||||
self.requests.append((path, copy.deepcopy(payload), method, scope))
|
||||
value = self.outcomes.pop(0)
|
||||
if isinstance(value, BaseException):
|
||||
raise value
|
||||
return value
|
||||
|
||||
|
||||
def transient(status=503):
|
||||
error = RuntimeErrorSafe('暂时无法读取')
|
||||
error.retryable = True
|
||||
error.http_status = status
|
||||
return error
|
||||
|
||||
|
||||
class QueryRecoveryTests(unittest.TestCase):
|
||||
def scoped(self, delegate, **kwargs):
|
||||
return execution.ScopedApi(delegate, 'node', 'scope', cancel_event=threading.Event(), **kwargs)
|
||||
|
||||
def test_transient_read_retries_identical_envelope_and_counts_each_attempt(self):
|
||||
delegate = Delegate([transient(), {'protocolEvidence': {'queryId': 'success'}}])
|
||||
events, diagnostics = [], []
|
||||
api = self.scoped(delegate, emit=events.append, recovery_log=diagnostics, read_limit=2)
|
||||
payload = {'releaseId': 'release', 'offset': 100, 'limit': 100}
|
||||
response = api.call('/api/analysis/investigate', payload, 'POST')
|
||||
self.assertEqual(response['protocolEvidence']['queryId'], 'success')
|
||||
self.assertEqual(delegate.requests[0], delegate.requests[1])
|
||||
self.assertEqual(api.read_count, 2)
|
||||
self.assertEqual(len(api.investigation_responses), 1)
|
||||
self.assertEqual(diagnostics[-1]['outcome'], 'recovered')
|
||||
self.assertTrue(any('重试' in event['message'] for event in events))
|
||||
|
||||
def test_http_retry_statuses_are_bounded_to_three_actual_attempts(self):
|
||||
for status in (429, 502, 503, 504):
|
||||
with self.subTest(status=status):
|
||||
delegate = Delegate([HTTPError('https://offline.invalid', status, '', {}, None) for _ in range(4)])
|
||||
api = self.scoped(delegate)
|
||||
with self.assertRaises(HTTPError), patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(len(delegate.requests), 3)
|
||||
self.assertEqual(api.read_count, 3)
|
||||
|
||||
def test_permission_parameter_binding_and_unknown_failures_are_not_retried(self):
|
||||
failures = [HTTPError('https://offline.invalid', code, '', {}, None) for code in (400, 401, 403, 409, 422)]
|
||||
failures += [ValueError('正式查询证据未绑定当前执行'), RuntimeErrorSafe('所选期间没有来源数据')]
|
||||
explicit_permanent = transient()
|
||||
explicit_permanent.retryable = False
|
||||
failures.append(explicit_permanent)
|
||||
for failure in failures:
|
||||
with self.subTest(failure=str(failure)):
|
||||
delegate = Delegate([failure, {}])
|
||||
api = self.scoped(delegate)
|
||||
with self.assertRaises(type(failure)):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(api.read_count, 1)
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
|
||||
def test_mutating_endpoints_are_never_retried(self):
|
||||
for path in ('/api/analysis/protocol/task', '/api/analysis/protocol/finish'):
|
||||
delegate = Delegate([transient(), {}])
|
||||
api = self.scoped(delegate)
|
||||
with self.assertRaises(RuntimeErrorSafe):
|
||||
api.call(path, {}, 'POST')
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
self.assertEqual(api.read_count, 0)
|
||||
|
||||
def test_budget_exhaustion_does_not_issue_an_extra_request(self):
|
||||
delegate = Delegate([transient(), {}])
|
||||
api = self.scoped(delegate, read_limit=1)
|
||||
with self.assertRaises(RuntimeErrorSafe):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(api.read_count, 1)
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
with self.assertRaisesRegex(ValueError, '查询上限'):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
|
||||
def test_finalization_reserve_and_cancellation_stop_retries(self):
|
||||
for remaining, cancelled in ((300, False), (3000, True)):
|
||||
with self.subTest(remaining=remaining, cancelled=cancelled):
|
||||
cancel = threading.Event()
|
||||
class Failing(Delegate):
|
||||
def call(self, *args, **kwargs):
|
||||
if cancelled:
|
||||
cancel.set()
|
||||
return super().call(*args, **kwargs)
|
||||
delegate = Failing([transient(), {}])
|
||||
api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel)
|
||||
with patch.object(execution, 'remaining_seconds', side_effect=[3000, remaining]):
|
||||
with self.assertRaises((RuntimeErrorSafe, RuntimeError)):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
|
||||
def test_cancelled_before_query_does_not_call_delegate(self):
|
||||
cancel = threading.Event(); cancel.set()
|
||||
delegate = Delegate([{}])
|
||||
api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel)
|
||||
with self.assertRaisesRegex(RuntimeError, 'cancelled'):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(len(delegate.requests), 0)
|
||||
|
||||
def test_expired_query_time_preserves_finalization_without_starting_a_request(self):
|
||||
delegate = Delegate([{}])
|
||||
api = self.scoped(delegate)
|
||||
with patch.object(execution, 'remaining_seconds', return_value=300):
|
||||
with self.assertRaisesRegex(ValueError, '时间不足'):
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(api.read_count, 0)
|
||||
self.assertEqual(len(delegate.requests), 0)
|
||||
|
||||
def test_server_receipt_with_missing_data_is_not_retried(self):
|
||||
delegate = Delegate([{'coverage': {'rawRows': 0}}, {}])
|
||||
api = self.scoped(delegate)
|
||||
self.assertEqual(api.call('/api/metrics?year=2026', method='GET')['coverage']['rawRows'], 0)
|
||||
self.assertEqual(len(delegate.requests), 1)
|
||||
|
||||
def test_query_socket_deadline_reserve_is_scoped_and_restored(self):
|
||||
class Reserved(Delegate):
|
||||
deadline_reserve_seconds = 60
|
||||
def call(self, *args, **kwargs):
|
||||
self.observed_reserve = self.deadline_reserve_seconds
|
||||
return super().call(*args, **kwargs)
|
||||
delegate = Reserved([{}])
|
||||
api = self.scoped(delegate)
|
||||
api.call('/api/metrics?year=2026', method='GET')
|
||||
self.assertEqual(delegate.observed_reserve, execution.FINALIZATION_RESERVE_SECONDS + 5)
|
||||
self.assertEqual(delegate.deadline_reserve_seconds, 60)
|
||||
|
||||
def test_only_failed_child_request_retries_and_completed_siblings_keep_their_evidence(self):
|
||||
class Intermittent(fixtures.QueryPlanApi):
|
||||
def __init__(self):
|
||||
super().__init__(); self.failed = False; self.failed_requests = []
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
if path.startswith('/api/metrics') and scope_id == 'trend-a' and not self.failed:
|
||||
self.failed = True
|
||||
self.failed_requests.append((path, node_id, scope_id))
|
||||
raise transient(502)
|
||||
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
api = Intermittent()
|
||||
with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
|
||||
result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api)
|
||||
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
|
||||
self.assertEqual(len(documents), 3)
|
||||
self.assertEqual(len(result['protocolEvidence']), 3)
|
||||
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1)
|
||||
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1)
|
||||
self.assertTrue(result['queryRecovery'])
|
||||
|
||||
def test_exhausted_failed_child_stays_partial_and_other_children_are_not_repeated(self):
|
||||
class Unavailable(fixtures.QueryPlanApi):
|
||||
def __init__(self):
|
||||
super().__init__(); self.failed_attempts = 0
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
if path.startswith('/api/metrics') and scope_id == 'trend-a':
|
||||
self.failed_attempts += 1
|
||||
raise transient(503)
|
||||
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
api = Unavailable()
|
||||
with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
|
||||
result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertEqual(api.failed_attempts, 3)
|
||||
self.assertEqual(len(documents), 2)
|
||||
self.assertEqual(len(result['protocolEvidence']), 2)
|
||||
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1)
|
||||
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1)
|
||||
self.assertTrue(any(item['outcome'] == 'stopped' for item in result['queryRecovery']))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""One system policy must reach planning, native calls and finalization."""
|
||||
import io
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
|
||||
|
||||
class RuntimePolicyTest(unittest.TestCase):
|
||||
def test_production_model_callers_have_no_independent_literal_budget(self):
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
for path in (root/'scripts').glob('nao*.py'):
|
||||
for node in ast.walk(ast.parse(path.read_text())):
|
||||
if isinstance(node,ast.Dict):
|
||||
for key,value in zip(node.keys,node.values):
|
||||
if isinstance(key,ast.Constant) and key.value in ('max_completion_tokens','max_tokens'):
|
||||
self.assertIsInstance(value,ast.Name,str(path))
|
||||
self.assertEqual(value.id,'MAX_OUTPUT_TOKENS',str(path))
|
||||
for path in (root/'backend/src/main/java/cn/yunsheng/quality/analysis').glob('*.java'):
|
||||
for value in re.findall(r'"max_(?:completion_)?tokens",\s*([^,\n]+)',path.read_text()):
|
||||
self.assertEqual(value,'AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS',str(path))
|
||||
|
||||
def test_planner_uses_global_budget(self):
|
||||
from nao_agent_task import _model_json
|
||||
from nao_quality_runtime import native_output_budget
|
||||
bodies = []
|
||||
def provider(request, timeout):
|
||||
bodies.append(json.loads(request.data))
|
||||
return io.BytesIO(b'{"choices":[{"finish_reason":"stop","message":{"content":"{}"}}]}')
|
||||
_model_json(base_url='http://localhost/v1', api_key='fixture', model='fixture',
|
||||
system='fixture', payload={}, cancel_event=threading.Event(), opener=provider)
|
||||
self.assertEqual(bodies[0]['max_completion_tokens'], native_output_budget())
|
||||
|
||||
def test_stale_environment_cannot_create_another_budget(self):
|
||||
from nao_quality_runtime import native_output_budget
|
||||
baseline = native_output_budget()
|
||||
with patch.dict('os.environ', {'QUALITY_ANALYSIS_MAX_OUTPUT_TOKENS': '2048'}):
|
||||
self.assertEqual(native_output_budget(), baseline)
|
||||
|
||||
def test_gate_replaces_both_legacy_caps_even_when_thinking_off(self):
|
||||
from nao_model_gate import ModelGate
|
||||
from nao_quality_runtime import native_output_budget
|
||||
gate = ModelGate(upstream_base='http://localhost/v1', upstream_key='fixture', model='fixture',
|
||||
port=0, client_key='fixture', authorize=lambda: None,
|
||||
thinking_options={'enabled':False, 'effort':'auto', 'protocol':'thinking'})
|
||||
try:
|
||||
body = gate.apply_thinking({'max_tokens':1024, 'max_completion_tokens':4096})
|
||||
self.assertNotIn('max_tokens', body)
|
||||
self.assertEqual(body['max_completion_tokens'], native_output_budget())
|
||||
finally:
|
||||
gate.close()
|
||||
|
||||
def test_finalization_time_is_reserved_and_deadline_is_shared(self):
|
||||
from nao_runtime_policy import turn_scope, remaining_seconds, request_timeout, native_timeout
|
||||
with patch('nao_runtime_policy.time.monotonic', return_value=100):
|
||||
with turn_scope():
|
||||
total = remaining_seconds()
|
||||
self.assertLess(native_timeout(), total)
|
||||
with patch('nao_runtime_policy.time.monotonic', return_value=100+total-40):
|
||||
self.assertLessEqual(request_timeout('final'), 35)
|
||||
with self.assertRaises(TimeoutError):
|
||||
request_timeout('planning')
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
import json,sys,threading,unittest
|
||||
from pathlib import Path
|
||||
from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
|
||||
from urllib.request import Request,urlopen
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_model_gate import ModelGate
|
||||
|
||||
|
||||
class ThinkingGateTest(unittest.TestCase):
|
||||
def test_nonstream_planner_receives_json_while_thinking_is_delivered_before_answer(self):
|
||||
callbacks=[];calls=[];thought_seen=threading.Event()
|
||||
class Provider(BaseHTTPRequestHandler):
|
||||
def log_message(self,*args):pass
|
||||
def do_POST(self):
|
||||
body=json.loads(self.rfile.read(int(self.headers['Content-Length'])));calls.append(body)
|
||||
self.send_response(200);self.send_header('Content-Type','text/event-stream');self.end_headers()
|
||||
for field,text in [('reasoning','先核对'),('reasoning','完整接口文本')]:
|
||||
raw=('data: '+json.dumps({'model':'fixture','choices':[{'index':0,'delta':{field:text}}]},ensure_ascii=False)+'\n\n').encode()
|
||||
self.wfile.write(raw);self.wfile.flush()
|
||||
if not thought_seen.wait(2):return
|
||||
self.wfile.write(b'data: {"choices":[{"index":0,"delta":{"content":"{\\"ok\\":true}"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n');self.wfile.flush()
|
||||
server=ThreadingHTTPServer(('127.0.0.1',0),Provider);thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
|
||||
def thinking(event):
|
||||
callbacks.append(event)
|
||||
if event['event']=='delta':thought_seen.set()
|
||||
gate=None
|
||||
try:
|
||||
gate=ModelGate(upstream_base=f'http://127.0.0.1:{server.server_port}/v1',upstream_key='fixture',
|
||||
model='fixture',port=0,client_key='fixture',authorize=lambda:None,on_thinking=thinking)
|
||||
req=Request(f'http://127.0.0.1:{gate._server.server_port}/v1/chat/completions',
|
||||
data=json.dumps({'model':'fixture','stream':False,'messages':[]}).encode(),
|
||||
headers={'Authorization':'Bearer fixture','Content-Type':'application/json','X-Quality-Analysis-Phase':'planning'})
|
||||
with urlopen(req,timeout=4) as response:result=json.loads(response.read())
|
||||
self.assertTrue(calls[0]['stream'])
|
||||
self.assertEqual(''.join(e.get('text','') for e in callbacks if e['event']=='delta'),'先核对完整接口文本')
|
||||
self.assertEqual(result['choices'][0]['message']['content'],'{"ok":true}')
|
||||
self.assertEqual(callbacks[-1]['event'],'end');self.assertTrue(all(e['phase']=='planning' for e in callbacks))
|
||||
finally:
|
||||
if gate:gate.close()
|
||||
server.shutdown();server.server_close();thread.join(timeout=1)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
from nao_thinking_stream import ProviderStream,ThinkingPublisher
|
||||
|
||||
|
||||
class ThinkingStreamTest(unittest.TestCase):
|
||||
def test_default_publisher_does_not_bundle_a_second_of_text_into_one_update(self):
|
||||
from unittest.mock import patch
|
||||
updates=[]
|
||||
with patch('nao_thinking_stream.time.monotonic') as clock:
|
||||
clock.return_value=10.0
|
||||
p=ThinkingPublisher(updates.append)
|
||||
base={'id':'stream-cadence','model':'fixture','phase':'analysis'}
|
||||
p.receive({**base,'event':'start'})
|
||||
for i in range(11):
|
||||
clock.return_value=10.0+i/10
|
||||
p.receive({**base,'event':'delta','text':'新字'})
|
||||
self.assertGreaterEqual(len([r for r in updates if r['text']]),4)
|
||||
|
||||
def test_converting_to_stream_does_not_change_tool_names_or_arguments(self):
|
||||
stream=ProviderStream(lambda text:None)
|
||||
stream.document({'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'id':'call-1','function':{'name':'quality_','arguments':'{"mode":'}}]}}]})
|
||||
stream.document({'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'name':'data','arguments':'"compare"}'}}]},'finish_reason':'tool_calls'}]})
|
||||
message=stream.completion()['choices'][0]['message']
|
||||
self.assertEqual(message['tool_calls'],[{'id':'call-1','type':'function','function':{'name':'quality_data','arguments':'{"mode":"compare"}'}}])
|
||||
self.assertIsNone(message['content'])
|
||||
|
||||
def test_utf8_split_stream_preserves_full_thought_and_final_json(self):
|
||||
thoughts=[];stream=ProviderStream(thoughts.append)
|
||||
rows=[{'model':'fixture','choices':[{'index':0,'delta':{'reasoning':'先核对🙂。'}}]},
|
||||
{'choices':[{'index':0,'delta':{'reasoning':'再比较。'}}]},
|
||||
{'choices':[{'index':0,'delta':{'content':'{"ok":'}}]},
|
||||
{'choices':[{'index':0,'delta':{'content':'true}'},'finish_reason':'stop'}]},
|
||||
{'usage':{'completion_tokens':12},'choices':[]}]
|
||||
wire=''.join('data: '+json.dumps(row,ensure_ascii=False)+'\n\n' for row in rows).encode()+b'data: [DONE]\n\n'
|
||||
for byte in wire:stream.feed(bytes([byte]))
|
||||
stream.finish();answer=stream.completion()
|
||||
self.assertEqual(''.join(thoughts),'先核对🙂。再比较。')
|
||||
self.assertEqual(json.loads(answer['choices'][0]['message']['content']),{'ok':True})
|
||||
self.assertEqual(answer['usage']['completion_tokens'],12)
|
||||
|
||||
def test_answer_is_not_invented_as_thinking_and_aliases_are_not_duplicated(self):
|
||||
thoughts=[];stream=ProviderStream(thoughts.append)
|
||||
stream.document({'choices':[{'message':{'content':'正式答案','reasoning_content':'接口思考','reasoning':'接口思考'},'finish_reason':'stop'}]})
|
||||
self.assertEqual(thoughts,['接口思考'])
|
||||
self.assertEqual(stream.completion()['choices'][0]['message']['content'],'正式答案')
|
||||
|
||||
def test_publisher_preserves_more_than_eight_thousand_characters_and_redacts_credentials(self):
|
||||
updates=[];p=ThinkingPublisher(updates.append,interval=0)
|
||||
base={'id':'model-1','model':'fixture','phase':'analysis'}
|
||||
p.receive({**base,'event':'start'});p.receive({**base,'event':'delta','text':'长文本'*5000+' token=do-not-display'})
|
||||
p.receive({**base,'event':'end'})
|
||||
self.assertGreater(len(updates[-1]['text']),8000)
|
||||
self.assertNotIn('do-not-display',updates[-1]['text'])
|
||||
self.assertEqual(updates[-1]['status'],'completed')
|
||||
self.assertFalse(updates[-1]['truncated'])
|
||||
|
||||
def test_missing_and_interrupted_provider_outputs_are_explicit(self):
|
||||
updates=[];p=ThinkingPublisher(updates.append,interval=999)
|
||||
base={'id':'model-1','model':'fixture','phase':'planning'}
|
||||
p.receive({**base,'event':'start'});p.receive({**base,'event':'end'})
|
||||
self.assertEqual(updates[-1]['status'],'not_returned')
|
||||
p.receive({**base,'id':'model-2','event':'start'});p.receive({**base,'id':'model-2','event':'delta','text':'未完成的接口文本'})
|
||||
p.close();self.assertEqual(updates[-1]['text'],'未完成的接口文本')
|
||||
self.assertEqual(updates[-1]['status'],'interrupted')
|
||||
count=len(updates);p.receive({**base,'event':'delta','text':'late'});self.assertEqual(len(updates),count)
|
||||
|
||||
def test_thought_timestamps_keep_first_text_position_and_close_time(self):
|
||||
updates=[];p=ThinkingPublisher(updates.append,interval=0)
|
||||
base={'id':'model-1','model':'fixture','phase':'analysis'}
|
||||
p.receive({**base,'event':'start'})
|
||||
start=updates[-1]['startedAt'];self.assertNotIn('at',updates[-1])
|
||||
p.receive({**base,'event':'delta','text':'第一段'})
|
||||
first=updates[-1]['at']
|
||||
p.receive({**base,'event':'delta','text':'第二段'})
|
||||
self.assertEqual(updates[-1]['at'],first)
|
||||
p.receive({**base,'event':'end'})
|
||||
self.assertEqual(updates[-1]['startedAt'],start)
|
||||
self.assertLessEqual(start,first);self.assertLessEqual(first,updates[-1]['finishedAt'])
|
||||
|
||||
def test_cutoff_is_never_claimed_to_be_full_text(self):
|
||||
updates=[];p=ThinkingPublisher(updates.append,interval=0,max_chars=5)
|
||||
base={'id':'model-1','model':'fixture','phase':'analysis'}
|
||||
p.receive({**base,'event':'start'});p.receive({**base,'event':'delta','text':'123456'})
|
||||
self.assertEqual(updates[-1]['text'],'12345');self.assertTrue(updates[-1]['truncated'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,85 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
|
||||
from nao_visual_recovery import rebuild, SavedResponses
|
||||
from test_nao_protocol_execution import task, ranking_task, BoundApi, BoundRankingApi, identity, RELEASE
|
||||
|
||||
|
||||
def query(value,api=None,**period):
|
||||
api=api or BoundApi()
|
||||
selected_period=period or value['period']
|
||||
params={'releaseId':RELEASE,**selected_period,'view':value.get('_view','overall'),'includeUnassigned':'false'}
|
||||
params.update(value['filters'])
|
||||
response=api.call('/api/metrics?'+urlencode(params))
|
||||
key='query-'+str(selected_period.get('month','year'))+'-'+params['view']
|
||||
proof={**identity(),'queryId':key,'responseHash':'a'*64}
|
||||
return {'queryId':key,'target':'GET /api/metrics','parameters':{k:str(v) for k,v in params.items()},
|
||||
'responseHash':'a'*64,'response':response,'proof':proof,'task':copy.deepcopy(value),'scope':{}}
|
||||
|
||||
|
||||
def bundle(value,queries):
|
||||
return {'version':1,**identity(),'releaseId':RELEASE,'question':value['question'],
|
||||
'task':value,'queries':queries,'scopes':[],'missingEvidence':[]}
|
||||
|
||||
|
||||
class VisualRecoveryTests(unittest.TestCase):
|
||||
def test_trend_is_rebuilt_from_only_the_frozen_response_without_api_or_model(self):
|
||||
value=task({'product':'18X15X4+T'});saved=query(value);original=copy.deepcopy(saved)
|
||||
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
|
||||
result=rebuild(bundle(value,[saved]))
|
||||
self.assertEqual(result['visualState'],'ready');self.assertEqual(len(result['visuals']),1)
|
||||
visual=result['visuals'][0]
|
||||
self.assertEqual(visual['series'][0]['values'],[float(row['compositeRate']) for row in saved['response']['trend']])
|
||||
self.assertEqual(visual['scope']['product'],'18X15X4+T')
|
||||
self.assertEqual(result['queryRefs'][0]['queryIds'],[saved['queryId']]);self.assertEqual(saved,original)
|
||||
|
||||
def test_comparison_uses_both_original_periods(self):
|
||||
value=task();value.update(trendRequired=False,comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='required')
|
||||
saved=[query(value,year=2026,month=month) for month in (9,8)]
|
||||
result=rebuild(bundle(value,saved));self.assertEqual(result['visualState'],'ready')
|
||||
self.assertEqual(set(result['queryRefs'][0]['queryIds']),{q['queryId'] for q in saved})
|
||||
self.assertTrue(all(chart['comparisonPeriod'].get('year')==2026 and chart['comparisonPeriod'].get('month')==8 for chart in result['visuals']))
|
||||
partial=rebuild(bundle(value,saved[:1]));self.assertNotEqual(partial['visualState'],'ready');self.assertTrue(partial['missingEvidence'])
|
||||
|
||||
def test_weight_ranking_retains_the_full_original_denominator(self):
|
||||
value=ranking_task(dimensions=['product'],dimensionStages={'product':['black']},stages=['black'],fields=['totalWeight'])
|
||||
raw=copy.deepcopy(value);raw['_view']='product';saved=query(raw,BoundRankingApi())
|
||||
result=rebuild(bundle(value,[saved]));self.assertEqual(result['visualState'],'ready')
|
||||
self.assertEqual(result['visuals'][0]['coverage']['denominator'],100)
|
||||
self.assertEqual(result['visuals'][0]['series'][0]['values'],[60.0,40.0])
|
||||
|
||||
def test_each_scope_uses_only_its_original_product_and_node_proof(self):
|
||||
root=task();root['queryPlan']=[{'nodeId':f'trend-{i}','purpose':'trend','comparisonRequired':False} for i in (1,2)]
|
||||
queries=[];scopes=[]
|
||||
for i,product in enumerate(('18X15X4+T','18X15X4+Z'),1):
|
||||
child=task({'product':product});saved=query(child);saved['queryId']+=str(i);saved['proof']['queryId']=saved['queryId']
|
||||
scope={'nodeId':f'trend-{i}','scopeId':f'scope-{i}','purpose':'trend','task':child}
|
||||
for key in ('nodeId','scopeId'):saved[key]=saved['proof'][key]=scope[key]
|
||||
saved['scope']=scope;queries.append(saved);scopes.append(scope)
|
||||
request=bundle(root,queries);request['scopes']=scopes
|
||||
result=rebuild(request);self.assertEqual(result['visualState'],'ready');self.assertEqual(len(result['visuals']),2)
|
||||
self.assertEqual({chart['scope']['product'] for chart in result['visuals']},{'18X15X4+T','18X15X4+Z'})
|
||||
request['queries']=queries[:1]
|
||||
partial=rebuild(request);self.assertEqual(len(partial['visuals']),1);self.assertTrue(partial['missingEvidence'])
|
||||
self.assertEqual(partial['visuals'][0]['scope']['product'],'18X15X4+T')
|
||||
|
||||
def test_execution_or_proof_changes_never_rebind_old_numbers(self):
|
||||
value=task();base=bundle(value,[query(value)])
|
||||
for key in ('executionId','attemptId','sessionId','turnId','revision','queryId','responseHash','nodeId'):
|
||||
changed=copy.deepcopy(base);changed['queries'][0]['proof'][key]='different'
|
||||
with self.subTest(key=key),self.assertRaises(ValueError):rebuild(changed)
|
||||
changed=copy.deepcopy(base);changed['queries']*=2
|
||||
with self.assertRaises(ValueError):rebuild(changed)
|
||||
|
||||
def test_an_unrequested_chart_or_unrecorded_endpoint_cannot_trigger_a_read(self):
|
||||
value=task();value['visualPreference']='none'
|
||||
with self.assertRaises(ValueError):rebuild(bundle(value,[query(value)]))
|
||||
with self.assertRaises(Exception):SavedResponses([])._call('GET','/api/metrics',params={})
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
||||
"""Native synthesis through the real ModelGate and a loopback fake provider."""
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
||||
from nao_evidence_finalizer import synthesize_native, FinalizationUnavailable
|
||||
from nao_model_gate import ModelGate
|
||||
from test_agent_answer import evidence
|
||||
|
||||
|
||||
class NativeAnswerGateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.allowed = True
|
||||
self.cancelled = threading.Event()
|
||||
self.calls = []
|
||||
self.events = []
|
||||
self.reject_before_followup = False
|
||||
self.malformed_synthesis = False
|
||||
self.cancel_after_synthesis = False
|
||||
owner = self
|
||||
class Provider(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_): pass
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
owner.calls.append(body)
|
||||
reviewing = body['messages'][0]['content'].startswith('逐句核验证据')
|
||||
content = ({'reviews': [{'index': 0, 'supported': True}]} if reviewing else
|
||||
{'blocks': [{'text': '本期正式率低于基期,需要继续核查阶段差异。',
|
||||
'factIds': ['F1'], 'claimType': 'inference'}],
|
||||
'summarySelection': {'leadIndex': 0, 'evidenceIndexes': []}})
|
||||
if owner.malformed_synthesis and not reviewing:
|
||||
content = 'invalid composition'
|
||||
data = json.dumps({'model': 'fixture-model', 'choices': [{'finish_reason': 'stop',
|
||||
'message': {'content': content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)}}]}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
|
||||
self.provider.daemon_threads = True
|
||||
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
def authorize():
|
||||
if not self.allowed or self.cancelled.is_set():
|
||||
raise PermissionError('fixture delegation revoked or cancelled')
|
||||
def thinking(event):
|
||||
self.events.append(event)
|
||||
# Revoke after the gate has fully authorized/read the first
|
||||
# completion, exactly before the finalizer's next model request.
|
||||
if event.get('event') == 'end' and not event.get('interrupted'):
|
||||
if self.reject_before_followup:
|
||||
self.allowed = False
|
||||
if self.cancel_after_synthesis:
|
||||
self.cancelled.set()
|
||||
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1',
|
||||
upstream_key='fixture-provider-key', model='fixture-model', port=0,
|
||||
client_key='fixture-gate-key', authorize=authorize,
|
||||
native_turn_marker='quality-turn:fixture-current',
|
||||
thinking_options={'enabled': True, 'effort': 'high', 'protocol': 'thinking'}, on_thinking=thinking)
|
||||
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
|
||||
|
||||
def tearDown(self):
|
||||
self.gate.close()
|
||||
self.provider.shutdown()
|
||||
self.provider.server_close()
|
||||
self.thread.join(timeout=1)
|
||||
|
||||
def synthesize(self):
|
||||
# No mocked opener: the finalizer, HTTP gate and upstream all run.
|
||||
return synthesize_native(base_url=self.base, api_key='fixture-gate-key', model='fixture-model',
|
||||
question='解释质量变化', evidence=evidence(), cancel_event=self.cancelled)
|
||||
|
||||
def test_synthesis_and_review_use_authorized_finalizing_phase_without_native_marker(self):
|
||||
result = self.synthesize()
|
||||
self.assertTrue(result['evidenceSufficient'], result)
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(self.gate.rejected_stale_requests, 0)
|
||||
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_synthesis', 'evidence_review'])
|
||||
self.assertTrue(all(call['thinking'] == {'type': 'enabled'} for call in self.calls))
|
||||
self.assertTrue(all(call['reasoning_effort'] == 'high' for call in self.calls))
|
||||
self.assertFalse(any('quality-turn:fixture-current' in json.dumps(call['messages']) for call in self.calls))
|
||||
self.assertTrue(self.events)
|
||||
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
|
||||
|
||||
def test_revoked_delegation_blocks_initial_synthesis_before_upstream(self):
|
||||
self.allowed = False
|
||||
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
|
||||
self.synthesize()
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
def test_revocation_is_checked_again_before_review_and_format_repair(self):
|
||||
for malformed in (False, True):
|
||||
with self.subTest(malformed=malformed):
|
||||
self.allowed = True
|
||||
self.calls.clear()
|
||||
self.reject_before_followup = True
|
||||
self.malformed_synthesis = malformed
|
||||
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
|
||||
self.synthesize()
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
|
||||
def test_cancellation_blocks_initial_and_subsequent_model_work(self):
|
||||
self.cancelled.set()
|
||||
with self.assertRaisesRegex(FinalizationUnavailable, '已取消'):
|
||||
self.synthesize()
|
||||
self.assertEqual(self.calls, [])
|
||||
self.cancelled.clear()
|
||||
self.cancel_after_synthesis = True
|
||||
with self.assertRaises(FinalizationUnavailable):
|
||||
self.synthesize()
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,393 @@
|
||||
"""Native tool loop boundaries, using actual tool adapters and captured-like APIs."""
|
||||
import copy
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
||||
from nao_native_exploration import prepare_step, NativeExplorationTools, _proofs, native_model_view
|
||||
from nao_agent_task import normalize_task, plan_task
|
||||
from nao_quality_tools import QualityToolError
|
||||
from test_nao_protocol_execution import BoundApi, identity
|
||||
from test_nao_quality_tools import RELEASE
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_query_plan_raw import InvestigationApi
|
||||
|
||||
|
||||
def root():
|
||||
return {'version': 1, 'goal': '解释质量变化并根据结果调查', 'question': '9月份合格率下降原因',
|
||||
'executionMode': 'native_agent', 'kind': 'analysis', 'period': {'year': 2026, 'month': 9},
|
||||
'comparisonPeriod': {'year': 2026, 'month': 8}, 'filters': {}, 'fields': ['rate'],
|
||||
'dimensions': [], 'stages': [], 'dimensionStages': {}, 'visualPreference': 'auto',
|
||||
'verifySource': False, 'fullDetails': False, 'freshQuery': True,
|
||||
'comparisonRequired': True, 'trendRequired': False}
|
||||
|
||||
|
||||
class Delegated(BoundApi):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.steps, self.authorization_calls = [], 0
|
||||
self.timeout, self.deadline_reserve_seconds = 30, None
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
if path == '/api/analysis/model-authorization':
|
||||
self.authorization_calls += 1
|
||||
return {}
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'scopeCoverage':[{'nodeId':step['node']['nodeId'], 'scopeId':'scope-'+step['node']['nodeId'],
|
||||
'complete':True,'missingEvidence':[]} for step in self.steps], 'rootGoalGaps':[]}
|
||||
if path == '/api/analysis/protocol/exploration/steps':
|
||||
self.steps.append(copy.deepcopy(payload))
|
||||
task = root()
|
||||
task.update(comparisonRequired=payload['node']['comparisonRequired'])
|
||||
task['filters'] = {}
|
||||
node = payload['node']
|
||||
selector = node['selector']
|
||||
if selector['kind'] == 'catalog_members': task['filters'][selector['dimension']] = selector['names'][0]
|
||||
return {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
|
||||
'task': task, 'scope': {'nodeId': node['nodeId'], 'task': task}}
|
||||
response = super().call(path, payload=payload, method=method)
|
||||
if path.startswith('/api/metrics'):
|
||||
from urllib.parse import urlsplit, parse_qs
|
||||
month = parse_qs(urlsplit(path).query).get('month', ['0'])[0]
|
||||
response['protocolEvidence'] = {**identity(), 'nodeId': node_id, 'scopeId': scope_id,
|
||||
'queryId': 'query-' + month, 'responseHash': 'a'*64}
|
||||
return response
|
||||
|
||||
|
||||
class NativeExplorationTests(unittest.TestCase):
|
||||
def state(self): return {'releaseId': RELEASE, 'scopes': []}
|
||||
def test_model_view_removes_only_transport_and_identical_signal_duplicates(self):
|
||||
row={'key':'甲','label':'甲','rank':1,'current':{'rate':0,'scrapWeight':None},
|
||||
'comparison':{'rate':None,'scrapWeight':0}}
|
||||
source={'kind':'quality_changes','rows':[row],'totalRows':9,'sort':'current_rate',
|
||||
'prioritySignals':{'scale':[{'key':'甲','label':'甲','current':copy.deepcopy(row['current']),
|
||||
'comparison':copy.deepcopy(row['comparison'])},{'key':'乙','current':{'rate':0.9}}]},
|
||||
'evidenceRef':'Q12345678','protocolEvidence':{'queryId':'private'},'evidencePath':'private/path'}
|
||||
view=native_model_view(source)
|
||||
self.assertEqual(view['rows'],[row]);self.assertEqual(view['totalRows'],9)
|
||||
self.assertEqual(view['prioritySignals']['scale'][0]['sameAsReturnedRow'],'甲')
|
||||
self.assertEqual(view['prioritySignals']['scale'][1],source['prioritySignals']['scale'][1])
|
||||
self.assertNotIn('protocolEvidence',view);self.assertNotIn('evidencePath',view)
|
||||
self.assertEqual(view['evidenceRef'],'Q12345678')
|
||||
self.assertIn('protocolEvidence',source)
|
||||
source['prioritySignals']['scale'][0]['rateDrop']=None
|
||||
changed=native_model_view(source)
|
||||
self.assertIn('rateDrop',changed['prioritySignals']['scale'][0])
|
||||
self.assertIsNone(changed['prioritySignals']['scale'][0]['rateDrop'])
|
||||
def test_explicit_cause_cannot_silently_become_a_static_recipe(self):
|
||||
from nao_native_planning import explicit_investigation
|
||||
self.assertTrue(explicit_investigation('为什么下降?请自行深挖'))
|
||||
self.assertFalse(explicit_investigation('取消原因分析,只查合格率'))
|
||||
native = task_value(executionMode='native_agent', dimensions=[], stages=[], fields=['rate'],
|
||||
period={'year':2026,'month':9}, comparisonPeriod={'year':2026,'month':8},
|
||||
comparisonRequired=True, trendRequired=False)
|
||||
wrong = copy.deepcopy(native); wrong.pop('executionMode')
|
||||
with patch('nao_agent_task._model_json', side_effect=[wrong,native]) as model:
|
||||
task = plan_task(base_url='http://offline.invalid', api_key='fixture', model='fixture',
|
||||
question='为什么下降?请深入分析', context={'_protocol':{'version':1,'autonomousEnabled':True}},
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(task['executionMode'],'native_agent')
|
||||
self.assertNotIn('queryPlan', task)
|
||||
self.assertEqual(model.call_count,2)
|
||||
self.assertNotIn('黑片投入重量排名后查看前两项',model.call_args.kwargs['system'])
|
||||
|
||||
def test_clear_ranking_rejects_native_route_before_confirmation(self):
|
||||
from nao_native_planning import explicit_batch_ranking
|
||||
question='2026年9月汽车交通哪些规格合格率最低?按本期综合合格率从低到高列出前5个,同时说明投入和报废规模。'
|
||||
self.assertTrue(explicit_batch_ranking(question))
|
||||
self.assertFalse(explicit_batch_ranking('产品合格率最低的原因是什么?请深入调查'))
|
||||
self.assertFalse(explicit_batch_ranking('不要排名,只查产品合格率'))
|
||||
native=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9},comparisonRequired=True,
|
||||
comparisonPeriod={'year':2026,'month':8})
|
||||
batch={**native,'dimensions':['product'],'fields':['rate','totalWeight','scrapWeight'],
|
||||
'dimensionStages':{'product':[]},'queryPlan':[{
|
||||
'nodeId':'rank','purpose':'changes','dimension':'product','sort':'current_rate','limit':5,
|
||||
'fields':['rate','totalWeight','scrapWeight'],'stages':[],
|
||||
'selector':{'kind':'base'},'comparisonRequired':True,'visualType':'bar','visualPreference':'auto'}]}
|
||||
batch.pop('executionMode')
|
||||
with patch('nao_agent_task._model_json',side_effect=[native,batch]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question=question,context={'_protocol':{'version':1,'autonomousEnabled':True}},
|
||||
cancel_event=threading.Event())
|
||||
self.assertNotEqual(task.get('executionMode'),'native_agent')
|
||||
self.assertEqual(task['queryPlan'][0]['sort'],'current_rate')
|
||||
self.assertEqual(task['queryPlan'][0]['limit'],5)
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_root_cannot_invent_domestic_and_explicit_group_gets_its_catalog_parent(self):
|
||||
from nao_native_planning import validate_root_scope
|
||||
catalog = {'groups':['汽车交通-国内','汽车交通-海外'],
|
||||
'groupDepartments':{'汽车交通-国内':'新能源汽车磁钢','汽车交通-海外':'新能源汽车磁钢'}}
|
||||
narrowed = root(); narrowed['filters'] = {'group':'汽车交通-国内'}
|
||||
with self.assertRaisesRegex(ValueError,'擅自缩'):
|
||||
validate_root_scope(copy.deepcopy(narrowed), '汽车交通为什么下降', catalog)
|
||||
valid = validate_root_scope(copy.deepcopy(narrowed), '那国内为什么下降', catalog)
|
||||
self.assertEqual(valid['filters'],{'group':'汽车交通-国内','department':'新能源汽车磁钢'})
|
||||
inherited = validate_root_scope(copy.deepcopy(narrowed),'继续分析原因',catalog,{'group':'汽车交通-国内'})
|
||||
self.assertEqual(inherited['filters']['group'],'汽车交通-国内')
|
||||
new = copy.deepcopy(narrowed); new['newTask'] = True
|
||||
with self.assertRaises(ValueError):
|
||||
validate_root_scope(new,'换个问题,公司为什么下降',catalog,{'group':'汽车交通-国内'})
|
||||
from dbgpt_trial_data import encoded_selection
|
||||
multi = root(); multi['filters'] = {'group':encoded_selection(['汽车交通-国内','汽车交通-海外'])}
|
||||
with self.assertRaises(ValueError):
|
||||
validate_root_scope(copy.deepcopy(multi),'公司为什么下降',catalog)
|
||||
explicit = validate_root_scope(copy.deepcopy(multi),'汽车交通国内和海外为什么下降',catalog)
|
||||
self.assertEqual(explicit['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(validate_root_scope(copy.deepcopy(multi),'继续分析',catalog,multi['filters'])['filters'],explicit['filters'])
|
||||
def test_explicit_authorized_department_cannot_disappear_from_root(self):
|
||||
from nao_native_planning import validate_root_scope
|
||||
catalog={'departments':['新能源汽车磁钢','消费电子-3C'],'groups':[], 'groupDepartments':{}}
|
||||
for question in ('汽车交通为什么下降','分析汽车交通不良原因','新能源汽车磁钢9月合格率', '3C合格率',
|
||||
'只分析汽车交通,不需要全公司数据','仅看汽车交通,不和消费电子比较',
|
||||
'汽车交通为什么下降,不看其他部门'):
|
||||
with self.subTest(question=question), self.assertRaises(ValueError):
|
||||
validate_root_scope({'filters':{}},question,catalog)
|
||||
for question in ('汽车交通不要了,改看公司整体','排除汽车交通看其余',
|
||||
'汽车交通和消费电子比较','比较3C与汽车交通',
|
||||
'汽车交通相比其他产业部如何','不含汽车交通,查全公司'):
|
||||
with self.subTest(question=question):
|
||||
self.assertEqual(validate_root_scope({'filters':{}},question,catalog)['filters'],{})
|
||||
self.assertEqual(validate_root_scope({'filters':{'department':'新能源汽车磁钢'}},
|
||||
'分析汽车交通不良原因',catalog)['filters']['department'],'新能源汽车磁钢')
|
||||
with self.assertRaises(ValueError):
|
||||
validate_root_scope({'filters':{'department':'消费电子-3C'}},'汽车交通为什么下降',catalog)
|
||||
# A product containing the suffix is not a department mention.
|
||||
self.assertEqual(validate_root_scope({'filters':{}},'查规格A3CX1',catalog)['filters'],{})
|
||||
|
||||
def test_department_omission_is_repaired_before_confirmation(self):
|
||||
value=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True},
|
||||
'_businessCatalog':{'departments':['新能源汽车磁钢','消费电子-3C']}}
|
||||
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'filterUpdates':{'department':'新能源汽车磁钢'}}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question='请深挖汽车交通为什么下降',context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
self.assertEqual(task['planRecovery']['errors'][0]['field'],'filterUpdates.department')
|
||||
|
||||
def test_negated_broad_scope_cannot_bypass_department_repair(self):
|
||||
value=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
|
||||
fields=['rate'],stages=[],period={'year':2026,'month':9})
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True},
|
||||
'_businessCatalog':{'departments':['新能源汽车磁钢','消费电子-3C']}}
|
||||
for question in ('只分析汽车交通,不需要全公司数据','仅看汽车交通,不和消费电子比较',
|
||||
'汽车交通为什么下降,不看其他部门'):
|
||||
with self.subTest(question=question), patch('nao_agent_task._model_json',side_effect=[
|
||||
value,{**value,'filterUpdates':{'department':'新能源汽车磁钢'}}]):
|
||||
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question=question,context=context,cancel_event=threading.Event())
|
||||
self.assertEqual(task['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'],2)
|
||||
|
||||
def test_business_cross_check_is_not_raw_file_permission(self):
|
||||
value = task_value(kind='analysis', dimensions=[], stages=[], fields=['rate'],
|
||||
period={'year': 2026, 'month': 9}, executionMode='native_agent',
|
||||
verifySource=True, fullDetails=True, trendRequired=False)
|
||||
context = {'_protocol': {'version': 1, 'autonomousEnabled': True}}
|
||||
task = normalize_task(value, '核对主要线索,先给分析', context)
|
||||
self.assertFalse(task['verifySource']); self.assertFalse(task['fullDetails'])
|
||||
explicit = normalize_task(value, '请核验原表,并逐行读取全部原始记录', context)
|
||||
self.assertTrue(explicit['verifySource']); self.assertTrue(explicit['fullDetails'])
|
||||
context['_protocol']['previousTask'] = explicit
|
||||
cancelled = normalize_task(value, '取消原表核验,不要逐行读取', context)
|
||||
self.assertFalse(cancelled['verifySource']); self.assertFalse(cancelled['fullDetails'])
|
||||
for wording in ('后面不要原始数据,只分析统计', '别再看原始行了', '原始数据不用了'):
|
||||
cancelled = normalize_task(value, wording, context)
|
||||
self.assertFalse(cancelled['verifySource'], wording)
|
||||
self.assertFalse(cancelled['fullDetails'], wording)
|
||||
for wording in ('不需要每行数据', '不要数据行', '不要每一条记录'):
|
||||
cancelled = normalize_task(value, wording, context)
|
||||
self.assertFalse(cancelled['fullDetails'], wording)
|
||||
retained = normalize_task(value, '不要逐行记录,继续核验原表', context)
|
||||
self.assertFalse(retained['fullDetails'])
|
||||
self.assertTrue(retained['verifySource'])
|
||||
for wording, flag in (('请核验不良品原表', 'verifySource'),
|
||||
('分析不合格原因并核验原表', 'verifySource'),
|
||||
('请逐行读取不合格的原始记录', 'fullDetails')):
|
||||
allowed = normalize_task(value, wording, {'_protocol': {'version': 1, 'autonomousEnabled': True}})
|
||||
self.assertTrue(allowed[flag], wording)
|
||||
|
||||
def test_disabled_comparison_does_not_revive_an_optional_old_comparison_date(self):
|
||||
value = task_value(kind='metrics', dimensions=[], fields=['rate'], stages=[],
|
||||
period={'year':2026,'month':10}, comparisonRequired=False,
|
||||
comparisonPeriod={'year':2026,'month':9})
|
||||
task = normalize_task(value,'只查十月,不比较',{'_protocol':{'version':1,'autonomousEnabled':True}})
|
||||
self.assertIsNone(task['comparisonPeriod'])
|
||||
|
||||
def test_new_confirmed_goal_does_not_restore_cancelled_historical_ranking(self):
|
||||
value=task_value(goal='只查询综合合格率,取消排名及规模要求',kind='metrics',
|
||||
dimensions=[],stages=[],fields=['rate'],period={'year':2026,'month':9},comparisonRequired=False)
|
||||
previous={'goal':'按低率排名五个规格及规模','rootGoal':'按低率排名五个规格及规模','kind':'analysis','filters':{}}
|
||||
with patch('nao_agent_task._model_json',return_value=value):
|
||||
planned=plan_task(base_url='http://offline.invalid',api_key='fixture',model='fixture',
|
||||
question='取消排名和规模,只查合格率',previous=previous,
|
||||
context={'_protocol':{'version':1,'autonomousEnabled':True},
|
||||
'_serverHistory':[{'question':previous['goal'],'evidenceRefs':['Q12345678']}]},
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(planned['rootGoal'],value['goal'])
|
||||
|
||||
def test_planner_failure_clarification_keeps_explicit_negative_product_constraint(self):
|
||||
context={'year':2026,'month':9,'_protocol':{'version':1,'autonomousEnabled':True},
|
||||
'_businessCatalog':{'explicitlyExcludedProducts':['18X15X4.1+Z'],
|
||||
'productMatch':{'status':'ambiguous','query':'18x15x4','truncated':False,
|
||||
'candidates':[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]}}}
|
||||
with patch('nao_agent_task._model_json',return_value=None):
|
||||
task=plan_task(base_url='http://offline.invalid',api_key='fixture',model='fixture',
|
||||
question='18X15X4合格率,不要包含18X15X4.1',context=context,cancel_event=threading.Event())
|
||||
self.assertTrue(task['needsClarification'])
|
||||
self.assertEqual(task['period'],{'year':2026,'month':9})
|
||||
self.assertEqual(task['filters']['exclusions']['product'],['18X15X4.1+Z'])
|
||||
def test_root_scope_cannot_be_changed_or_old_filter_reintroduced(self):
|
||||
task = root(); task['filters'] = {'department': '甲'}
|
||||
with self.assertRaises(QualityToolError):
|
||||
prepare_step({'mode': 'compare', 'filters': {'department': '乙'}}, task, self.state())
|
||||
args, _ = prepare_step({'mode': 'compare'}, task, self.state())
|
||||
self.assertEqual(args['filters'], {'department': '甲'})
|
||||
self.assertEqual(task['filters'], {'department': '甲'})
|
||||
|
||||
def test_next_observation_can_choose_a_new_dimension_and_another_stage(self):
|
||||
task = root()
|
||||
first, a = prepare_step({'mode': 'changes', 'dimension': 'group', 'sort': 'rate_drop'}, task, self.state())
|
||||
second, b = prepare_step({'mode': 'defects', 'stage': 'bright', 'filters': {'product': '规格乙'}}, task, self.state())
|
||||
self.assertEqual(a['node']['dimension'], 'group')
|
||||
self.assertEqual(b['node']['selector']['names'], ['规格乙'])
|
||||
self.assertEqual(b['node']['stages'], ['bright'])
|
||||
self.assertNotEqual(a['node']['nodeId'], b['node']['nodeId'])
|
||||
self.assertEqual(task, root())
|
||||
|
||||
def test_raw_reads_require_explicit_root_request(self):
|
||||
with self.assertRaises(QualityToolError):
|
||||
prepare_step({'mode': 'investigate', 'stage': 'black', 'action': 'rows'}, root(), self.state())
|
||||
|
||||
def test_exact_product_without_defect_uses_its_defect_breakdown_not_product_ranking(self):
|
||||
args, step=prepare_step({'mode':'defects','stage':'black','dimension':'product',
|
||||
'filters':{'product':'规格甲'}},root(),self.state())
|
||||
self.assertEqual(args['dimension'],'defect')
|
||||
self.assertEqual(step['node']['dimension'],'defect')
|
||||
self.assertEqual(args['filters'],{'product':'规格甲'})
|
||||
unchanged, _=prepare_step({'mode':'defects','stage':'black','dimension':'product','defect':'责废缺角重量'},root(),self.state())
|
||||
self.assertEqual(unchanged['dimension'],'product')
|
||||
|
||||
def test_raw_tools_use_protocol_endpoints_and_exact_defaults(self):
|
||||
task = root(); task['fullDetails'] = True; task['verifySource'] = True
|
||||
args, proposal = prepare_step({'mode': 'details', 'stage': 'black'}, task, self.state())
|
||||
self.assertEqual(args['mode'], 'investigate')
|
||||
self.assertEqual((args['action'], args['offset'], args['recordFilter']), ('rows', 0, 'all'))
|
||||
self.assertEqual(proposal['node']['purpose'], 'records')
|
||||
args, proposal = prepare_step({'mode': 'investigate', 'stage': 'bright', 'action': 'source'}, task, self.state())
|
||||
self.assertEqual(args['limit'], 100)
|
||||
self.assertEqual(proposal['node']['purpose'], 'source')
|
||||
|
||||
def test_multilevel_child_inherits_verified_parent(self):
|
||||
state = self.state(); state['scopes'] = [{'filters': {'group': '甲'}, 'queryIds': ['q1']}]
|
||||
_, step = prepare_step({'mode': 'defects', 'stage': 'black', 'filters': {'group': '甲', 'product': '乙'}}, root(), state)
|
||||
self.assertEqual(step['parentQueryIds'], ['q1'])
|
||||
self.assertEqual(step['node']['selector'], {'kind': 'catalog_members', 'dimension': 'product', 'names': ['乙']})
|
||||
|
||||
def test_equivalent_single_month_syntax_keeps_the_confirmed_frame(self):
|
||||
task = root()
|
||||
args, _ = prepare_step({'mode': 'compare', 'period': {'year': 2026, 'monthStart': 9, 'monthEnd': 9}}, task, self.state())
|
||||
self.assertEqual(args['period'], {'year': 2026, 'month': 9})
|
||||
with self.assertRaises(QualityToolError):
|
||||
prepare_step({'mode': 'compare', 'period': {'year': 2026, 'monthStart': 8, 'monthEnd': 9}}, task, self.state())
|
||||
|
||||
def test_invalid_shape_is_rejected_before_registering_a_scope(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
api = Delegated(); p = Path(folder)
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
|
||||
with self.assertRaises(QualityToolError):
|
||||
tools.quality_data({'mode': 'compare', 'dimension': 'product'})
|
||||
self.assertEqual(api.steps, [])
|
||||
|
||||
def test_actual_compare_preserves_both_proofs_and_duplicate_reads_are_cached(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
api = Delegated(); p = Path(folder)
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
|
||||
result = tools.quality_data({'mode': 'compare'})
|
||||
self.assertEqual(len(_proofs(result)), 2)
|
||||
self.assertEqual(len(api.steps), 1)
|
||||
for _ in range(3):
|
||||
duplicate = tools.quality_data({'mode': 'compare'})
|
||||
self.assertTrue(duplicate['reusedEvidence'])
|
||||
self.assertEqual(duplicate['evidenceRef'], result['evidenceRef'])
|
||||
self.assertEqual(len(api.steps), 1)
|
||||
self.assertEqual(api.authorization_calls, 4)
|
||||
with self.assertRaisesRegex(QualityToolError, '停止'):
|
||||
tools.quality_data({'mode': 'compare'})
|
||||
|
||||
def test_new_execution_never_reuses_previous_attempt_cache(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
api = Delegated(); p = Path(folder)
|
||||
kwargs = dict(delegated=api, evidence_dir=p/'evidence', task=root(), state_path=p/'state.json')
|
||||
a = NativeExplorationTools(**kwargs, identity={**identity(), 'releaseId': RELEASE})
|
||||
first = a.quality_data({'mode': 'compare'})
|
||||
b = NativeExplorationTools(**kwargs, identity={**identity(), 'executionId': 'new-execution', 'releaseId': RELEASE})
|
||||
second = b.quality_data({'mode': 'compare'})
|
||||
self.assertNotEqual(first['evidenceRef'], second['evidenceRef'])
|
||||
self.assertEqual(len(api.steps), 2)
|
||||
|
||||
def test_explicit_refresh_reexecutes_but_identical_responses_are_still_no_progress(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p=Path(folder); api=Delegated()
|
||||
tools=NativeExplorationTools(delegated=api,evidence_dir=p/'evidence',task=root(),
|
||||
identity={**identity(),'releaseId':RELEASE},state_path=p/'state.json')
|
||||
first=tools.quality_data({'mode':'compare'})
|
||||
for _ in range(3):
|
||||
checked=tools.quality_data({'mode':'compare','freshQuery':True})
|
||||
self.assertNotEqual(first['evidenceRef'],checked['evidenceRef'])
|
||||
self.assertIn('没有新增',checked['investigationNotice'])
|
||||
self.assertEqual(len(api.steps),1)
|
||||
with self.assertRaisesRegex(QualityToolError,'停止'):
|
||||
tools.quality_data({'mode':'compare','freshQuery':True})
|
||||
|
||||
def test_record_pages_share_authorized_scope_but_have_distinct_cached_evidence(self):
|
||||
task = root()
|
||||
task.update(fullDetails=True, comparisonRequired=False, comparisonPeriod=None, stages=['black'])
|
||||
class Pages(InvestigationApi):
|
||||
timeout = 30
|
||||
deadline_reserve_seconds = None
|
||||
def __init__(self):
|
||||
super().__init__(total_rows=165)
|
||||
self.registered = []
|
||||
def call(self, path, payload=None, method=None, **kwargs):
|
||||
if path == '/api/analysis/model-authorization': return {}
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'scopeCoverage':[{'nodeId':s['nodeId'],'scopeId':s['scopeId'],
|
||||
'complete':self.query_count >= 2,'missingEvidence':[] if self.query_count >= 2 else ['后续页尚未读取']}
|
||||
for s in self.scopes], 'rootGoalGaps':[]}
|
||||
if path == '/api/analysis/protocol/exploration/steps':
|
||||
self.registered.append(copy.deepcopy(payload))
|
||||
self.add_scope(payload['node']['nodeId'], 'record-scope', 'records', task)
|
||||
scope = self.scopes[-1]
|
||||
return {'nodeId':scope['nodeId'], 'scopeId':scope['scopeId'], 'task':scope['task'], 'scope':scope}
|
||||
return super().call(path, payload, method, **kwargs)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p = Path(folder); api = Pages()
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=task,
|
||||
identity={**identity(), 'releaseId':RELEASE}, state_path=p/'state.json')
|
||||
first = tools.quality_data({'mode':'details', 'stage':'black', 'offset':0})
|
||||
last = tools.quality_data({'mode':'details', 'stage':'black', 'offset':100})
|
||||
self.assertEqual(len(api.registered), 1)
|
||||
self.assertFalse(first['investigationCoverage']['scopeComplete'])
|
||||
self.assertTrue(last['investigationCoverage']['scopeComplete'])
|
||||
reads = [c for c in api.calls if c['path'] == '/api/analysis/investigate']
|
||||
self.assertEqual([c['payload']['offset'] for c in reads], [0,100])
|
||||
self.assertEqual(len({(c['nodeId'],c['scopeId']) for c in reads}), 1)
|
||||
self.assertNotEqual(first['evidenceRef'], last['evidenceRef'])
|
||||
cached = tools.quality_data({'mode':'details', 'stage':'black', 'offset':100})
|
||||
self.assertTrue(cached['reusedEvidence'])
|
||||
self.assertEqual(api.query_count, 2)
|
||||
tools.quality_data({'mode':'details', 'stage':'black', 'offset':100, 'freshQuery':True})
|
||||
self.assertEqual(api.query_count, 3)
|
||||
self.assertEqual(len(api.registered), 1)
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Exercise the native runner with isolated files and mocked upstream/bridge calls."""
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts')]
|
||||
from nao_native_run import run_native_exploration
|
||||
from nao_quality_runtime import _atomic_json
|
||||
|
||||
|
||||
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
|
||||
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
|
||||
RELEASE = 'release-current'
|
||||
|
||||
|
||||
def root_task():
|
||||
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
|
||||
'executionMode': 'native_agent', 'kind': 'analysis',
|
||||
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
|
||||
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
|
||||
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
|
||||
'comparisonRequired': True, 'trendRequired': False,
|
||||
'verifySource': False, 'fullDetails': False}
|
||||
|
||||
|
||||
def proof(query_id, *, node='root', scope='scope-root', execution='execution-current'):
|
||||
return {**IDENTITY, 'executionId': execution, 'queryId': query_id,
|
||||
'nodeId': node, 'scopeId': scope, 'responseHash': 'a' * 64}
|
||||
|
||||
|
||||
def document(ref, *proofs):
|
||||
result = ({'current': {'protocolEvidence': proofs[0]},
|
||||
'comparison': {'protocolEvidence': proofs[1]}}
|
||||
if len(proofs) == 2 else {'protocolEvidence': proofs[0]})
|
||||
return {'evidenceRef': ref, 'result': result}
|
||||
|
||||
|
||||
class NativeRunTests(unittest.TestCase):
|
||||
def test_full_facts_remain_private_instead_of_being_appended_to_the_business_answer(self):
|
||||
self.final['fullEvidenceFacts'] = [{'text': '后台保留的完整统计事实', 'evidenceRef': 'Q00000001'}]
|
||||
result = self.invoke()
|
||||
self.assertNotIn('完整统计依据', result['answer'])
|
||||
self.assertNotIn('后台保留的完整统计事实', result['answer'])
|
||||
stored = json.loads((self.actor_root / 'private/native-full-evidence-facts.json').read_text())
|
||||
self.assertEqual(stored, self.final['fullEvidenceFacts'])
|
||||
|
||||
def setUp(self):
|
||||
self.folder = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.folder.cleanup)
|
||||
self.actor_root = Path(self.folder.name)
|
||||
self.cancel_event = threading.Event()
|
||||
self.task = root_task()
|
||||
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
|
||||
'actorId': 'actor-current', 'releaseId': RELEASE, 'question': self.task['question'],
|
||||
'context': {'product': '已取消的旧规格',
|
||||
'_serverQuestions': [{'question': '上一轮问题'}],
|
||||
'_protocol': {'autonomousEnabled': True}}}
|
||||
self.p_current = proof('query-current')
|
||||
self.p_compare = proof('query-compare')
|
||||
self.p_group = proof('query-group', node='group', scope='scope-group')
|
||||
self.p_failed = proof('query-failed', node='failed', scope='scope-failed')
|
||||
self.p_old = proof('query-old', execution='execution-old')
|
||||
self.good = document('Q00000001', self.p_current, self.p_compare)
|
||||
self.unused = document('Q00000002', self.p_group)
|
||||
self.documents = [self.good, self.unused]
|
||||
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task},
|
||||
{'nodeId': 'group', 'scopeId': 'scope-group', 'task': self.task}],
|
||||
'queries': [{'proof': p} for p in (self.p_current, self.p_compare, self.p_group)],
|
||||
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True},
|
||||
{'nodeId': 'group', 'scopeId': 'scope-group', 'complete': True}],
|
||||
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
|
||||
self.final = {'answer': '按已核实结果解释[Q00000001]', 'reportTitle': '质量数据分析',
|
||||
'readingSummary': {'version': 1, 'lead': '已核实解释[Q00000001]',
|
||||
'evidence': [], 'notices': [], 'evidenceRefs': ['Q00000001']},
|
||||
'usedEvidenceRefs': ['Q00000001'], 'evidenceSufficient': True,
|
||||
'missingEvidence': [], 'rawModelResponses': [], 'compositionDiagnostics': {}}
|
||||
self.accumulator = SimpleNamespace(answer='原生草稿', finish_reasons=['stop'],
|
||||
tool_calls=[{'tool': 'quality_data', 'status': 'failed', 'input': {'mode': 'compare'}},
|
||||
{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
|
||||
timeline=[{'kind': 'tool_error'}, {'kind': 'tool_completed'}, {'kind': 'stop'}])
|
||||
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
|
||||
self.cancel_during_native = False
|
||||
self.events = []
|
||||
self.manager = Mock()
|
||||
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
|
||||
self.delegated = Mock()
|
||||
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
|
||||
stack = self.enterContext(ExitStack())
|
||||
self.auth = stack.enter_context(patch('nao_quality_runtime._authenticate', return_value=object()))
|
||||
self.sse = stack.enter_context(patch('nao_quality_runtime._sse_request', side_effect=self.native))
|
||||
self.read_evidence = stack.enter_context(patch('nao_evidence_finalizer.evidence_summaries',
|
||||
side_effect=lambda *args, **kwargs: copy.deepcopy(self.documents)))
|
||||
self.synthesis = stack.enter_context(patch('nao_evidence_finalizer.synthesize_native',
|
||||
side_effect=lambda **kwargs: copy.deepcopy(self.final)))
|
||||
self.visuals = stack.enter_context(patch('analysis_visual_evidence.protocol_visual_result',
|
||||
return_value={'visualState': 'not_requested', 'visuals': [], 'missingEvidence': []}))
|
||||
stack.enter_context(patch('nao_runtime_policy.native_timeout', return_value=30))
|
||||
|
||||
def native(self, **kwargs):
|
||||
# Let the real runner decide when a useful response first became available.
|
||||
kwargs['emit']({'type': 'progress', 'toolCalls': [
|
||||
{'status': 'completed', 'input': {'mode': 'catalog'}, 'outputSummary': '读取目录'}]})
|
||||
kwargs['emit']({'type': 'progress', 'toolCalls': [
|
||||
{'status': 'completed', 'input': {'mode': 'compare'}, 'outputSummary': '证据:Q00000001'}]})
|
||||
if self.cancel_during_native:
|
||||
self.cancel_event.set()
|
||||
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
|
||||
|
||||
def invoke(self):
|
||||
return run_native_exploration(request=copy.deepcopy(self.request), task=copy.deepcopy(self.task),
|
||||
authority=copy.deepcopy(IDENTITY), actor_root=self.actor_root, delegated=self.delegated,
|
||||
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
|
||||
manager=self.manager, state={}, cancel_event=self.cancel_event, emit=self.events.append,
|
||||
prior_evidence_files={'prior-evidence.json'},
|
||||
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
|
||||
|
||||
def test_corrected_tool_error_and_natural_stop_can_complete_with_verified_root(self):
|
||||
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
||||
self.snapshot['explorationWarnings'] = ['先前一次探索失败,后续已用完整证据核实根目标。']
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['nativeStopReason'], 'mcp_error')
|
||||
self.assertEqual(result['nativeFinishReason'], 'stop')
|
||||
self.assertIsNone(result['nativeStopSucceeded'])
|
||||
|
||||
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertEqual(result['timeline'], self.accumulator.timeline)
|
||||
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
|
||||
self.assertIn(self.snapshot['explorationWarnings'][0], result['readingSummary']['notices'])
|
||||
self.assertIn('firstUsefulMs', result['phaseTimingsMs'])
|
||||
self.assertGreaterEqual(result['phaseTimingsMs']['firstUsefulMs'], 30)
|
||||
self.delegated.call.assert_called_once_with('/api/analysis/protocol/queries', payload={}, method='POST')
|
||||
context = self.sse.call_args.kwargs['context']
|
||||
self.assertEqual(context['filters'], self.task['filters'])
|
||||
self.assertNotIn('product', context)
|
||||
self.assertIsNone(self.sse.call_args.kwargs['chat_id'])
|
||||
stored = json.loads((self.actor_root / 'private/native-completion.json').read_text())
|
||||
self.assertEqual(stored['status'], 'COMPLETED')
|
||||
self.assertEqual(self.events[-1], result)
|
||||
|
||||
def test_generated_chart_contract_is_available_before_the_single_synthesis(self):
|
||||
self.task['visualPreference']='required'
|
||||
self.visuals.return_value={'visualState':'ready','missingEvidence':[], 'visuals':[{
|
||||
'id':'actual-chart','type':'line','metric':'composite_rate','title':'正式趋势',
|
||||
'scope':{'department':'当前产业部'},'period':{'year':2026,'month':9},
|
||||
'categories':[{'key':'2026-09'}],'series':[{'values':[.95]}],
|
||||
'evidenceRefs':['Q00000001']}]}
|
||||
def synthesize(**kwargs):
|
||||
preview=kwargs['context']['_platformVisuals']
|
||||
self.assertEqual(preview['visualState'],'ready')
|
||||
self.assertEqual(preview['charts'][0]['id'],'actual-chart')
|
||||
self.assertEqual(preview['charts'][0]['evidenceRefs'],['Q00000001'])
|
||||
self.assertEqual(preview['charts'][0]['pointCount'],1)
|
||||
self.assertNotIn('series',preview['charts'][0])
|
||||
return copy.deepcopy(self.final)
|
||||
self.synthesis.side_effect=synthesize
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.visuals.assert_called_once();self.synthesis.assert_called_once()
|
||||
|
||||
def test_root_gap_remains_partial_even_with_stop_and_available_evidence(self):
|
||||
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertIn(self.snapshot['rootGoalGaps'][0], result['answer'])
|
||||
self.assertIn(self.snapshot['rootGoalGaps'][0], result['readingSummary']['notices'])
|
||||
|
||||
def test_three_no_progress_attempts_remain_partial_after_natural_stop(self):
|
||||
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
|
||||
{'executionId': IDENTITY['executionId'], 'halted': True,
|
||||
'noProgress': 3, 'queryElapsedMs': 1234})
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
self.assertEqual(result['phaseTimingsMs']['formalQuery'], 1234)
|
||||
self.assertTrue(any('连续三次' in notice for notice in result['readingSummary']['notices']))
|
||||
|
||||
def test_controlled_no_progress_stop_does_not_add_generic_tool_failure(self):
|
||||
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
|
||||
{'executionId': IDENTITY['executionId'], 'halted': True,
|
||||
'noProgress': 3, 'queryElapsedMs': 1234})
|
||||
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertEqual(result['nativeStopReason'],'mcp_error')
|
||||
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
self.assertFalse(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
|
||||
def test_corrected_error_still_requires_real_stop_and_successful_stop_control(self):
|
||||
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
||||
for reasons, stopped in (([], None), (['stop'], False)):
|
||||
with self.subTest(finish_reasons=reasons, stop_succeeded=stopped):
|
||||
self.accumulator.finish_reasons = reasons
|
||||
self.stop_succeeded = stopped
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertTrue(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
|
||||
def test_only_complete_current_scope_documents_reach_synthesis(self):
|
||||
self.snapshot['queries'].extend([{'proof': self.p_failed}, {'proof': self.p_old}])
|
||||
self.snapshot['scopeCoverage'].append(
|
||||
{'nodeId': 'failed', 'scopeId': 'scope-failed', 'complete': False})
|
||||
self.documents.extend([
|
||||
document('Q00000003', self.p_failed),
|
||||
document('Q00000004', self.p_old),
|
||||
document('Q00000005', proof('query-not-in-current-receipts')),
|
||||
document('Q00000006', self.p_current, self.p_failed),
|
||||
{'evidenceRef': 'Q00000007', 'result': {'summary': {'unproved': True}}},
|
||||
])
|
||||
result = self.invoke()
|
||||
admitted = self.synthesis.call_args.kwargs['evidence']
|
||||
self.assertEqual([doc['evidenceRef'] for doc in admitted], ['Q00000001', 'Q00000002'])
|
||||
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001', 'Q00000002'])
|
||||
self.read_evidence.assert_called_once_with(self.actor_root / 'evidence', RELEASE,
|
||||
exclude_files={'prior-evidence.json'}, allow_empty=True)
|
||||
|
||||
def test_used_refs_map_only_to_actual_current_proofs_not_unused_or_foreign_documents(self):
|
||||
self.snapshot['queries'].append({'proof': self.p_old})
|
||||
self.documents.append(document('Q00000003', self.p_old))
|
||||
self.final['usedEvidenceRefs'] = ['Q00000001', 'Q00000003', 'QDEADBEEF', 'Q00000001']
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
|
||||
self.assertNotIn('query-group', result['conclusionProtocolQueryIds'])
|
||||
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
|
||||
|
||||
def test_claimed_sufficient_answer_without_current_cited_proof_remains_partial(self):
|
||||
self.final['usedEvidenceRefs'] = ['QDEADBEEF']
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(result['conclusionProtocolQueryIds'], [])
|
||||
self.assertTrue(any('实际查询证明' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
||||
|
||||
def test_cancelled_native_turn_never_reads_query_snapshot_or_synthesizes(self):
|
||||
for cancellation in ('event', 'native_status'):
|
||||
with self.subTest(cancellation=cancellation):
|
||||
self.cancel_event.clear()
|
||||
self.cancel_during_native = cancellation == 'event'
|
||||
self.native_status = 'CANCELLED' if cancellation == 'native_status' else 'COMPLETED'
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], 'CANCELLED')
|
||||
self.assertEqual(result['protocolEvidence'], [])
|
||||
self.assertEqual(result['answer'], '')
|
||||
self.delegated.call.assert_not_called()
|
||||
self.read_evidence.assert_not_called()
|
||||
self.synthesis.assert_not_called()
|
||||
self.visuals.assert_not_called()
|
||||
|
||||
def test_auto_visual_failure_stays_in_appendix_but_required_visual_failure_is_partial(self):
|
||||
self.visuals.return_value = {'visualState': 'failed', 'visuals': [],
|
||||
'missingEvidence': ['图表尚未完整生成,可独立恢复。',
|
||||
'queryPlan共有15张图表,超过12张呈现上限,已截取前12张;结果为部分完成。']}
|
||||
for preference, expected in (('auto', 'COMPLETED'), ('required', 'PARTIAL')):
|
||||
with self.subTest(visualPreference=preference):
|
||||
self.task['visualPreference'] = preference
|
||||
result = self.invoke()
|
||||
self.assertEqual(result['status'], expected)
|
||||
self.assertIn('图表尚未完整生成,可独立恢复。', result['answer'])
|
||||
if preference == 'auto':
|
||||
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['readingSummary']['notices'])
|
||||
self.assertNotIn('结果为部分完成', result['answer'])
|
||||
self.assertIn('共有15张图表', result['answer'])
|
||||
else:
|
||||
self.assertIn('图表尚未完整生成,可独立恢复。', result['readingSummary']['notices'])
|
||||
self.assertIn('结果为部分完成', result['answer'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,125 @@
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)]
|
||||
from nao_agent_task import plan_task, normalize_task, TaskUnavailable
|
||||
from test_nao_agent_task import task_value, semantic_ok
|
||||
|
||||
|
||||
def group_plan(**updates):
|
||||
return task_value(kind='metrics', goal='比较9月与8月各业务分组综合合格率,找出下降最多的分组',
|
||||
dimensions=['group'], dimensionStages={}, stages=[], fields=['rate'],
|
||||
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8},
|
||||
comparisonRequired=True, trendRequired=False, queryPlan=[{
|
||||
'nodeId': 'groups', 'purpose': 'metrics', 'fields': ['rate'], 'stages': [],
|
||||
'comparisonRequired': True, 'selector': {'kind': 'catalog_members', 'dimension': 'group',
|
||||
'names': ['部门甲-国内', '部门甲-海外']}, 'visualPreference': 'none'}], **updates)
|
||||
|
||||
|
||||
CONTEXT = {'_protocol': {'version': 1}, '_businessCatalog': {
|
||||
'groups': ['部门甲-国内', '部门甲-海外'], 'departments': ['部门甲'],
|
||||
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
|
||||
|
||||
|
||||
class PlanRecoveryTests(unittest.TestCase):
|
||||
def run_plan(self, replies, **kwargs):
|
||||
self.payloads = []
|
||||
def opener(request, timeout):
|
||||
body = json.loads(request.data)
|
||||
self.payloads.append(json.loads(body['messages'][1]['content']))
|
||||
response = replies.pop(0)
|
||||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
|
||||
'content': json.dumps(response)}}]}).encode())
|
||||
return plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='9月份哪个业务分组下滑最多', context=copy.deepcopy(CONTEXT),
|
||||
cancel_event=threading.Event(), opener=opener, **kwargs)
|
||||
|
||||
def test_derivable_dimension_stage_summary_is_repaired_without_another_generation(self):
|
||||
broken = group_plan()
|
||||
broken['dimensionStages'] = {'group': ['black']}
|
||||
task = self.run_plan([broken, {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertEqual(task['dimensionStages'], {'group': []})
|
||||
self.assertTrue(task['planRecovery']['repairs'])
|
||||
self.assertEqual(task['queryPlan'][0]['selector']['names'], CONTEXT['_businessCatalog']['groups'])
|
||||
|
||||
def test_omitted_read_flags_are_safely_defaulted_then_independently_reviewed(self):
|
||||
broken = group_plan(); del broken['verifySource']; broken['fullDetails'] = None
|
||||
task = self.run_plan([broken, {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertFalse(task['verifySource']); self.assertFalse(task['fullDetails'])
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'], 1)
|
||||
self.assertEqual(len(self.payloads), 2)
|
||||
|
||||
def test_root_group_cannot_be_silently_removed_when_nodes_only_query_overall(self):
|
||||
broken = group_plan()
|
||||
broken['queryPlan'][0]['selector'] = {'kind': 'base'}
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task(broken, '9月份哪个业务分组下滑最多', CONTEXT)
|
||||
task = self.run_plan([broken, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertEqual(task['dimensions'], ['group'])
|
||||
self.assertEqual(self.payloads[1]['validationDetails']['expected'], [])
|
||||
self.assertIn('authorizedCatalog', self.payloads[1]['repairContext'])
|
||||
|
||||
def test_three_distinct_protocol_errors_do_not_exhaust_a_progressing_repair(self):
|
||||
a = group_plan(); del a['queryPlan']
|
||||
b = group_plan(); b['queryPlan'][0]['nodeId'] = '业务分组'
|
||||
c = group_plan(); c['queryPlan'][0]['selector']['names'] = ['未授权组']
|
||||
task = self.run_plan([a, b, c, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertEqual(task['dimensions'], ['group'])
|
||||
self.assertEqual(task['planRecovery']['modelAttempts'], 4)
|
||||
self.assertEqual(self.payloads[3]['repairMode'], 'rebuild_nodes')
|
||||
|
||||
def test_automatic_summary_repair_never_drops_requested_fields_or_scopes(self):
|
||||
from nao_plan_recovery import repair_summary
|
||||
broken = group_plan(); broken['fields'] = ['rate', 'scrapWeight']
|
||||
self.assertIsNone(repair_summary(broken, CONTEXT))
|
||||
|
||||
broken = group_plan(); broken['dimensions'] = ['group', 'product']
|
||||
self.assertIsNone(repair_summary(broken, CONTEXT))
|
||||
broken = group_plan(); broken['comparisonRequired'] = True
|
||||
broken['queryPlan'][0]['comparisonRequired'] = False
|
||||
self.assertIsNone(repair_summary(broken, CONTEXT))
|
||||
|
||||
def test_malformed_model_shapes_get_another_attempt_instead_of_crashing_recovery(self):
|
||||
for corrupt in ('dimensions', 'ranking_dimension', 'selector_dimension'):
|
||||
broken = group_plan()
|
||||
if corrupt == 'dimensions': broken['dimensions'] = 1
|
||||
elif corrupt == 'ranking_dimension':
|
||||
broken['queryPlan'][0].update(purpose='ranking', dimension={}, fields=['totalWeight'], stages=['black'], limit=3)
|
||||
else: broken['queryPlan'][0]['selector']['dimension'] = []
|
||||
with self.subTest(corrupt=corrupt):
|
||||
planned = self.run_plan([broken, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertEqual(planned['planRecovery']['modelAttempts'], 2)
|
||||
|
||||
def test_real_business_decision_returns_choices_without_executing_nodes(self):
|
||||
pending = group_plan()
|
||||
pending.pop('queryPlan'); pending.pop('comparisonPeriod')
|
||||
pending.update(needsClarification=True, clarification='请确认要比较的年份',
|
||||
clarificationRequest={'options': [{'label': '2026年', 'value': '2026年'}]})
|
||||
task = self.run_plan([pending, {**semantic_ok(), 'requiresComparison': True}])
|
||||
self.assertTrue(task['needsClarification'])
|
||||
self.assertEqual(task['clarificationRequest']['options'], [{'label': '2026年', 'value': '2026年'}])
|
||||
self.assertNotIn('queryPlan', task)
|
||||
|
||||
def test_all_group_query_accepts_only_its_catalog_bound_parent_department(self):
|
||||
from nao_protocol_execution import _validate_protocol_scope
|
||||
root = normalize_task(group_plan(), '9月份哪个业务分组下滑最多', CONTEXT)
|
||||
node = root['queryPlan'][0]
|
||||
child = {**root, 'kind': 'metrics', 'dimensions': [], 'dimensionStages': {},
|
||||
'filters': {'group': '部门甲-国内', 'department': '部门甲'},
|
||||
'visualPreference': node['visualPreference'], 'visualType': node['visualType']}
|
||||
child.pop('queryPlan')
|
||||
scope = {'nodeId': 'groups', 'scopeId': 'group-domestic', 'purpose': 'metrics', 'task': child}
|
||||
catalog = {'groupDepartments': {'部门甲-国内': '部门甲'}}
|
||||
accepted = _validate_protocol_scope(scope, node, root, 'release', catalog=catalog)
|
||||
self.assertEqual(accepted['task']['filters']['department'], '部门甲')
|
||||
child['filters']['department'] = '未授权部门'
|
||||
with self.assertRaises(ValueError):
|
||||
_validate_protocol_scope(scope, node, root, 'release', catalog=catalog)
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""A saved clarification selection survives only authorized fresh retries."""
|
||||
import io
|
||||
import json
|
||||
import copy
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_product_identification import identify_products
|
||||
|
||||
ACTOR = '11111111-1111-4111-8111-111111111111'
|
||||
SESSION = '22222222-2222-4222-8222-222222222222'
|
||||
RELEASE = '33333333-3333-4333-8333-333333333333'
|
||||
TURN = '55555555-5555-4555-8555-555555555555'
|
||||
|
||||
|
||||
class ProductClarificationRuntimeTests(unittest.TestCase):
|
||||
def exercise(self, *, restart=False, revoke=False, other_release=False):
|
||||
opts = {'products': ['18X15X4+T', '18X15X4+Z'], 'organizationPaths': []}
|
||||
pending_match = identify_products('今年18x15x4合格率变化趋势', opts)['productMatch']
|
||||
previous = {'kind':'analysis', 'question':'选+T', 'goal':'今年18x15x4合格率变化趋势',
|
||||
'rootGoal':'今年18x15x4合格率变化趋势', 'period':{'year':2026},
|
||||
'filters':{'product':'18X15X4+T'}, 'stages':[], 'needsClarification':False,
|
||||
'productResolution':{'originalQuestion':'今年18x15x4合格率变化趋势',
|
||||
'productMatch':pending_match,'selectedName':'18X15X4+T',
|
||||
'selectionQuestion':'选+T'}}
|
||||
if revoke:
|
||||
opts['products'] = ['18X15X4+Z', '28X20X3+T']
|
||||
context = {'releaseId':RELEASE, '_serverQuestions':[
|
||||
{'turnId':'old-hello', 'question':'你好', 'status':'COMPLETED'},
|
||||
{'turnId':'failed-choice', 'question':'选+T', 'status':'FAILED'}],
|
||||
'_serverHistory':[{'turnId':'failed-choice','question':'选+T','answer':'错误旧答复'}]}
|
||||
if restart:
|
||||
context['_restartConversation'] = True
|
||||
payload = {'turnId':TURN,'sessionId':SESSION,'actorId':ACTOR,'releaseId':RELEASE,
|
||||
'accessToken':'synthetic-delegation','question':'选+T','context':context}
|
||||
planned = {'kind':'analysis','goal':'test','question':'选+T','filters':{},'stages':[],
|
||||
'needsClarification':True,'clarification':'请选择规格','verifySource':False,'fullDetails':False}
|
||||
captured = {}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
||||
saved=root/'private'/'last-agent-task.json'
|
||||
saved.write_text(json.dumps({'releaseId':('44444444-4444-4444-8444-444444444444' if other_release else RELEASE),
|
||||
'sessionId':SESSION,'baseMetricFrame':None,'task':previous}),encoding='utf8')
|
||||
saved.chmod(0o600)
|
||||
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
|
||||
state=MagicMock();state.get_chat_id.return_value=None
|
||||
gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1'
|
||||
api=MagicMock()
|
||||
api.call.side_effect=lambda path,**kwargs: (opts if path.startswith('/api/options') else
|
||||
{'kind':'analysis_catalog','extraFields':[{'key':'customerCode','label':'客户代码'}]}
|
||||
if path=='/api/analysis/tools/catalog' else {})
|
||||
delegated=MagicMock();delegated.login.return_value=api
|
||||
def plan(**kwargs):
|
||||
captured.update({k:copy.deepcopy(v) for k,v in kwargs.items() if k != 'cancel_event'})
|
||||
return dict(planned)
|
||||
with ExitStack() as stack:
|
||||
for name,value in {
|
||||
'RuntimeManager':MagicMock(return_value=manager), 'SessionState':MagicMock(return_value=state),
|
||||
'write_access_file':MagicMock(return_value=root/'access.json'), '_input_cancel_listener':MagicMock(),
|
||||
'_model_settings':MagicMock(return_value=('https://offline.invalid/v1','key','model',
|
||||
{'enabled':False,'effort':'auto','protocol':'thinking'})),
|
||||
'_model_gate_settings':MagicMock(return_value=(1,'key')), 'DelegatedApi':MagicMock(return_value=delegated),
|
||||
'plan_agent_task':plan,
|
||||
}.items():
|
||||
stack.enter_context(patch.object(runtime,name,value))
|
||||
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
|
||||
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
|
||||
self.assertEqual(result['status'],'NEEDS_CLARIFICATION')
|
||||
return captured, previous
|
||||
|
||||
def test_duplicate_failed_selection_keeps_goal_but_discards_failed_answer(self):
|
||||
captured, previous = self.exercise()
|
||||
self.assertEqual(captured['previous'], previous)
|
||||
self.assertTrue(captured['fresh_retry'])
|
||||
self.assertEqual(captured['context']['_serverHistory'], [])
|
||||
match = captured['context']['_businessCatalog']['productMatch']
|
||||
self.assertTrue(match['selectedFromClarification'])
|
||||
self.assertEqual(match['candidates'][0]['name'], '18X15X4+T')
|
||||
self.assertEqual(match['previousQuestion'], '今年18x15x4合格率变化趋势')
|
||||
self.assertEqual(captured['context']['_businessCatalog']['extraFields'], [{'key':'customerCode','label':'客户代码'}])
|
||||
|
||||
def test_explicit_retry_also_restores_only_authorized_selection(self):
|
||||
captured, previous = self.exercise(restart=True)
|
||||
self.assertEqual(captured['previous'], previous)
|
||||
self.assertTrue(captured['fresh_retry'])
|
||||
self.assertEqual(captured['context']['_businessCatalog']['productMatch']['candidates'][0]['name'], '18X15X4+T')
|
||||
|
||||
def test_revoked_or_different_release_never_restores_selection(self):
|
||||
for kwargs in ({'restart':True,'revoke':True},{'restart':True,'other_release':True}):
|
||||
with self.subTest(kwargs=kwargs):
|
||||
captured, _ = self.exercise(**kwargs)
|
||||
self.assertIsNone(captured['previous'])
|
||||
self.assertEqual(captured['context']['_businessCatalog']['productMatch']['status'], 'none')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,13 @@
|
||||
"""An explicit local deployment setting survives the normal project launcher."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
spec=importlib.util.spec_from_file_location('project_launch',Path(__file__).resolve().parents[1]/'scripts/project.py')
|
||||
project=importlib.util.module_from_spec(spec);spec.loader.exec_module(project)
|
||||
class ProjectProtocolSettingTest(unittest.TestCase):
|
||||
def test_setting_is_explicit_and_defaults_to_disabled(self):
|
||||
base=dict(mysqlPort=53306,mysqlDatabase='synthetic',mysqlUser='u',mysqlPassword='unused',adminUsername='u',adminPassword='unused')
|
||||
for setting,expected in [(None,'false'),(False,'false'),(True,'true')]:
|
||||
value={**base,**({} if setting is None else {'analysisProtocolV1Enabled':setting})}
|
||||
with self.subTest(setting=setting),patch.object(project,'config',return_value=value):self.assertEqual(project.environment()['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'],expected)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""An explicitly versioned request must never downgrade into actor-local state."""
|
||||
import io
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
|
||||
|
||||
class ProtocolRuntimeBoundaryTest(unittest.TestCase):
|
||||
def payload(self, context):
|
||||
return dict(turnId='55555555-5555-4555-8555-555555555555',
|
||||
sessionId='22222222-2222-4222-8222-222222222222',
|
||||
actorId='11111111-1111-4111-8111-111111111111',
|
||||
releaseId='33333333-3333-4333-8333-333333333333',
|
||||
accessToken='synthetic', question='取消客户筛选', context=context)
|
||||
|
||||
def test_invalid_protocol_marker_stops_before_any_actor_or_model_state(self):
|
||||
invalid = [None, [], '1', {}, {'version': 2}, {'version': True},
|
||||
{'version': 1.0}, {'version': '1'}]
|
||||
for marker in invalid:
|
||||
with self.subTest(marker=marker):
|
||||
manager = MagicMock()
|
||||
manager.acquire_actor.side_effect = AssertionError('legacy actor state reached')
|
||||
with patch.object(runtime, 'RuntimeManager', return_value=manager), \
|
||||
patch.object(runtime, 'plan_agent_task') as planner:
|
||||
result = runtime.run_turn(self.payload({'_protocol': marker,
|
||||
'_serverHistory': [{'answer': 'OLD'}], '_task': {'filters': {'product': 'OLD'}}}),
|
||||
stdout=io.StringIO(), stdin=io.StringIO())
|
||||
self.assertEqual(result['status'], 'FAILED')
|
||||
self.assertIn('分析请求', result['message'])
|
||||
self.assertNotIn('本机', result['message'])
|
||||
manager.acquire_actor.assert_not_called()
|
||||
planner.assert_not_called()
|
||||
self.assertEqual(result['toolCalls'], [])
|
||||
|
||||
def test_valid_version_and_absent_legacy_marker_remain_distinct(self):
|
||||
legacy = runtime._normalize_request(self.payload({'year': 2026}))
|
||||
self.assertNotIn('_protocol', legacy['context'])
|
||||
valid = runtime._normalize_request(self.payload({'_protocol': {'version': 1,
|
||||
'attemptId': '22222222-2222-4222-8222-222222222222'}}))
|
||||
self.assertEqual(valid['context']['_protocol']['version'], 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from analysis_visuals import build_visuals
|
||||
from nao_agent_report import _build_trend_report
|
||||
|
||||
|
||||
RELEASE = 'signed-release'
|
||||
FILTERS = {'department': '部门甲'}
|
||||
|
||||
|
||||
def trend_document(black_total, bright_total=100.0):
|
||||
rows = []
|
||||
for month, black_weight in ((1, 120.0), (2, black_total)):
|
||||
rows.append({'period': f'2026-{month:02d}', 'periodLabel': f'2026-{month:02d}',
|
||||
'black': {'totalWeight': black_weight, 'scrapWeight': 10.0,
|
||||
'scrapRate': 0.08 if month == 1 else 0.09,
|
||||
'rate': 0.95, 'qualifiedWeight': 110.0},
|
||||
'bright': {'totalWeight': bright_total, 'scrapWeight': 8.0,
|
||||
'scrapRate': 0.08, 'rate': 0.96, 'qualifiedWeight': 92.0},
|
||||
'coverage': {'blackRows': 1, 'brightRows': 1}})
|
||||
result = {'kind': 'quality_metrics', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2},
|
||||
'filters': copy.deepcopy(FILTERS), 'view': 'overall',
|
||||
'summary': {'black': {'totalWeight': black_total, 'scrapWeight': 20.0},
|
||||
'bright': {'totalWeight': bright_total, 'scrapWeight': 16.0}},
|
||||
'trend': rows, 'trendCount': 2, 'trendTruncated': False,
|
||||
'coverage': {'sourceMonths': [
|
||||
{'year': 2026, 'month': 1, 'blackRows': 1, 'brightRows': 1},
|
||||
{'year': 2026, 'month': 2, 'blackRows': 1, 'brightRows': 1}]}}
|
||||
return {'tool': 'quality_data', 'mode': 'query', 'releaseId': RELEASE,
|
||||
'evidenceRef': 'Q1234ABCD', 'result': result}
|
||||
|
||||
|
||||
def child_task(field, *, visual_type='line', period_coverage='available'):
|
||||
return {'version': 1, 'kind': 'metrics', 'question': '黑片投入重量趋势',
|
||||
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2},
|
||||
'filters': copy.deepcopy(FILTERS), 'dimensions': [], 'dimensionStages': {},
|
||||
'stages': ['black'], 'fields': [field], 'trendRequired': True,
|
||||
'comparisonRequired': False, 'comparisonPeriod': None,
|
||||
'periodCoverage': period_coverage, 'visualPreference': 'auto',
|
||||
'visualType': visual_type}
|
||||
|
||||
|
||||
class QueryPlanSignedWeightTrendTests(unittest.TestCase):
|
||||
def test_black_total_weight_keeps_signed_correction_in_two_point_line_and_report(self):
|
||||
document = trend_document(-10.05)
|
||||
task = child_task('totalWeight', visual_type='line')
|
||||
context = {'releaseId': RELEASE, 'period': task['period'], 'filters': FILTERS,
|
||||
'_agentTask': task}
|
||||
visuals = build_visuals([document], task['question'], 'turn-signed', context=context)
|
||||
chart = next(item for item in visuals['visuals'] if item['metric'] == 'black_total_weight')
|
||||
self.assertEqual(chart['type'], 'line')
|
||||
self.assertEqual(chart['series'][0]['values'], [120.0, -10.05])
|
||||
self.assertIn('负向修正原值保留', chart['subtitle'])
|
||||
report = _build_trend_report([document], context, task['question'])
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('2月-10.05 kg', report['facts'][0]['text'])
|
||||
|
||||
def test_negative_bright_total_is_rejected_and_area_still_requires_three_points(self):
|
||||
document = trend_document(100.0, bright_total=-1.0)
|
||||
bright_task = {**child_task('totalWeight', visual_type='line'), 'stages': ['bright'],
|
||||
'question': '亮片投入重量趋势'}
|
||||
bright_context = {'releaseId': RELEASE, 'period': bright_task['period'],
|
||||
'filters': FILTERS, '_agentTask': bright_task}
|
||||
bright_charts = build_visuals([document], bright_task['question'], 'turn-bright',
|
||||
context=bright_context)['visuals']
|
||||
self.assertFalse(any(item['metric'] == 'bright_total_weight' for item in bright_charts))
|
||||
|
||||
area_task = child_task('totalWeight', visual_type='area')
|
||||
area_context = {'releaseId': RELEASE, 'period': area_task['period'],
|
||||
'filters': FILTERS, '_agentTask': area_task}
|
||||
area_charts = build_visuals([trend_document(-10.05)], area_task['question'], 'turn-area',
|
||||
context=area_context)['visuals']
|
||||
self.assertFalse(any(item['metric'] == 'black_total_weight' for item in area_charts))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Regressions for source/row requirements that cannot be satisfied by a draft."""
|
||||
import copy
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
|
||||
from nao_evidence_finalizer import coverage_sufficient
|
||||
import test_agent_runtime as agent_runtime_fixture
|
||||
import test_weight_ranking as weight_fixture
|
||||
from test_nao_agent_task import task_value
|
||||
|
||||
|
||||
def rows_plan():
|
||||
return task_value(kind='explain',goal='逐条提供全部原始记录',freshQuery=False,historyMode='none',
|
||||
dimensions=[],dimensionStages={},stages=['black'],fields=[],fullDetails=True,
|
||||
period={'year':2026,'month':9})
|
||||
|
||||
|
||||
class RequiredReadCompletionTests(unittest.TestCase):
|
||||
def test_full_details_always_requires_a_fresh_read(self):
|
||||
planned=normalize_task(rows_plan(),'请逐条列出全部原始记录')
|
||||
self.assertTrue(planned['fullDetails'])
|
||||
self.assertTrue(planned['freshQuery'])
|
||||
|
||||
def test_reviewer_cannot_claim_all_rows_are_required_without_evidence(self):
|
||||
inconsistent={'aligned':True,'requiresEvidence':False,
|
||||
'requiresSourceVerification':False,'requiresAllSourceRows':True}
|
||||
with patch('nao_agent_task._model_json',side_effect=[rows_plan(),inconsistent]*3):
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',
|
||||
question='请逐条列出全部原始记录',context={},cancel_event=threading.Event())
|
||||
|
||||
def test_unverified_native_draft_cannot_finish_when_rows_required(self):
|
||||
harness=agent_runtime_fixture.AgentRuntimeIntegrationTest()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
# Simulate a legacy or malformed plan even after planner safeguards.
|
||||
planned=agent_runtime_fixture.task('请逐条列出全部原始记录',kind='explain',freshQuery=False,
|
||||
historyMode='none',dimensions=[],dimensionStages={},fields=[],stages=['black'],fullDetails=True)
|
||||
planned['freshQuery']=False;planned['requiresVerifiedEvidence']=False
|
||||
def native(**kwargs):
|
||||
stream=runtime.ConversationAccumulator()
|
||||
stream.blocks['native']='已经列出全部原始记录。';stream.block_order.append('native')
|
||||
stream.record_event('assistant',text='已经列出全部原始记录。')
|
||||
return stream,'COMPLETED','stop',None
|
||||
with patch('nao_agent_task.plan_repairs',return_value=[]):
|
||||
result=harness.run_mocked(Path(folder),planned,native)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertNotIn('已经列出全部原始记录',result['answer'])
|
||||
self.assertFalse(any('已经列出全部原始记录' in entry.get('text','') for entry in result['timeline']))
|
||||
|
||||
def test_complete_rankings_do_not_satisfy_source_or_row_requirements(self):
|
||||
doc,_=weight_fixture.WeightRankingTests().run_rank()
|
||||
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
|
||||
for flags in ({'verifySource':True},{'fullDetails':True}):
|
||||
with self.subTest(flags=flags):
|
||||
planned=weight_fixture.planned(**flags);context=bind_task({'releaseId':'release-a'},planned)
|
||||
self.assertFalse(coverage_sufficient([doc,department],weight_fixture.QUESTION,context))
|
||||
|
||||
def test_complete_rankings_with_required_source_or_all_rows_can_finish(self):
|
||||
doc,_=weight_fixture.WeightRankingTests().run_rank()
|
||||
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
|
||||
side_context={'releaseId':'release-a','year':2026,'stage':'bright','includeUnassigned':False}
|
||||
def investigation(action,side):
|
||||
return {'mode':'investigate','evidenceRef':'Q00000003','result':{'kind':'quality_investigation',
|
||||
'releaseId':'release-a','period':{'year':2026},'filters':{},'stage':'bright','action':action,'current':side}}
|
||||
source=investigation('source',{'context':side_context,'aggregateComplete':True,'totalRows':2,
|
||||
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,'expectedRows':2,
|
||||
'verifiedRows':2,'missingRows':0,'mismatchRows':0}})
|
||||
def page(offset,more):
|
||||
return investigation('rows',{'context':side_context,'recordFilter':'all','matchingRows':4,
|
||||
'offset':offset,'returnedRows':2,'rows':[{'id':offset},{'id':offset+1}],'hasMore':more})
|
||||
for flags,extra in (({'verifySource':True},[source]),({'fullDetails':True},[page(0,True),page(2,False)])):
|
||||
with self.subTest(flags=flags):
|
||||
planned=weight_fixture.planned(**flags);context=bind_task({'releaseId':'release-a'},planned)
|
||||
self.assertTrue(coverage_sufficient([doc,department,*extra],weight_fixture.QUESTION,context))
|
||||
|
||||
def test_semantic_all_rows_accepts_complete_multiple_pages_but_not_gaps(self):
|
||||
question='列出黑片全部原始记录'
|
||||
planned=normalize_task(task_value(kind='details',fullDetails=True,dimensions=[],dimensionStages={},
|
||||
stages=['black'],fields=[],period={'year':2026,'month':9},comparisonRequired=False),question)
|
||||
context=bind_task({'releaseId':'release-a'},planned)
|
||||
def page(offset, more, total=4):
|
||||
side={'context':{'releaseId':'release-a','year':2026,'month':9,'stage':'black','includeUnassigned':False},
|
||||
'recordFilter':'all','matchingRows':total,'offset':offset,'returnedRows':2,
|
||||
'rows':[{'id':offset},{'id':offset+1}],'hasMore':more}
|
||||
return {'mode':'investigate','evidenceRef':'QAAAAAAAA','result':{'kind':'quality_investigation',
|
||||
'releaseId':'release-a','period':{'year':2026,'month':9},'filters':{},'stage':'black',
|
||||
'action':'rows','current':side}}
|
||||
pages=[page(0,True),page(2,False)]
|
||||
self.assertTrue(coverage_sufficient(pages,question,context))
|
||||
self.assertFalse(coverage_sufficient([pages[0],page(3,False)],question,context))
|
||||
self.assertFalse(coverage_sufficient([pages[0],page(2,False,total=5)],question,context))
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,48 @@
|
||||
import json,sys,unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
class TimelineTests(unittest.TestCase):
|
||||
def test_real_text_tools_results_are_interleaved_and_reasoning_is_private(self):
|
||||
a=runtime.ConversationAccumulator()
|
||||
a.feed({'type':'reasoning-delta','delta':'hidden chain'})
|
||||
a.feed({'type':'text-delta','id':'intro','delta':'先核查期间与覆盖。'})
|
||||
call={'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q1','input':{'tool':'quality_data','arguments':{'mode':'defects','stage':'black','apiKey':'secret'}}}
|
||||
a.feed(call); a.feed(call)
|
||||
a.feed({'type':'tool-output-available','toolCallId':'q1','output':{'evidenceRef':'Q123','rows':[{'weight':12}], 'categoryReconciliation':[{'reconciliationStatus':'MISMATCH'}]}})
|
||||
a.feed({'type':'text-delta','id':'final','delta':'来源缺陷只作为变化线索。'})
|
||||
self.assertEqual([e['kind'] for e in a.timeline],['assistant','tool_start','tool_result','assistant'])
|
||||
self.assertEqual([e['sequence'] for e in a.timeline],[1,2,3,4])
|
||||
encoded=json.dumps(a.timeline,ensure_ascii=False)
|
||||
self.assertNotIn('hidden chain',encoded); self.assertNotIn('secret',encoded)
|
||||
self.assertIn('Q123',encoded); self.assertIn('未对账',encoded)
|
||||
def test_unmatched_result_does_not_invent_a_tool_start(self):
|
||||
a=runtime.ConversationAccumulator();a.feed({'type':'tool-output-available','toolCallId':'missing','output':{'rows':[]}})
|
||||
self.assertEqual(a.timeline,[])
|
||||
def test_resumed_provider_text_block_stays_after_the_tool(self):
|
||||
a=runtime.ConversationAccumulator()
|
||||
a.feed({'type':'text-delta','id':'same','delta':'先查询。'})
|
||||
a.feed({'type':'tool-input-available','toolName':'read','toolCallId':'r1'})
|
||||
a.feed({'type':'tool-output-available','toolCallId':'r1','output':{}})
|
||||
a.feed({'type':'text-delta','id':'same','delta':'已查询。'})
|
||||
self.assertEqual([e['kind'] for e in a.timeline],['assistant','tool_start','tool_result','assistant'])
|
||||
self.assertEqual(a.timeline[0]['text'],'先查询。')
|
||||
self.assertEqual(a.timeline[-1]['text'],'已查询。')
|
||||
def test_comparison_side_reconciliation_is_also_disclosed_and_null_is_safe(self):
|
||||
text=runtime.public_tool_result({'categoryReconciliation':None,'comparisonCategoryReconciliation':[{'reconciliationStatus':'DIFF'}],'truncated':True})
|
||||
self.assertIn('未对账',text);self.assertIn('仅覆盖返回集合',text)
|
||||
text=runtime.public_tool_result({'evidenceRef':'QABC12345','summary':{'compositeRate':0.95},'token':'private'})
|
||||
self.assertIn('QABC12345',text);self.assertNotIn('private',text);self.assertIn('95.00%',text)
|
||||
if __name__=='__main__': unittest.main()
|
||||
|
||||
class InvestigationEvidenceStatusTests(unittest.TestCase):
|
||||
def test_expected_vcm_is_not_an_unresolved_warning_but_non_vcm_remains(self):
|
||||
from nao_quality_runtime import public_evidence_checks
|
||||
base={'kind':'quality_investigation','releaseId':'r','stage':'bright','period':{'year':2026,'month':9},'filters':{'group':'工业电机-VCM'},'current':{'categories':[{'category':'镀废','expectedTotalsOnly':{'rows':419},'unexpectedDifferenceRows':0,'requiresFurtherInvestigation':False}],'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True}}}
|
||||
self.assertEqual(list(public_evidence_checks(base).values()),['expected'])
|
||||
base['current']['categories'][0].update(unexpectedDifferenceRows=12,requiresFurtherInvestigation=True)
|
||||
self.assertEqual(list(public_evidence_checks(base).values()),['unresolved'])
|
||||
def test_source_verification_failure_cannot_be_hidden_by_vcm_policy(self):
|
||||
from nao_quality_runtime import public_evidence_checks
|
||||
r={'kind':'quality_investigation','current':{'categories':[{'category':'镀废','expectedTotalsOnly':{'rows':419},'unexpectedDifferenceRows':0,'requiresFurtherInvestigation':False}],'sourceVerification':{'complete':False}}}
|
||||
self.assertEqual(list(public_evidence_checks(r).values()),['unresolved'])
|
||||
@@ -0,0 +1,181 @@
|
||||
import sys,unittest,json,io,threading
|
||||
from pathlib import Path
|
||||
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
|
||||
from nao_query_contract import bind_query,canonical_filters
|
||||
from nao_answer_focus import answer_focus,focus_fallback,missing_focus,with_focus_conclusion
|
||||
from nao_quality_tools import normalize_filters,QualityTools,QualityToolError
|
||||
from nao_agent_task import normalize_task
|
||||
from test_nao_agent_task import task_value
|
||||
import nao_evidence_finalizer as finalizer
|
||||
|
||||
class ScopeContractTest(unittest.TestCase):
|
||||
def task(self):
|
||||
return normalize_task(task_value(dimensions=['group'],dimensionStages={'group':[]},
|
||||
filterUpdates={'department':'新能源汽车磁钢'},period={'year':2026,'month':9},
|
||||
comparisonPeriod={'year':2026,'month':8}), '主要是国内还是国外的')
|
||||
def contract(self):return {'task':self.task(),'releaseId':'same-release'}
|
||||
def test_business_alias_is_department_not_union_of_groups(self):
|
||||
self.assertEqual(canonical_filters({'department':'汽车交通'}),{'department':'新能源汽车磁钢'})
|
||||
self.assertEqual(canonical_filters({'group':'汽车交通'}),{'department':'新能源汽车磁钢'})
|
||||
filters,_=normalize_filters('auto_combined',{},scope_was_explicit=True)
|
||||
self.assertEqual(filters,{'department':'新能源汽车磁钢','includeUnassigned':False})
|
||||
task=normalize_task(task_value(dimensions=['group'],filterUpdates={'department':'汽车交通'}),'汽车交通合格率')
|
||||
self.assertEqual(task['filters'],{'department':'新能源汽车磁钢'})
|
||||
def test_missing_query_scope_inherits_frozen_parent(self):
|
||||
q=bind_query({'mode':'changes','dimension':'group'},self.contract())
|
||||
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(q['period'],{'year':2026,'month':9})
|
||||
self.assertEqual(q['comparisonPeriod'],{'year':2026,'month':8})
|
||||
self.assertEqual(q['releaseId'],'same-release')
|
||||
def test_native_cannot_replace_department_with_group_union(self):
|
||||
with self.assertRaisesRegex(ValueError,'范围'):
|
||||
bind_query({'mode':'changes','dimension':'group','filters':{'group':'union-domestic-overseas'}},self.contract())
|
||||
def test_native_cannot_change_release_period_or_expand_scope(self):
|
||||
for changed in ({'releaseId':'different'}, {'period':{'year':2026,'month':7}},
|
||||
{'filters':{'department':'其他产业部'}},{'filters':{'includeUnassigned':True}}):
|
||||
with self.subTest(changed=changed),self.assertRaises(ValueError):
|
||||
bind_query({'mode':'changes','dimension':'group',**changed},self.contract())
|
||||
def test_explicit_child_compare_retains_parent_and_catalog_remains_available(self):
|
||||
q=bind_query({'mode':'compare','filters':{'group':'汽车交通-国内'}},self.contract())
|
||||
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(q['filters']['group'],'汽车交通-国内')
|
||||
self.assertEqual(bind_query({'mode':'catalog'},self.contract()),{'mode':'catalog'})
|
||||
def test_query_guard_runs_before_any_http_call(self):
|
||||
called=[]
|
||||
tools=QualityTools(api_factory=lambda:called.append(True),query_contract=self.contract())
|
||||
with self.assertRaises(QualityToolError):
|
||||
tools.quality_data({'mode':'changes','dimension':'group','filters':{'group':'wrong-union'}})
|
||||
self.assertEqual(called,[])
|
||||
def test_rank_binds_authorized_column_filters_but_other_modes_still_reject_them(self):
|
||||
contract=self.contract()
|
||||
rule={'field':'customer_code','stages':['bright'],'operator':'in','values':['0012']}
|
||||
contract['task']['filters']['columnFilters']=[rule]
|
||||
bound=bind_query({'mode':'rank','dimension':'group','stage':'bright','metric':'totalWeight'},contract)
|
||||
self.assertEqual(bound['columnFilters'],[rule])
|
||||
self.assertEqual(bound['period'],contract['task']['period'])
|
||||
self.assertEqual(bound['filters']['department'],'新能源汽车磁钢')
|
||||
with self.assertRaisesRegex(ValueError,'columnFilters'):
|
||||
bind_query({'mode':'changes','dimension':'group','columnFilters':[rule]},contract)
|
||||
def test_requested_child_drilldown_keeps_parent_without_filtering_its_own_rank(self):
|
||||
contract=self.contract();contract['task']['dimensions']=['group','product']
|
||||
contract['task']['dimensionStages']['product']=[]
|
||||
q=bind_query({'mode':'changes','dimension':'product','filters':{'group':'汽车交通-国内'}},contract)
|
||||
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(q['filters']['group'],'汽车交通-国内')
|
||||
with self.assertRaises(ValueError):
|
||||
bind_query({'mode':'changes','dimension':'group','filters':{'group':'汽车交通-国内'}},contract)
|
||||
def test_combined_coverage_allows_supplemental_stage_query_without_changing_contract(self):
|
||||
contract=self.contract()
|
||||
q=bind_query({'mode':'changes','dimension':'group','stage':'black'},contract)
|
||||
self.assertEqual(q['stage'],'black')
|
||||
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertEqual(contract['task']['dimensionStages']['group'],[])
|
||||
def test_understood_followup_executes_missing_group_read_even_when_native_made_no_calls(self):
|
||||
from unittest.mock import patch,MagicMock
|
||||
from test_agent_runtime import comparison
|
||||
from nao_agent_task import bind_task
|
||||
import nao_quality_runtime as runtime
|
||||
task=self.task();primary=comparison();release=primary['releaseId'];primary['filterBinding']=task['filters']
|
||||
docs=[{'evidenceRef':'QAAAAAAAA','mode':'compare','result':primary}];calls=[]
|
||||
def read(arguments):
|
||||
calls.append(arguments)
|
||||
result={'kind':'quality_changes','releaseId':release,'dimension':'group','stage':None,
|
||||
'filters':task['filters'],'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
||||
'totalRows':2,'returnedRows':2,'truncated':False,'sort':'scrap_delta',
|
||||
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
|
||||
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
|
||||
doc={'evidenceRef':'QBBBBBBBB','mode':'changes','result':result};docs.append(doc);return doc
|
||||
request={'actorId':'actor','sessionId':'session','releaseId':release,'question':task['question'],
|
||||
'context':bind_task({'releaseId':release},task)}
|
||||
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
|
||||
patch('nao_agent_task.plan_repairs',side_effect=AssertionError('Known missing query must execute directly')),\
|
||||
patch.object(runtime,'DelegatedApi'),patch('nao_quality_tools.QualityTools') as tools:
|
||||
tools.return_value.quality_data.side_effect=read
|
||||
runtime.complete_agent_evidence(task=task,request=request,actor_root=Path('/offline/session'),
|
||||
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
|
||||
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
|
||||
self.assertEqual(len(calls),1)
|
||||
self.assertEqual(calls[0]['dimension'],'group')
|
||||
self.assertEqual(calls[0]['filters']['department'],'新能源汽车磁钢')
|
||||
self.assertNotIn('stage',calls[0])
|
||||
self.assertTrue(finalizer.coverage_sufficient(docs,task['question'],request['context']))
|
||||
|
||||
class AnswerFocusTest(unittest.TestCase):
|
||||
def facts(self):
|
||||
return [{'id':'F1','kind':'overall','text':'总体合格率下降。','evidenceRef':'QAAAAAAAA'},
|
||||
{'id':'F2','kind':'group','dimension':'group','stage':None,'label':'国内','text':'国内:96.39%→95.76%,下降0.63个百分点。','evidenceRef':'QBBBBBBBB'},
|
||||
{'id':'F3','kind':'group','dimension':'group','stage':None,'label':'海外','text':'海外:93.70%→93.61%,下降0.09个百分点。','evidenceRef':'QBBBBBBBB'}]
|
||||
def task(self):return {'kind':'analysis','dimensions':['group'],'dimensionStages':{'group':[]},'question':'主要是国内还是国外的'}
|
||||
def test_group_followup_requires_both_objects_not_only_overall(self):
|
||||
focus=answer_focus(self.facts(),self.task())
|
||||
self.assertEqual(set(focus['requiredFactIds']),{'F2','F3'})
|
||||
self.assertTrue(missing_focus(['F1'],focus))
|
||||
self.assertFalse(missing_focus(['F2','F3'],focus))
|
||||
def test_department_fact_cannot_satisfy_group_followup(self):
|
||||
facts=self.facts();facts.insert(1,{'id':'D1','kind':'group','dimension':'department','stage':None,
|
||||
'label':'父产业部','text':'父产业部变化。','evidenceRef':'QAAAAAAAA'})
|
||||
self.assertEqual(set(answer_focus(facts,self.task())['requiredFactIds']),{'F2','F3'})
|
||||
def test_explicit_group_comparison_cannot_be_replaced_by_other_top_groups(self):
|
||||
facts=self.facts();facts[1]['label']='汽车交通-国内';facts[2]['label']='汽车交通-海外'
|
||||
facts[1]['text']='汽车交通-国内:下降0.62个百分点。';facts[2]['text']='汽车交通-海外:下降0.09个百分点。'
|
||||
facts[1:1]=[{'id':'X1','kind':'group','dimension':'group','stage':None,'label':'其他甲','text':'其他甲','evidenceRef':'QBBBBBBBB'},
|
||||
{'id':'X2','kind':'group','dimension':'group','stage':None,'label':'其他乙','text':'其他乙','evidenceRef':'QBBBBBBBB'}]
|
||||
task={**self.task(),'targetEntities':{'group':['汽车交通-国内','汽车交通-海外']}}
|
||||
focus=answer_focus(facts,task)
|
||||
self.assertEqual(set(focus['requiredFactIds']),{'F2','F3'})
|
||||
self.assertTrue(missing_focus(['X1','X2'],focus))
|
||||
self.assertFalse(answer_focus([f for f in facts if f['id']!='F3'],task)['available'])
|
||||
def test_failed_composition_fallback_answers_current_focus_without_big_parent_tables(self):
|
||||
report={'facts':self.facts(),'sections':['### 总体与阶段\n\n一张大表'],'policy':''}
|
||||
answer,ids=focus_fallback(report,self.task(),[])
|
||||
self.assertIn('国内',answer);self.assertIn('海外',answer)
|
||||
self.assertNotIn('总体与阶段',answer);self.assertNotIn('一张大表',answer)
|
||||
self.assertEqual(set(ids),{'F2','F3'})
|
||||
def test_missing_group_facts_explain_gap_instead_of_dumping_parent_report(self):
|
||||
report={'facts':self.facts()[:1],'sections':['一张大表'],'policy':''}
|
||||
answer,ids=focus_fallback(report,self.task(),['范围尚未一致'])
|
||||
self.assertIn('业务分组',answer);self.assertIn('范围尚未一致',answer)
|
||||
self.assertNotIn('一张大表',answer);self.assertEqual(ids,[])
|
||||
|
||||
def test_direct_priority_requires_both_verified_change_signals_to_agree(self):
|
||||
facts=self.facts();facts[1].update(rateDrop=.0062,scrapDelta=5572.52);facts[2].update(rateDrop=.0009,scrapDelta=738.91)
|
||||
focus=answer_focus(facts,self.task());answer=with_focus_conclusion('正式事实',focus)
|
||||
self.assertTrue(answer.startswith('按本轮可比分组的变化,优先关注国内'))
|
||||
self.assertIn('不代表对总体下降的精确贡献',answer)
|
||||
facts.append({'id':'F4','kind':'group','stage':None,'label':'新进分组','text':'本期新增,缺少比较期。','evidenceRef':'QBBBBBBBB','rateDrop':None,'scrapDelta':None})
|
||||
self.assertEqual(answer_focus(facts,self.task())['priority']['label'],'国内')
|
||||
facts[2]['scrapDelta']=6000
|
||||
self.assertIsNone(answer_focus(facts,self.task())['priority'])
|
||||
facts[2].pop('rateDrop')
|
||||
self.assertIsNone(answer_focus(facts,self.task())['priority'])
|
||||
|
||||
def test_real_synthesis_rejects_off_target_model_answer_and_falls_back_to_group_pair(self):
|
||||
from test_agent_runtime import comparison
|
||||
from nao_agent_task import bind_task
|
||||
release='33333333-3333-4333-8333-333333333333';period={'year':2026,'month':9};prior={'year':2026,'month':8}
|
||||
filters={'department':'新能源汽车磁钢'};primary=comparison();primary['filterBinding']=filters
|
||||
changes={'kind':'quality_changes','releaseId':release,'dimension':'group','stage':None,
|
||||
'filters':filters,'currentPeriod':period,'comparisonPeriod':prior,'sort':'scrap_delta',
|
||||
'totalRows':2,'returnedRows':2,'truncated':False,
|
||||
'rows':[{'key':'国内','label':'汽车交通-国内','comparison':{'rate':.9639},'current':{'rate':.9576},'rateDrop':.0063,'scrapDelta':5},
|
||||
{'key':'海外','label':'汽车交通-海外','comparison':{'rate':.9370},'current':{'rate':.9361},'rateDrop':.0009,'scrapDelta':1}]}
|
||||
docs=[{'evidenceRef':'QAAAAAAAA','mode':'compare','result':primary},
|
||||
{'evidenceRef':'QBBBBBBBB','mode':'changes','result':changes}]
|
||||
q='主要是国内还是国外的';task=normalize_task(task_value(dimensions=['group'],dimensionStages={'group':[]},
|
||||
filterUpdates=filters,period=period,comparisonPeriod=prior,comparisonRequired=True),q)
|
||||
task['rootGoal']='之前全面分析产品和缺陷';context=bind_task({'releaseId':release},task);calls=[]
|
||||
def opener(request,timeout):
|
||||
body=json.loads(request.data);payload=json.loads(body['messages'][1]['content']);calls.append(payload)
|
||||
overall=next(f['id'] for f in payload['facts'] if f['kind']=='overall')
|
||||
value={'reportTitle':'质量变化分析','answerBlocks':[{'type':'paragraph','factIds':[overall]}],
|
||||
'evidenceSufficient':True,'missingEvidence':[]}
|
||||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(value)}}]}).encode())
|
||||
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=q,evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
|
||||
self.assertEqual(len(calls),2)
|
||||
self.assertTrue(result['compositionRepairAttempted'])
|
||||
self.assertIn('汽车交通-国内',result['answer']);self.assertIn('汽车交通-海外',result['answer'])
|
||||
self.assertNotIn('总体与阶段',result['answer'])
|
||||
self.assertTrue(result['evidenceSufficient'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Formal stage scrap-rate evidence, coverage, report, and chart contracts."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
|
||||
from analysis_visuals import _comparison_charts, _trend_chart_from_points, build_visuals
|
||||
from nao_agent_report import _build_trend_report, build_noncomparison_report
|
||||
from nao_analysis_coverage import formal_metric_coverage, semantic_metric_gaps
|
||||
from nao_evidence_finalizer import _formal_metric_report
|
||||
from nao_evidence_report import build_report
|
||||
|
||||
|
||||
RELEASE = 'release-scrap-rate'
|
||||
QUESTION = '查询2026年9月黑片报废率'
|
||||
PERIOD = {'year': 2026, 'month': 9}
|
||||
COMPARISON_PERIOD = {'year': 2026, 'month': 8}
|
||||
|
||||
|
||||
def task(question=QUESTION, *, period=PERIOD, comparison_period=None, stages=None,
|
||||
fields=None, trend=False, visual='none'):
|
||||
return {
|
||||
'version': 1,
|
||||
'question': question,
|
||||
'kind': 'metrics',
|
||||
'filters': {'product': '产品甲'},
|
||||
'period': period,
|
||||
'comparisonPeriod': comparison_period,
|
||||
'stages': stages if stages is not None else ['black'],
|
||||
'fields': fields if fields is not None else ['scrapRate'],
|
||||
'dimensions': [],
|
||||
'trendRequired': trend,
|
||||
'comparisonRequired': comparison_period is not None,
|
||||
'freshQuery': True,
|
||||
'verifySource': False,
|
||||
'fullDetails': False,
|
||||
'needsClarification': False,
|
||||
'visualPreference': visual,
|
||||
'visualType': None,
|
||||
'responseStyle': 'standard',
|
||||
'periodCoverage': 'available',
|
||||
'historyMode': 'none',
|
||||
}
|
||||
|
||||
|
||||
def metric_doc(*, value=0.125, month=9, filters=None, stage='black', rate=None,
|
||||
evidence='A', rows=2):
|
||||
period = {'year': 2026, 'month': month}
|
||||
summary_stage = {'scrapRate': value, 'totalWeight': 100,
|
||||
'scrapWeight': value * 100 if isinstance(value, (int, float)) and value >= 0 else 0}
|
||||
if rate is not None:
|
||||
summary_stage['rate'] = rate
|
||||
return {
|
||||
'tool': 'quality_data', 'mode': 'query', 'releaseId': RELEASE,
|
||||
'evidenceRef': 'Q' + evidence * 8,
|
||||
'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
|
||||
'filters': filters or {'product': '产品甲'}, 'view': 'overall',
|
||||
'summary': {stage: summary_stage},
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
|
||||
'blackRows': rows if stage == 'black' else 0,
|
||||
'brightRows': rows if stage == 'bright' else 0}]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def comparison_doc(*, black_values=(0.05, 0.0), release=RELEASE, filters=None,
|
||||
black_rows=(2, 2), evidence='B'):
|
||||
def snapshot(month, value, rows):
|
||||
return {
|
||||
'period': {'year': 2026, 'month': month},
|
||||
'compositeRate': 0.94,
|
||||
'black': {'scrapRate': value, 'rate': 0.95, 'totalWeight': 100, 'scrapWeight': 5},
|
||||
'bright': {'scrapRate': 0.06, 'rate': 0.96, 'totalWeight': 90, 'scrapWeight': 4},
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
|
||||
'blackRows': rows, 'brightRows': 2}]},
|
||||
}
|
||||
result = {
|
||||
'kind': 'quality_comparison', 'releaseId': release,
|
||||
'filterBinding': filters or {'product': '产品甲'},
|
||||
'comparison': snapshot(8, black_values[0], black_rows[0]),
|
||||
'current': snapshot(9, black_values[1], black_rows[1]),
|
||||
'delta': {'available': True, 'percentagePoints': 0},
|
||||
'stageRateChangesPP': {'black': 0, 'bright': 0},
|
||||
}
|
||||
return {'tool': 'quality_data', 'mode': 'compare', 'releaseId': release,
|
||||
'evidenceRef': 'Q' + evidence * 8, 'result': result}
|
||||
|
||||
|
||||
class ScrapRateEvidenceTest(unittest.TestCase):
|
||||
def context(self, selected_task):
|
||||
return {'releaseId': RELEASE, 'period': selected_task['period'],
|
||||
'comparisonPeriod': selected_task['comparisonPeriod'],
|
||||
'filters': selected_task['filters'], '_agentTask': selected_task}
|
||||
|
||||
def test_single_period_uses_scrap_rate_directly_and_keeps_zero_and_over_one(self):
|
||||
for raw, expected in ((0, '0.00%'), (1.25, '125.00%')):
|
||||
selected = task()
|
||||
report = build_noncomparison_report(
|
||||
[metric_doc(value=raw, rate=None)], self.context(selected), QUESTION)
|
||||
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率'])
|
||||
self.assertIn(expected, report['facts'][0]['text'])
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
|
||||
def test_requested_numerator_and_denominator_are_displayed_as_independent_formal_values(self):
|
||||
question='黑片报废率是多少,同时列出分子分母'
|
||||
selected=task(question,fields=['scrapRate','scrapWeight','totalWeight'])
|
||||
doc=metric_doc(value=.175,rate=.99)
|
||||
doc['result']['summary']['black'].update(totalWeight=200,scrapWeight=35)
|
||||
report=build_noncomparison_report([doc],self.context(selected),question)
|
||||
rendered='\n'.join(report['sections'])
|
||||
for expected in ('17.50%','35.00','200.00'):
|
||||
self.assertIn(expected,rendered)
|
||||
self.assertEqual([f['label'] for f in report['facts']],['黑片报废率','黑片报废重量','黑片投入重量'])
|
||||
self.assertTrue(report['coverageSufficient'])
|
||||
self.assertNotIn('1.00%',rendered)
|
||||
|
||||
def test_missing_requested_denominator_is_incomplete_even_when_formal_rate_exists(self):
|
||||
question='黑片报废率及报废总量和修正总重'
|
||||
selected=task(question,fields=['scrapRate','scrapWeight','totalWeight'])
|
||||
doc=metric_doc();doc['result']['summary']['black'].pop('totalWeight')
|
||||
report=build_noncomparison_report([doc],self.context(selected),question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(any('重量' in gap for gap in report['missingEvidence']))
|
||||
|
||||
def test_missing_or_negative_scrap_rate_is_not_treated_as_zero_or_clamped(self):
|
||||
for raw in (None, -0.01):
|
||||
selected = task()
|
||||
report = build_noncomparison_report(
|
||||
[metric_doc(value=raw, rate=0.99)], self.context(selected), QUESTION)
|
||||
self.assertIn('未提供', report['facts'][0]['text'])
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertEqual(semantic_metric_gaps(
|
||||
[metric_doc(value=None)], QUESTION, self.context(task())),
|
||||
['未取得指定期间、范围的黑片报废率正式指标或来源覆盖。'])
|
||||
|
||||
def test_scrap_rate_coverage_does_not_require_qualified_rate_or_other_stage(self):
|
||||
selected = task()
|
||||
doc = metric_doc(value=0.02, rate=None)
|
||||
self.assertEqual(semantic_metric_gaps([doc], QUESTION, self.context(selected)), [])
|
||||
report = build_noncomparison_report([doc], self.context(selected), QUESTION)
|
||||
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率'])
|
||||
self.assertNotIn('亮片', '\n'.join(fact['text'] for fact in report['facts']))
|
||||
self.assertNotIn('合格率', '\n'.join(fact['text'] for fact in report['facts']))
|
||||
|
||||
def test_scope_and_period_mismatch_are_rejected(self):
|
||||
selected = task()
|
||||
wrong_scope = metric_doc(value=0.02, filters={'product': '产品乙'})
|
||||
wrong_period = metric_doc(value=0.02, month=8)
|
||||
self.assertTrue(semantic_metric_gaps([wrong_scope], QUESTION, self.context(selected)))
|
||||
self.assertTrue(semantic_metric_gaps([wrong_period], QUESTION, self.context(selected)))
|
||||
|
||||
def test_two_period_formal_contract_and_body_use_only_scrap_rate(self):
|
||||
question = '只看产品甲,2026年9月与8月黑片报废率'
|
||||
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD)
|
||||
context = self.context(selected)
|
||||
doc = comparison_doc(black_values=(1.25, 0.0))
|
||||
contract = formal_metric_coverage([doc], question, context)
|
||||
self.assertEqual(contract['fields'], ['scrapRate'])
|
||||
self.assertEqual(contract['missingEvidence'], [])
|
||||
self.assertEqual([item['value'] for item in contract['metrics']], [0.0, 1.25])
|
||||
report = _formal_metric_report(contract)
|
||||
text = '\n'.join(report['sections']) + '\n' + '\n'.join(item['text'] for item in report['facts'])
|
||||
self.assertIn('125.00%', text)
|
||||
self.assertIn('0.00%', text)
|
||||
self.assertNotIn('合格率', text)
|
||||
|
||||
wrong_scope = comparison_doc(filters={'product': '产品乙'})
|
||||
rejected = formal_metric_coverage([wrong_scope], question, context)
|
||||
self.assertTrue(rejected['missingEvidence'])
|
||||
|
||||
comparison_question = '对比2026年9月与8月黑片报废率'
|
||||
selected = task(comparison_question, period=PERIOD,
|
||||
comparison_period=COMPARISON_PERIOD, stages=['black'])
|
||||
body = build_report([doc], self.context(selected), comparison_question)
|
||||
body_text = '\n'.join(body['sections']) + '\n' + '\n'.join(fact['text'] for fact in body['facts'])
|
||||
self.assertTrue(body['coverageSufficient'], body['missingEvidence'])
|
||||
self.assertIn('125.00%', body_text)
|
||||
self.assertIn('0.00%', body_text)
|
||||
self.assertIn('黑片报废率', body_text)
|
||||
self.assertNotIn('亮片报废率', body_text)
|
||||
|
||||
def test_formal_scrap_weight_contract_rejects_negative_backend_weight(self):
|
||||
question = '只看产品甲,2026年9月与8月黑片报废重量'
|
||||
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
|
||||
fields=['scrapWeight'])
|
||||
document = comparison_doc()
|
||||
document['result']['current']['black']['scrapWeight'] = -0.01
|
||||
contract = formal_metric_coverage([document], question, self.context(selected))
|
||||
self.assertTrue(contract['missingEvidence'])
|
||||
invalid = [metric for metric in contract['metrics']
|
||||
if metric['field'] == 'scrapWeight' and metric['value'] < 0]
|
||||
self.assertEqual(len(invalid), 1)
|
||||
self.assertFalse(invalid[0]['covered'])
|
||||
|
||||
def test_omitted_stage_scrap_rate_stays_two_stage_not_a_composite(self):
|
||||
question = '查询2026年9月产品甲报废率'
|
||||
selected = task(question, stages=[])
|
||||
context = self.context(selected)
|
||||
doc = metric_doc(value=0.04)
|
||||
doc['result']['summary']['bright'] = {'scrapRate': 0.03, 'totalWeight': 90, 'scrapWeight': 2.7}
|
||||
doc['result']['coverage']['sourceMonths'][0]['brightRows'] = 2
|
||||
self.assertEqual(semantic_metric_gaps([doc], question, context), [])
|
||||
report = build_noncomparison_report([doc], context, question)
|
||||
self.assertEqual({fact['label'] for fact in report['facts']}, {'黑片报废率', '亮片报废率'})
|
||||
self.assertFalse(any('综合报废率' in fact['text'] for fact in report['facts']))
|
||||
|
||||
def test_monthly_trend_uses_server_values_and_preserves_same_value(self):
|
||||
question = '查询2026年黑片报废率趋势'
|
||||
selected = task(question, period={'year': 2026}, trend=True, visual='required')
|
||||
trend = []
|
||||
source_months = []
|
||||
for month in (1, 2, 3):
|
||||
trend.append({'period': f'2026-{month:02d}',
|
||||
'black': {'scrapRate': 0.08, 'totalWeight': 100, 'scrapWeight': 8},
|
||||
'coverage': {'blackRows': 2, 'brightRows': 0}})
|
||||
source_months.append({'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 0})
|
||||
doc = metric_doc(value=0.08)
|
||||
doc['result'].update(period={'year': 2026}, trend=trend, trendCount=3,
|
||||
trendTruncated=False,
|
||||
coverage={'sourceMonths': source_months})
|
||||
report = _build_trend_report([doc], self.context(selected), question)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率月度趋势'])
|
||||
self.assertIn('1月8.00%', report['facts'][0]['text'])
|
||||
self.assertIn('3月8.00%', report['facts'][0]['text'])
|
||||
self.assertIn('持平', report['facts'][0]['text'])
|
||||
self.assertNotIn('合格率', report['facts'][0]['text'])
|
||||
|
||||
def test_trend_visual_uses_scrap_rate_contract_and_has_no_good_rate_series(self):
|
||||
points = [
|
||||
{'period': f'2026-0{month}', 'snapshot': {
|
||||
'black': {'scrapRate': value, 'rate': 0.95,
|
||||
'totalWeight': 100, 'scrapWeight': value * 100},
|
||||
'coverage': {'blackRows': 2, 'brightRows': 0}}}
|
||||
for month, value in ((1, 0.0), (2, 1.25), (3, 0.0))
|
||||
]
|
||||
charts = _trend_chart_from_points(
|
||||
points, release_id=RELEASE, scope={'product': '产品甲'}, turn_id='turn-scrap',
|
||||
evidence_refs=['QAAAAAAAA'], question='黑片报废率趋势', supplied_trend=True,
|
||||
coverage_truncated=False, semantic_fields={'scrapRate'}, semantic_stages=['black'])
|
||||
self.assertEqual([chart['metric'] for chart in charts], ['black_scrap_rate'])
|
||||
self.assertEqual(charts[0]['unit'], 'ratio')
|
||||
self.assertEqual(charts[0]['requiredFields'], ['black_scrap_weight', 'black_total_weight'])
|
||||
self.assertEqual(charts[0]['series'][0]['values'], [0.0, 1.25, 0.0])
|
||||
|
||||
def test_comparison_chart_is_stage_specific_and_uses_scrap_authorization(self):
|
||||
document = comparison_doc(black_values=(1.25, 0.0))
|
||||
charts = _comparison_charts(
|
||||
document, document['result'], 'turn-compare', '', requested_stage='black',
|
||||
semantic_fields={'scrapRate'})
|
||||
self.assertEqual([chart['metric'] for chart in charts], ['black_scrap_rate'])
|
||||
self.assertEqual(charts[0]['unit'], 'ratio')
|
||||
self.assertEqual(charts[0]['requiredFields'], ['black_scrap_weight', 'black_total_weight'])
|
||||
self.assertEqual(charts[0]['series'][0]['values'], [1.25, 0.0])
|
||||
self.assertEqual(charts[0]['type'], 'bar')
|
||||
|
||||
def test_comparison_chart_rejects_missing_coverage_and_semantic_output_has_no_extra_rates(self):
|
||||
document = comparison_doc(black_rows=(0, 2))
|
||||
charts = _comparison_charts(document, document['result'], 'turn-compare', '',
|
||||
requested_stage='black', semantic_fields={'scrapRate'})
|
||||
self.assertEqual(charts, [])
|
||||
|
||||
question = '对比2026年9月与8月黑片报废率'
|
||||
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
|
||||
visual='required')
|
||||
context = self.context(selected)
|
||||
visuals = build_visuals([comparison_doc()], question, 'turn-semantic', context=context)['visuals']
|
||||
self.assertEqual([chart['metric'] for chart in visuals], ['black_scrap_rate'])
|
||||
|
||||
both_question = '对比2026年9月与8月黑片和亮片报废率'
|
||||
both_task = task(both_question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
|
||||
stages=['black', 'bright'], visual='required')
|
||||
both_visuals = build_visuals([comparison_doc()], both_question, 'turn-both',
|
||||
context=self.context(both_task))['visuals']
|
||||
self.assertEqual({chart['metric'] for chart in both_visuals},
|
||||
{'black_scrap_rate', 'bright_scrap_rate'})
|
||||
|
||||
plain_question = '比较黑片报废率'
|
||||
plain_visuals = build_visuals([comparison_doc()], plain_question, 'turn-plain')['visuals']
|
||||
self.assertEqual([chart['metric'] for chart in plain_visuals], ['black_scrap_rate'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Single-stage source references for official composite rates."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
|
||||
from nao_agent_report import build_noncomparison_report, _build_trend_report
|
||||
from nao_analysis_coverage import semantic_metric_gaps
|
||||
from nao_evidence_report import _display_rate, build_report
|
||||
from nao_quality_tools import QualityTools, _valid_composite_rate_reference
|
||||
|
||||
|
||||
RELEASE = 'release-stage-reference'
|
||||
|
||||
|
||||
def side(month, valid_stage='black', composite=.91, *, unknown=False,
|
||||
black_total=100, bright_total=100, source_rows=True):
|
||||
"""A backend snapshot where the missing stage may carry the default rate 1."""
|
||||
black_valid = valid_stage == 'black'
|
||||
bright_valid = valid_stage == 'bright'
|
||||
if valid_stage == 'none':
|
||||
black_valid = bright_valid = False
|
||||
black_rows = (2 if black_valid else 0) if source_rows else 0
|
||||
bright_rows = (2 if bright_valid else 0) if source_rows else 0
|
||||
coverage = {} if unknown else {'sourceMonths': [
|
||||
{'year': 2026, 'month': month, 'blackRows': black_rows, 'brightRows': bright_rows}]}
|
||||
return {
|
||||
'period': {'year': 2026, 'month': month},
|
||||
'compositeRate': composite,
|
||||
'black': {'rate': .97 if black_valid else 1, 'qualifiedWeight': 97 if black_valid else 0,
|
||||
'totalWeight': black_total if black_valid else 0, 'scrapWeight': 3 if black_valid else 0},
|
||||
'bright': {'rate': .98 if bright_valid else 1, 'qualifiedWeight': 98 if bright_valid else 0,
|
||||
'totalWeight': bright_total if bright_valid else 0, 'scrapWeight': 2 if bright_valid else 0},
|
||||
'coverage': coverage,
|
||||
}
|
||||
|
||||
|
||||
def metric_doc(month=9, valid_stage='black', composite=.91, **kwargs):
|
||||
snapshot = side(month, valid_stage, composite, **kwargs)
|
||||
return {'evidenceRef': 'QSTAGE001', 'mode': 'query', 'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': snapshot['period'],
|
||||
'filters': {}, 'view': 'overall', 'summary': {k: v for k, v in snapshot.items()
|
||||
if k not in ('period', 'coverage')},
|
||||
'coverage': snapshot['coverage'],
|
||||
}}
|
||||
|
||||
|
||||
def comparison_doc(current_stage='black', previous_stage='black', *, current_composite=.91,
|
||||
previous_composite=.90, current_kwargs=None, previous_kwargs=None):
|
||||
current = side(9, current_stage, current_composite, **(current_kwargs or {}))
|
||||
previous = side(8, previous_stage, previous_composite, **(previous_kwargs or {}))
|
||||
return {'evidenceRef': 'QSTAGE002', 'mode': 'compare', 'result': {
|
||||
'kind': 'quality_comparison', 'releaseId': RELEASE, 'filterBinding': {},
|
||||
'current': current, 'comparison': previous,
|
||||
'delta': {'available': True, 'percentagePoints': (current_composite - previous_composite) * 100},
|
||||
'stageRateChangesPP': {'black': None, 'bright': None},
|
||||
}}
|
||||
|
||||
|
||||
def metrics_context(question='综合合格率是多少', *, period=None, stages=None, fields=None):
|
||||
period = period or {'year': 2026, 'month': 9}
|
||||
task = {'version': 1, 'question': question, 'kind': 'metrics', 'historyMode': 'none',
|
||||
'filters': {}, 'period': period, 'comparisonPeriod': None,
|
||||
'dimensions': [], 'stages': stages or [], 'fields': fields or ['rate'],
|
||||
'verifySource': False, 'fullDetails': False, 'freshQuery': False,
|
||||
'needsClarification': False, 'comparisonRequired': False, 'trendRequired': False,
|
||||
'visualPreference': 'none', 'periodCoverage': 'complete'}
|
||||
return {'releaseId': RELEASE, 'period': period, 'filters': {}, '_agentTask': task}
|
||||
|
||||
|
||||
def range_snapshot(source_by_month):
|
||||
month_rows = []
|
||||
for month, stages in source_by_month.items():
|
||||
month_rows.append({'year': 2026, 'month': month,
|
||||
'blackRows': stages.get('black', 0),
|
||||
'brightRows': stages.get('bright', 0)})
|
||||
black_present = any(row['blackRows'] > 0 for row in month_rows)
|
||||
bright_present = any(row['brightRows'] > 0 for row in month_rows)
|
||||
return {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 3},
|
||||
'compositeRate': .89,
|
||||
'black': {'rate': .97 if black_present else 1,
|
||||
'qualifiedWeight': 97 if black_present else 0,
|
||||
'totalWeight': 100 if black_present else 0},
|
||||
'bright': {'rate': .98 if bright_present else 1,
|
||||
'qualifiedWeight': 98 if bright_present else 0,
|
||||
'totalWeight': 100 if bright_present else 0},
|
||||
'coverage': {'sourceMonths': month_rows}}
|
||||
|
||||
|
||||
class SingleStageReferenceTest(unittest.TestCase):
|
||||
def test_composite_accepts_either_stage_and_preserves_server_zero(self):
|
||||
for stage in ('black', 'bright'):
|
||||
snapshot = side(9, stage, composite=0)
|
||||
self.assertEqual(_valid_composite_rate_reference(snapshot, snapshot['period']), (True, None))
|
||||
self.assertEqual(_display_rate(snapshot), 0)
|
||||
|
||||
def test_comparison_delta_accepts_both_single_stage_directions(self):
|
||||
for current_stage, previous_stage in (('black', 'bright'), ('bright', 'black')):
|
||||
current_side = side(9, current_stage, .93)
|
||||
previous_side = side(8, previous_stage, .90)
|
||||
current = {'summary': {key: value for key, value in current_side.items()
|
||||
if key not in ('period', 'coverage')},
|
||||
'coverage': current_side['coverage']}
|
||||
previous = {'summary': {key: value for key, value in previous_side.items()
|
||||
if key not in ('period', 'coverage')},
|
||||
'coverage': previous_side['coverage']}
|
||||
current['_period'] = {'year': 2026, 'month': 9}
|
||||
previous['_period'] = {'year': 2026, 'month': 8}
|
||||
delta = QualityTools._metric_difference(current, previous)
|
||||
self.assertTrue(delta['available'], delta)
|
||||
self.assertAlmostEqual(float(delta['percentagePoints']), 3.0)
|
||||
|
||||
def test_two_missing_unknown_and_zero_denominator_placeholder_are_rejected(self):
|
||||
both_missing = side(9, 'none', composite=1)
|
||||
self.assertFalse(_valid_composite_rate_reference(both_missing, both_missing['period'])[0])
|
||||
|
||||
unknown = side(9, 'black', unknown=True)
|
||||
self.assertFalse(_valid_composite_rate_reference(unknown, unknown['period'])[0])
|
||||
|
||||
zero_denominators = side(9, 'none', composite=1)
|
||||
zero_denominators['coverage']['sourceMonths'] = [
|
||||
{'year': 2026, 'month': 9, 'blackRows': 2, 'brightRows': 2}]
|
||||
self.assertFalse(_valid_composite_rate_reference(zero_denominators,
|
||||
zero_denominators['period'])[0])
|
||||
|
||||
no_official_value = side(9, 'black')
|
||||
no_official_value['compositeRate'] = None
|
||||
self.assertFalse(_valid_composite_rate_reference(no_official_value,
|
||||
no_official_value['period'])[0])
|
||||
|
||||
def test_mixed_stage_month_range_requires_known_source_each_month(self):
|
||||
mixed = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {'black': 1, 'bright': 4}})
|
||||
self.assertTrue(_valid_composite_rate_reference(mixed, mixed['period'])[0])
|
||||
|
||||
missing_month = range_snapshot({1: {'black': 2}, 3: {'bright': 4}})
|
||||
self.assertFalse(_valid_composite_rate_reference(missing_month, missing_month['period'])[0])
|
||||
|
||||
unknown_count = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {'black': 1}})
|
||||
unknown_count['coverage']['sourceMonths'][1]['blackRows'] = None
|
||||
self.assertFalse(_valid_composite_rate_reference(unknown_count, unknown_count['period'])[0])
|
||||
|
||||
empty_month = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {}})
|
||||
self.assertFalse(_valid_composite_rate_reference(empty_month, empty_month['period'])[0])
|
||||
|
||||
question = '综合合格率是多少'
|
||||
doc = {'evidenceRef': 'QSTAGE004', 'mode': 'query', 'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': mixed['period'],
|
||||
'filters': {}, 'view': 'overall',
|
||||
'summary': {key: value for key, value in mixed.items() if key not in ('period', 'coverage')},
|
||||
'coverage': mixed['coverage']}}
|
||||
self.assertEqual(semantic_metric_gaps([doc], question,
|
||||
metrics_context(question, period=mixed['period'])), [])
|
||||
|
||||
def test_legacy_positive_source_without_denominator_fields_remains_supported(self):
|
||||
snapshot = side(9, 'black')
|
||||
for stage in ('black', 'bright'):
|
||||
snapshot[stage].pop('totalWeight')
|
||||
self.assertTrue(_valid_composite_rate_reference(snapshot, snapshot['period'])[0])
|
||||
|
||||
def test_single_period_report_explains_missing_stage_without_claiming_100_percent(self):
|
||||
question = '综合合格率是多少'
|
||||
report = build_noncomparison_report([metric_doc(9, 'black')],
|
||||
metrics_context(question), question)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
text = report['facts'][0]['text']
|
||||
self.assertIn('综合合格率:91.00%', text)
|
||||
self.assertIn('缺少亮片可用独立阶段指标', text)
|
||||
self.assertIn('缺失阶段不作为实测100%', text)
|
||||
self.assertNotIn('亮片合格率:100.00%', text)
|
||||
self.assertEqual(semantic_metric_gaps([metric_doc(9, 'black')], question,
|
||||
metrics_context(question)), [])
|
||||
|
||||
def test_explicit_stage_request_still_rejects_missing_stage_in_both_directions(self):
|
||||
for selected, available in (('black', 'bright'), ('bright', 'black')):
|
||||
label = '黑片' if selected == 'black' else '亮片'
|
||||
question = f'只看{label}合格率'
|
||||
context = metrics_context(question, stages=[selected])
|
||||
report = build_noncomparison_report([metric_doc(9, available)], context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn(f'{label}合格率:未提供', report['facts'][0]['text'])
|
||||
self.assertNotIn('100.00%', report['facts'][0]['text'])
|
||||
|
||||
def test_missing_stage_weight_and_scrap_rate_remain_uncovered(self):
|
||||
for selected, label, field, question in (
|
||||
('black', '黑片', 'totalWeight', '只看黑片投入重量'),
|
||||
('bright', '亮片', 'scrapRate', '只看亮片报废率')):
|
||||
context = metrics_context(question, stages=[selected], fields=[field])
|
||||
report = build_noncomparison_report([metric_doc(9, 'bright' if selected == 'black' else 'black')],
|
||||
context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertIn(f'{label}', report['facts'][0]['text'])
|
||||
self.assertIn('未提供', report['facts'][0]['text'])
|
||||
|
||||
def test_scrap_rate_requires_positive_stage_total_and_keeps_true_zero(self):
|
||||
question = '只看黑片报废率'
|
||||
context = metrics_context(question, stages=['black'], fields=['scrapRate'])
|
||||
valid = metric_doc(9, 'black')
|
||||
valid['result']['summary']['black'].update(scrapRate=0, scrapWeight=0)
|
||||
report = build_noncomparison_report([valid], context, question)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('黑片报废率:0.00%', report['facts'][0]['text'])
|
||||
self.assertEqual(semantic_metric_gaps([valid], question, context), [])
|
||||
|
||||
zero_denominator = metric_doc(9, 'black')
|
||||
zero_denominator['result']['summary']['black'].update(
|
||||
totalWeight=0, scrapRate=0, scrapWeight=0)
|
||||
report = build_noncomparison_report([zero_denominator], context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(semantic_metric_gaps([zero_denominator], question, context))
|
||||
|
||||
negative_weight = metric_doc(9, 'black')
|
||||
negative_weight['result']['summary']['black'].update(
|
||||
scrapRate=0, scrapWeight=-0.1)
|
||||
report = build_noncomparison_report([negative_weight], context, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(semantic_metric_gaps([negative_weight], question, context))
|
||||
|
||||
def test_comparison_report_handles_both_directions_and_discloses_missing_stage(self):
|
||||
for current_stage, previous_stage, missing_label in (
|
||||
('black', 'bright', '亮片'), ('bright', 'black', '黑片')):
|
||||
doc = comparison_doc(current_stage, previous_stage)
|
||||
report = build_report([doc])
|
||||
text = report['headline'] + '\n'.join(report['sections']) + report['policy']
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('综合合格率:2026年8月90.00%→2026年9月91.00%', text)
|
||||
self.assertIn(missing_label + '来源月份', text)
|
||||
self.assertIn('解释为实测100%', text)
|
||||
self.assertNotIn('| 黑片合格率 | 100.00%', text)
|
||||
self.assertNotIn('| 亮片合格率 | 100.00%', text)
|
||||
self.assertIn(f'| {missing_label}投入重量 | 100.00 | — |', text)
|
||||
|
||||
def test_stage_missing_composite_does_not_block_a_valid_explicit_stage(self):
|
||||
question = '只看黑片合格率'
|
||||
context = metrics_context(question, stages=['black'])
|
||||
doc = metric_doc(9, 'black')
|
||||
doc['result']['summary']['compositeRate'] = None
|
||||
report = build_noncomparison_report([doc], context, question)
|
||||
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
|
||||
self.assertIn('黑片合格率:97.00%', report['facts'][0]['text'])
|
||||
|
||||
def test_trend_excludes_default_one_when_both_explicit_denominators_are_zero(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 3}
|
||||
question = '综合合格率月度趋势'
|
||||
task = {'version': 1, 'question': question, 'kind': 'metrics', 'historyMode': 'none',
|
||||
'filters': {}, 'period': period, 'comparisonPeriod': None, 'dimensions': [],
|
||||
'stages': [], 'fields': ['rate'], 'verifySource': False, 'fullDetails': False,
|
||||
'freshQuery': False, 'needsClarification': False, 'comparisonRequired': False,
|
||||
'trendRequired': True, 'visualPreference': 'none', 'periodCoverage': 'complete'}
|
||||
trend = []
|
||||
source_months = []
|
||||
for month in (1, 2, 3):
|
||||
missing = month == 3
|
||||
row = side(month, 'none' if missing else 'black', 1 if missing else .91)
|
||||
row['period'] = f'2026-{month:02d}'
|
||||
row['monthStatus'] = 'published'
|
||||
row['noData'] = False
|
||||
row['coverage'] = {'blackRows': 2, 'brightRows': 0}
|
||||
row['black'] = {**row['black'], 'coverage': {'blackRows': 0 if missing else 2}}
|
||||
row['bright'] = {**row['bright'], 'coverage': {'brightRows': 0}}
|
||||
trend.append(row)
|
||||
source_months.append({'year': 2026, 'month': month,
|
||||
'blackRows': 2 if not missing else 2,
|
||||
'brightRows': 2 if missing else 0})
|
||||
document = {'evidenceRef': 'QSTAGE003', 'mode': 'query', 'result': {
|
||||
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
|
||||
'filters': {}, 'view': 'overall', 'trend': trend, 'trendCount': 3,
|
||||
'trendTruncated': False, 'coverage': {'sourceMonths': source_months}}}
|
||||
report = _build_trend_report([document], {'releaseId': RELEASE, 'period': period,
|
||||
'filters': {}, '_agentTask': task}, question)
|
||||
self.assertFalse(report['coverageSufficient'])
|
||||
self.assertTrue(any('全部月份' in item for item in report['missingEvidence']))
|
||||
self.assertIn('3月未提供', report['facts'][0]['text'])
|
||||
self.assertNotIn('3月100.00%', report['facts'][0]['text'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,501 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
sys.path.insert(0, str(ROOT / 'verification'))
|
||||
|
||||
from nao_agent_task import TaskUnavailable, bind_task, normalize_task
|
||||
from nao_agent_task import plan_task
|
||||
from nao_protocol_execution import execute
|
||||
from nao_quality_tools import QualityToolError, QualityTools
|
||||
from nao_query_contract import bind_query
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_quality_tools import FakeApi, RELEASE
|
||||
|
||||
|
||||
def source_field(key, label, black, bright, stages=('black', 'bright')):
|
||||
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
|
||||
'stages': list(stages), 'type': 'source_value',
|
||||
'operators': ['in', 'not_in', 'is_empty', 'not_empty']}
|
||||
|
||||
|
||||
SOURCE_FIELDS = [
|
||||
source_field('product', '产品', '黑片产品', '亮片产品'),
|
||||
source_field('department', '产业部', '黑片产业部', '亮片产业部'),
|
||||
source_field('group', '业务分组', '黑片分组', '亮片分组'),
|
||||
source_field('projectGroup', '项目分组', '黑片项目分组', '亮片项目分组'),
|
||||
source_field('src_hash', '来源哈希', '黑片行哈希', '亮片行哈希'),
|
||||
source_field('batchNumber', '批号 | 代码', '黑片批次号', '亮片批号'),
|
||||
source_field('customerSpec', '客户规格', '客户规格编码', '检包规格'),
|
||||
source_field('dispatchOrder', '委托单', '黑片委托单号', '亮片工单委托'),
|
||||
source_field('blackOnlyCode', '黑片专用代码', '黑片专用列', '', ('black',)),
|
||||
]
|
||||
CATALOG = {'sourceFields': SOURCE_FIELDS, 'departments': [], 'groups': []}
|
||||
AUTHORITY = {'attemptId': 'attempt-1', 'sessionId': 'session-1', 'turnId': 'turn-1',
|
||||
'revision': 4, 'executionId': 'execution-1'}
|
||||
RULES = [
|
||||
{'field': 'batchNumber', 'stages': ['black'], 'operator': 'in', 'values': ['0012', 'A|B']},
|
||||
{'field': 'customerSpec', 'stages': ['bright'], 'operator': 'not_in', 'values': ['xY']},
|
||||
{'field': 'dispatchOrder', 'stages': ['black', 'bright'], 'operator': 'not_empty'},
|
||||
]
|
||||
|
||||
|
||||
def protocol_task(**changes):
|
||||
value = {**task_value(kind='metrics', dimensions=[], stages=['black', 'bright'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False,
|
||||
trendRequired=False, columnFilters=RULES), **changes}
|
||||
return normalize_task(value, '查询本期正式指标', {'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
|
||||
|
||||
class EchoColumnApi(FakeApi):
|
||||
def __init__(self, *, echo='flat'):
|
||||
super().__init__()
|
||||
self.echo = echo
|
||||
|
||||
def call(self, path, payload=None, method=None):
|
||||
result = super().call(path, payload, method)
|
||||
parsed = urlsplit(path)
|
||||
if parsed.path == '/api/analysis/tools/catalog':
|
||||
result['sourceFields'] = SOURCE_FIELDS
|
||||
if parsed.path == '/api/metrics' and 'columnFilters' in parse_qs(parsed.query):
|
||||
requested = json.loads(parse_qs(parsed.query)['columnFilters'][0])
|
||||
if self.echo == 'request':
|
||||
result['context']['filters'] = {'columnFilters': requested}
|
||||
elif self.echo == 'flat':
|
||||
result['context']['columnFilters'] = requested
|
||||
elif self.echo == 'mismatch':
|
||||
result['context']['filters'] = {'columnFilters': []}
|
||||
return result
|
||||
|
||||
|
||||
class SourceColumnProtocolTests(unittest.TestCase):
|
||||
def test_final_answer_selector_preserves_the_confirmed_column_scope(self):
|
||||
from nao_evidence_finalizer import _agent_task_summary
|
||||
task=protocol_task()
|
||||
summary=_agent_task_summary(bind_task({'releaseId':RELEASE},task))
|
||||
self.assertEqual(summary['filters']['columnFilters'],RULES)
|
||||
self.assertEqual(summary['sourceFieldLabels'],task['sourceFieldLabels'])
|
||||
def test_member_intrinsic_defaults_do_not_trigger_expensive_replanning(self):
|
||||
value=task_value(kind='members',dimensions=[],fields=[],stages=['black'],memberFields=['dispatchOrder'],columnFilters=[],trendRequired=False)
|
||||
for key in ('freshQuery','verifySource','fullDetails','responseStyle','historyMode','periodCoverage'):
|
||||
value.pop(key,None)
|
||||
task=normalize_task(value,'列出本期黑片委托单',{'_protocol':{'version':1},'_businessCatalog':CATALOG})
|
||||
self.assertTrue(task['freshQuery']);self.assertFalse(task['verifySource']);self.assertFalse(task['fullDetails'])
|
||||
self.assertEqual(task['responseStyle'],'standard')
|
||||
|
||||
def test_members_allow_authorized_canonical_dimension_projection_only(self):
|
||||
canonical_product = {**SOURCE_FIELDS[0], 'type': 'dimension'}
|
||||
catalog = {'sourceFields': [canonical_product, *SOURCE_FIELDS[1:]], 'departments': [], 'groups': []}
|
||||
task = normalize_task(task_value(kind='members', dimensions=[], fields=[], stages=['black', 'bright'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
|
||||
memberFields=['product'], columnFilters=[]), '列出客户规格清单',
|
||||
{'_protocol': {'version': 1}, '_businessCatalog': catalog})
|
||||
self.assertEqual(task['memberFields'], ['product'])
|
||||
self.assertEqual(task['sourceFieldLabels'], {'product': '产品'})
|
||||
|
||||
# The same canonical directory entry is a member projection, not an
|
||||
# authorized source_value filter rule.
|
||||
with self.assertRaisesRegex(TaskUnavailable, '不是受支持的原表文本字段'):
|
||||
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
|
||||
columnFilters=[{'field': 'product', 'stages': ['black'], 'operator': 'in', 'values': ['18X15X4']}]),
|
||||
'按产品筛选指标', {'_protocol': {'version': 1}, '_businessCatalog': catalog})
|
||||
|
||||
def test_column_filter_value_length_matches_backend_256_character_limit(self):
|
||||
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'in', 'values': ['X' * 256]}
|
||||
task = protocol_task(columnFilters=[rule])
|
||||
self.assertEqual(task['filters']['columnFilters'][0]['values'], ['X' * 256])
|
||||
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
|
||||
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
|
||||
|
||||
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
|
||||
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False,
|
||||
trendRequired=False, columnFilters=[])
|
||||
base.pop('responseStyle')
|
||||
base['visualPreference'] = None
|
||||
base['historyMode'] = None
|
||||
base['periodCoverage'] = None
|
||||
task = normalize_task(base, '查询本期正式指标', {
|
||||
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
self.assertEqual(task['responseStyle'], 'standard')
|
||||
self.assertEqual(task['visualPreference'], 'auto')
|
||||
self.assertEqual(task['historyMode'], 'none')
|
||||
self.assertEqual(task['periodCoverage'], 'complete')
|
||||
|
||||
trend = normalize_task({**base, 'trendRequired': True}, '查询本期正式指标', {
|
||||
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
self.assertEqual(trend['periodCoverage'], 'available')
|
||||
for invalid in ('', 'verbose', False):
|
||||
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
|
||||
normalize_task({**base, 'responseStyle': invalid}, '查询本期正式指标', {
|
||||
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task({**base, 'historyMode': ''}, '查询本期正式指标', {
|
||||
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
normalize_task({**base, 'periodCoverage': ''}, '查询本期正式指标', {
|
||||
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
|
||||
def test_protocol_short_prompt_lists_required_fields_and_enum_defaults(self):
|
||||
# Capture the planner request without invoking a live model.
|
||||
with patch('nao_agent_task._model_json', side_effect=[task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False,
|
||||
trendRequired=False, columnFilters=[]),
|
||||
{'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False,
|
||||
'requiresComparison': False, 'columnFiltersAligned': True}]) as model_call:
|
||||
plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question='查询本期正式指标',
|
||||
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
|
||||
cancel_event=threading.Event())
|
||||
prompt = model_call.call_args_list[0].kwargs['system']
|
||||
self.assertIn('responseStyle(brief/standard/detailed,可省略或null,默认standard)', prompt)
|
||||
self.assertIn('historyMode(none/last/all,可省略或null,默认none)', prompt)
|
||||
self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
|
||||
self.assertIn('freshQuery(bool)', prompt)
|
||||
self.assertIn('needsClarification(bool)', prompt)
|
||||
|
||||
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
|
||||
task = protocol_task()
|
||||
self.assertNotIn('columnFilters', task)
|
||||
self.assertEqual(task['filters']['columnFilters'], RULES)
|
||||
self.assertEqual(task['sourceFieldLabels'], {
|
||||
'batchNumber': '批号 | 代码', 'customerSpec': '客户规格', 'dispatchOrder': '委托单'})
|
||||
self.assertEqual(task['filters']['columnFilters'][0]['values'], ['0012', 'A|B'])
|
||||
self.assertEqual(task['filters']['columnFilters'][0]['stages'], ['black'])
|
||||
self.assertEqual(task['filters']['columnFilters'][1]['stages'], ['bright'])
|
||||
self.assertEqual(task['filters']['columnFilters'][2]['stages'], ['black', 'bright'])
|
||||
|
||||
def test_column_filter_rules_are_complete_replace_clear_and_inherit(self):
|
||||
previous = protocol_task(columnFilters=[RULES[0]])
|
||||
context = bind_task({'_protocol': {'version': 1}, '_businessCatalog': CATALOG}, previous)
|
||||
inherited = normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, columnFilters=None), '继续查询', context)
|
||||
self.assertEqual(inherited['filters']['columnFilters'], []) # explicit null clears
|
||||
omitted = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3})
|
||||
self.assertEqual(normalize_task(omitted, '继续查询', context)['filters']['columnFilters'], [RULES[0]])
|
||||
omitted['columnFilters'] = [RULES[1]]
|
||||
self.assertEqual(normalize_task(omitted, '改查亮片规格', context)['filters']['columnFilters'], [RULES[1]])
|
||||
cleared = normalize_task({**omitted, 'columnFilters': []}, '清除原表列条件', context)
|
||||
self.assertEqual(cleared['filters']['columnFilters'], [])
|
||||
cleared_context = bind_task(context, cleared)
|
||||
self.assertEqual(normalize_task(omitted | {'columnFilters': None}, '继续', cleared_context)['filters']['columnFilters'], [])
|
||||
new_task = normalize_task({**omitted, 'newTask': True, 'columnFilters': None}, '另开任务', context)
|
||||
self.assertEqual(new_task['filters']['columnFilters'], [])
|
||||
|
||||
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
|
||||
for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
|
||||
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
|
||||
[{**RULES[0], 'operator': 'contains'}]):
|
||||
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
|
||||
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026}, columnFilters=bad), '筛选原表字段',
|
||||
{'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
with self.assertRaisesRegex(TaskUnavailable, '目录不可用'):
|
||||
normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
|
||||
columnFilters=[RULES[0]]), '按原表列过滤', {'_protocol': {'version': 1}, '_businessCatalog': {}})
|
||||
|
||||
def test_semantic_review_must_confirm_source_filters_and_member_projection(self):
|
||||
source_plan = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, columnFilters=[RULES[0]])
|
||||
bad_review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
|
||||
'columnFiltersAligned': False}
|
||||
with patch('nao_agent_task._model_json', side_effect=[source_plan, bad_review] * 3) as model:
|
||||
with self.assertRaisesRegex(TaskUnavailable, '字段、阶段或成员清单'):
|
||||
plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
|
||||
question='只看黑片批号 0012 的合格率',
|
||||
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
|
||||
cancel_event=threading.Event(), source_value_resolver=self._value_resolver)
|
||||
self.assertEqual(model.call_count, 6)
|
||||
self.assertTrue(model.call_args_list[1].kwargs['payload']['columnFilterReviewRequired'])
|
||||
self.assertFalse(model.call_args_list[1].kwargs['payload']['memberListReviewRequired'])
|
||||
|
||||
member_plan = task_value(kind='members', dimensions=[], stages=['black'], fields=[],
|
||||
period={'year': 2026, 'month': 3}, memberFields=['batchNumber'], columnFilters=[])
|
||||
good_review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
|
||||
'memberListAligned': True}
|
||||
with patch('nao_agent_task._model_json', side_effect=[member_plan, good_review]) as model:
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
|
||||
question='列出黑片批号清单',
|
||||
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
|
||||
cancel_event=threading.Event())
|
||||
self.assertEqual(planned['kind'], 'members')
|
||||
self.assertEqual(model.call_count, 2)
|
||||
self.assertTrue(model.call_args_list[1].kwargs['payload']['memberListReviewRequired'])
|
||||
|
||||
@staticmethod
|
||||
def _value_resolver(*, release_id, period, filters, field, stages, search, limit, offset):
|
||||
return {'candidateOnly': True, 'releaseId': release_id, 'period': period, 'filters': filters,
|
||||
'field': field, 'selectedStages': stages, 'offset': offset, 'limit': limit,
|
||||
'totalCount': 1, 'hasMore': False, 'values': [search], 'rows': []}
|
||||
|
||||
def test_unconfirmed_source_value_is_clarified_without_partial_rule(self):
|
||||
source_plan = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, columnFilters=[
|
||||
{'field': 'dispatchOrder', 'stages': ['black'], 'operator': 'in', 'values': ['宝鑫加工']}])
|
||||
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
|
||||
'columnFiltersAligned': True}
|
||||
def resolver(**contract):
|
||||
return {'candidateOnly': True, 'releaseId': contract['release_id'],
|
||||
'period': contract['period'], 'filters': contract['filters'],
|
||||
'field': contract['field'], 'selectedStages': contract['stages'],
|
||||
'offset': contract['offset'], 'limit': contract['limit'],
|
||||
'totalCount': 1, 'hasMore': False, 'values': ['宝鑫加工厂'], 'rows': []}
|
||||
with patch('nao_agent_task._model_json', side_effect=[source_plan, review]) as model:
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
|
||||
question='按委托单宝鑫加工查询黑片合格率',
|
||||
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG,
|
||||
'releaseId': RELEASE}, cancel_event=threading.Event(), source_value_resolver=resolver)
|
||||
self.assertTrue(planned['needsClarification'])
|
||||
self.assertEqual(planned['filters']['columnFilters'], [])
|
||||
self.assertIn('不会自动替换', planned['clarification'])
|
||||
resolution = planned['sourceValueResolution']
|
||||
self.assertEqual(resolution['status'], 'unconfirmed')
|
||||
self.assertEqual(resolution['candidates'], ['宝鑫加工厂'])
|
||||
|
||||
def test_candidate_lookup_scopes_other_rules_to_the_requested_stage_only(self):
|
||||
source_plan = task_value(kind='metrics', dimensions=[], stages=['black', 'bright'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, columnFilters=RULES)
|
||||
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
|
||||
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
|
||||
'columnFiltersAligned': True}
|
||||
seen = []
|
||||
def resolver(**contract):
|
||||
seen.append(contract)
|
||||
return {'candidateOnly': True, 'releaseId': contract['release_id'],
|
||||
'period': contract['period'], 'filters': contract['filters'],
|
||||
'field': contract['field'], 'selectedStages': contract['stages'],
|
||||
'offset': contract['offset'], 'limit': contract['limit'],
|
||||
'totalCount': 1, 'hasMore': False, 'values': [contract['search']], 'rows': []}
|
||||
with patch('nao_agent_task._model_json', side_effect=[source_plan, review]):
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
|
||||
question='按黑片批号0012筛选,同时保留亮片客户规格条件',
|
||||
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG,
|
||||
'releaseId': RELEASE}, cancel_event=threading.Event(), source_value_resolver=resolver)
|
||||
self.assertEqual(planned['filters']['columnFilters'], RULES)
|
||||
self.assertEqual(len(seen), 3)
|
||||
batch_lookup = next(call for call in seen if call['field'] == 'batchNumber')
|
||||
self.assertEqual(batch_lookup['stages'], ['black'])
|
||||
self.assertEqual(batch_lookup['filters']['columnFilters'], [
|
||||
{'field': 'dispatchOrder', 'stages': ['black'], 'operator': 'not_empty'}])
|
||||
customer_lookup = next(call for call in seen if call['field'] == 'customerSpec')
|
||||
self.assertEqual(customer_lookup['stages'], ['bright'])
|
||||
self.assertEqual(customer_lookup['filters']['columnFilters'], [
|
||||
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
|
||||
|
||||
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
|
||||
from nao_agent_task import _resolve_source_column_values
|
||||
task = protocol_task(columnFilters=[RULES[0]])
|
||||
calls = []
|
||||
def resolver(**contract):
|
||||
calls.append(contract)
|
||||
response = self._value_resolver(**contract)
|
||||
response.update(values=[] if contract['search'] else ['亮片实际单位'],
|
||||
totalCount=0 if contract['search'] else 1)
|
||||
return response
|
||||
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
|
||||
self.assertEqual([c['search'] for c in calls], ['0012', ''])
|
||||
self.assertEqual({k:v for k,v in calls[0].items() if k!='search'},
|
||||
{k:v for k,v in calls[1].items() if k!='search'})
|
||||
self.assertEqual(result['status'], 'unconfirmed')
|
||||
self.assertEqual(result['candidates'], ['亮片实际单位'])
|
||||
self.assertTrue(task['needsClarification'])
|
||||
self.assertEqual(task['filters']['columnFilters'], [])
|
||||
self.assertIn('亮片实际单位', task['clarification'])
|
||||
|
||||
def test_alternative_candidate_receipt_cannot_broaden_the_scope(self):
|
||||
from nao_agent_task import _resolve_source_column_values
|
||||
task = protocol_task(columnFilters=[RULES[0]])
|
||||
def resolver(**contract):
|
||||
response = self._value_resolver(**contract)
|
||||
response.update(values=[] if contract['search'] else ['不能显示的其他阶段值'],
|
||||
totalCount=0 if contract['search'] else 1)
|
||||
if not contract['search']:response['selectedStages']=['bright']
|
||||
return response
|
||||
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
|
||||
self.assertEqual(result['status'], 'unconfirmed')
|
||||
self.assertEqual(result['candidates'], [])
|
||||
self.assertNotIn('不能显示', task['clarification'])
|
||||
self.assertEqual(task['filters']['columnFilters'], [])
|
||||
|
||||
def test_member_plan_repair_preserves_new_task_reset_and_customer_scope(self):
|
||||
previous = protocol_task()
|
||||
catalog = {**CATALOG, 'extraFields': [{'key': 'customerCode'}]}
|
||||
context = bind_task({'_protocol': {'version': 1}, '_businessCatalog': catalog,
|
||||
'releaseId': RELEASE}, previous)
|
||||
good = task_value(kind='members', dimensions=[], stages=['black', 'bright'], fields=[],
|
||||
memberFields=['product'], newTask=True, columnFilters=[],
|
||||
period={'year':2026,'month':9}, filterUpdates={'extraFilters':{'customerCode':['800001']}})
|
||||
bad = {**good, 'filterUpdates': {'customerCode':['800001']}}
|
||||
review = {'aligned':True, 'requiresEvidence':True, 'requiresSourceVerification':False,
|
||||
'requiresAllSourceRows':False, 'requiresTrend':False, 'requiresComparison':False,
|
||||
'columnFiltersAligned':True, 'memberListAligned':True, 'customerCodes':['800001']}
|
||||
with patch('nao_agent_task._model_json', side_effect=[bad,good,review]) as model:
|
||||
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
|
||||
question='新任务:800001客户2026年9月有哪些规格,黑片亮片一起看',
|
||||
context=context, previous=previous, cancel_event=threading.Event())
|
||||
repair = model.call_args_list[1].kwargs['payload']
|
||||
self.assertTrue(repair['invalidPlan']['newTask'])
|
||||
self.assertEqual(planned['filters']['extraFilters'], {'customerCode':['800001']})
|
||||
self.assertEqual(planned['filters']['columnFilters'], [])
|
||||
self.assertEqual(planned['memberFields'], ['product'])
|
||||
self.assertEqual(planned['stages'], ['black','bright'])
|
||||
self.assertTrue(planned['newTask'])
|
||||
|
||||
def test_quality_tools_sends_column_filters_and_requires_exact_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
api = EchoColumnApi()
|
||||
tools = QualityTools(api_factory=lambda: api, evidence_dir=folder)
|
||||
result = tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
|
||||
'columnFilters': RULES})
|
||||
metrics_path = next(path for _, path, _ in api.calls if urlsplit(path).path == '/api/metrics')
|
||||
sent = json.loads(parse_qs(urlsplit(metrics_path).query)['columnFilters'][0])
|
||||
self.assertEqual(sent, RULES)
|
||||
self.assertEqual(result['filters']['columnFilters'], RULES)
|
||||
self.assertEqual(result['trend'][0]['period'], '2026-03')
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tools = QualityTools(api_factory=lambda: EchoColumnApi(echo='mismatch'), evidence_dir=folder)
|
||||
with self.assertRaisesRegex(QualityToolError, '未原样确认 filters.columnFilters'):
|
||||
tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
|
||||
'columnFilters': RULES})
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tools = QualityTools(api_factory=lambda: EchoColumnApi(echo='missing'), evidence_dir=folder)
|
||||
with self.assertRaisesRegex(QualityToolError, '未原样确认 filters.columnFilters'):
|
||||
tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
|
||||
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
|
||||
'columnFilters': RULES})
|
||||
|
||||
def test_query_contract_extracts_authoritative_filters_and_rejects_changes(self):
|
||||
task = protocol_task()
|
||||
contract = {'task': task, 'releaseId': RELEASE}
|
||||
args = {'mode': 'query', 'period': task['period'], 'filters': {}, 'view': 'overall'}
|
||||
bound = bind_query(args, contract)
|
||||
self.assertEqual(bound['columnFilters'], RULES)
|
||||
self.assertNotIn('columnFilters', bound['filters'])
|
||||
with self.assertRaisesRegex(ValueError, '偏离本轮任务约定'):
|
||||
bind_query({**args, 'columnFilters': [RULES[0]]}, contract)
|
||||
with self.assertRaisesRegex(ValueError, '只能用于已确认的正式query'):
|
||||
bind_query({'mode': 'changes'}, contract)
|
||||
|
||||
def member_task(self, stages=('black', 'bright')):
|
||||
value = task_value(kind='members', dimensions=[], fields=[], stages=list(stages),
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
|
||||
memberFields=['batchNumber', 'dispatchOrder'], columnFilters=[])
|
||||
task = normalize_task(value, '列出委托单和批号',
|
||||
{'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
|
||||
task['filters'].pop('columnFilters', None) # confirmed backend state omits cleared filters
|
||||
return task
|
||||
|
||||
def _member_page(self, payload, offset, total, rows, *, proof=True):
|
||||
response = {'rows': rows, 'totalCount': total, 'offset': offset, 'limit': 200,
|
||||
'hasMore': offset + len(rows) < total,
|
||||
'complete': offset == 0 and offset + len(rows) >= total,
|
||||
'pageComplete': True,
|
||||
'releaseId': payload['releaseId'], 'filters': payload['filters'],
|
||||
'period': payload['period'], 'memberFields': payload['memberFields'],
|
||||
'selectedStages': payload['stages']}
|
||||
if proof:
|
||||
response['protocolEvidence'] = {'queryId': f'q-{offset}', **AUTHORITY, 'responseHash': 'a' * 64}
|
||||
return response
|
||||
|
||||
def test_members_paginates_checks_each_proof_and_renders_without_model(self):
|
||||
class MemberApi:
|
||||
def __init__(self): self.calls = []
|
||||
def call(self, path, payload=None, method=None):
|
||||
self.calls.append((path, payload, method))
|
||||
rows = ([{'values': {'batchNumber': '001|A' if i == 0 else f'B-{i}',
|
||||
'dispatchOrder': 'https://bad.test/<img>' if i == 0 else f'委托-{i}'},
|
||||
'stages': ['black', 'bright']} for i in range(200)]
|
||||
if payload['offset'] == 0 else
|
||||
[{'values': {'batchNumber': None, 'dispatchOrder': '委托-2'}, 'stages': ['bright']}])
|
||||
return SourceColumnProtocolTests()._member_page(payload, payload['offset'], 201, rows)
|
||||
api = MemberApi()
|
||||
task = self.member_task()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize') as synth:
|
||||
result = execute(request={'releaseId': RELEASE, 'turnId': AUTHORITY['turnId'], 'question': task['question']},
|
||||
task=task, authority=AUTHORITY, actor_root=root, delegated=api,
|
||||
connection=('offline', 'secret-unused', 'model-id'), cancel_event=threading.Event(), emit=lambda _: None)
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(len(result['protocolEvidence']), 2)
|
||||
self.assertEqual(result['queryEvidenceRefs'], [])
|
||||
self.assertEqual(result['visualState'], 'not_requested')
|
||||
self.assertIn('共 201 个去重项,已查全', result['answer'])
|
||||
self.assertNotIn('<img>', result['answer'])
|
||||
self.assertIn('翻页查看全部', result['answer'])
|
||||
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertEqual(len(api.calls), 2)
|
||||
self.assertEqual(api.calls[0][1]['offset'], 0)
|
||||
self.assertEqual(api.calls[1][1]['offset'], 200)
|
||||
self.assertNotIn('columnFilters', api.calls[0][1]['filters'])
|
||||
self.assertFalse(synth.called)
|
||||
proof_store = json.loads((root / 'private' / 'protocol-members-evidence.json').read_text())
|
||||
self.assertEqual(len(proof_store['pages']), 2)
|
||||
self.assertEqual(proof_store['displayedCount'], 201)
|
||||
self.assertTrue(proof_store['displayComplete'])
|
||||
self.assertEqual(proof_store['pages'][0]['response']['rows'][0]['values']['dispatchOrder'], 'https://bad.test/<img>')
|
||||
self.assertEqual(len(json.loads((root / 'private' / 'protocol-completion.json').read_text())['protocolEvidence']), 2)
|
||||
|
||||
def test_members_empty_result_and_proof_failure_are_distinct(self):
|
||||
task = self.member_task(('black',))
|
||||
class EmptyApi:
|
||||
def call(inner, path, payload=None, method=None):
|
||||
return SourceColumnProtocolTests()._member_page(payload, 0, 0, [])
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
empty = execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
|
||||
authority=AUTHORITY, actor_root=root, delegated=EmptyApi(),
|
||||
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
|
||||
self.assertEqual(empty['status'], 'COMPLETED')
|
||||
self.assertIn('共 0 个去重项,已查全', empty['answer'])
|
||||
class NoProof(EmptyApi):
|
||||
def call(inner, path, payload=None, method=None):
|
||||
return SourceColumnProtocolTests()._member_page(payload, 0, 0, [], proof=False)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
with self.assertRaisesRegex(ValueError, '缺少完整协议证明'):
|
||||
execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
|
||||
authority=AUTHORITY, actor_root=root, delegated=NoProof(),
|
||||
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
|
||||
|
||||
def test_members_stops_after_twenty_pages_and_marks_truncation(self):
|
||||
task = self.member_task(('black',))
|
||||
class LargeApi:
|
||||
def __init__(self): self.calls = 0
|
||||
def call(inner, path, payload=None, method=None):
|
||||
inner.calls += 1
|
||||
rows = [{'values': {'batchNumber': str(payload['offset'] + i), 'dispatchOrder': 'D'},
|
||||
'stages': ['black']} for i in range(200)]
|
||||
return SourceColumnProtocolTests()._member_page(payload, payload['offset'], 5000, rows)
|
||||
api = LargeApi()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
|
||||
result = execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
|
||||
authority=AUTHORITY, actor_root=root, delegated=api,
|
||||
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
|
||||
self.assertEqual(api.calls, 20)
|
||||
self.assertEqual(result['status'], 'PARTIAL')
|
||||
self.assertEqual(len(result['protocolEvidence']), 20)
|
||||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||||
self.assertIn('另有 1000 项未读取', result['answer'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,185 @@
|
||||
import copy
|
||||
import io
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
|
||||
from nao_analysis_coverage import plan_missing_source_queries
|
||||
from nao_evidence_finalizer import coverage_sufficient
|
||||
from test_nao_agent_task import task_value
|
||||
from test_agent_runtime import comparison
|
||||
|
||||
RELEASE='33333333-3333-4333-8333-333333333333'
|
||||
CURRENT={'year':2026,'month':9}
|
||||
PREVIOUS={'year':2026,'month':8}
|
||||
FILTERS={'department':'新能源汽车磁钢'}
|
||||
|
||||
|
||||
def review(source, rows, *, evidence=True):
|
||||
return {'aligned':True,'requiresEvidence':evidence,
|
||||
'requiresSourceVerification':source,'requiresAllSourceRows':rows,'requiresTrend':False}
|
||||
|
||||
|
||||
def plan_value(**kwargs):
|
||||
return task_value(dimensions=['group'],dimensionStages={'group':[]},stages=['black','bright'],
|
||||
fields=['rate','scrapWeight'],period=CURRENT,comparisonPeriod=PREVIOUS,
|
||||
comparisonRequired=True,filterUpdates=FILTERS,**kwargs)
|
||||
|
||||
|
||||
def source_document(stage, group=None, *, bad=False, missing_comparison=False):
|
||||
filters={**FILTERS,**({'group':group} if group else {})}
|
||||
def side(period):
|
||||
context={'releaseId':RELEASE,**period,**filters,'stage':stage,'includeUnassigned':False}
|
||||
return {'context':context,'aggregateComplete':True,'totalRows':2,
|
||||
'sourceVerification':{'complete':not bad,'allSelectedValuesMatchStored':not bad,
|
||||
'expectedRows':2,'verifiedRows':1 if bad else 2,'missingRows':1 if bad else 0,'mismatchRows':0}}
|
||||
result={'kind':'quality_investigation','action':'source','releaseId':RELEASE,'stage':stage,
|
||||
'period':CURRENT,'comparisonPeriod':PREVIOUS,'filters':filters,'current':side(CURRENT)}
|
||||
if not missing_comparison:result['comparison']=side(PREVIOUS)
|
||||
return {'mode':'investigate','evidenceRef':'QAAAAAAAA','result':result}
|
||||
|
||||
|
||||
class SemanticSourceIntentTests(unittest.TestCase):
|
||||
def run_plan(self, question, replies, context=None):
|
||||
with patch('nao_agent_task._model_json',side_effect=replies) as call:
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question=question,context=context or {},cancel_event=threading.Event())
|
||||
return result,call
|
||||
|
||||
def test_unrequested_source_read_is_replanned_not_silently_rewritten(self):
|
||||
initial=plan_value(verifySource=True)
|
||||
corrected=plan_value()
|
||||
context={'_serverQuestions':[{'question':'9月合格率为什么下降?','status':'PARTIAL'}]}
|
||||
result,call=self.run_plan('主要是国内还是国外的',[initial,review(False,False),corrected,review(False,False)],context)
|
||||
self.assertFalse(result['verifySource']);self.assertEqual(call.call_count,4)
|
||||
self.assertTrue(call.call_args_list[2].kwargs['payload']['invalidPlan']['verifySource'])
|
||||
self.assertIn('verifySource',call.call_args_list[2].kwargs['payload']['validationFeedback'])
|
||||
|
||||
def test_explicit_source_and_all_rows_requirements_are_preserved(self):
|
||||
source=plan_value(verifySource=True)
|
||||
result,_=self.run_plan('请对照原表核验两期数据',[source,review(True,False)])
|
||||
self.assertTrue(result['verifySource'])
|
||||
rows=plan_value(fullDetails=True)
|
||||
result,_=self.run_plan('请逐条列出所有原始记录',[rows,review(False,True)])
|
||||
self.assertTrue(result['fullDetails'])
|
||||
|
||||
def test_explicit_source_omitted_by_planner_is_replanned(self):
|
||||
first=plan_value()
|
||||
corrected=plan_value(verifySource=True)
|
||||
result,call=self.run_plan('请对照原表核验',[first,review(True,False),corrected,review(True,False)])
|
||||
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
|
||||
|
||||
def test_missing_read_requirement_fields_and_explicit_conflict_fail_closed(self):
|
||||
source=plan_value(verifySource=True)
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
self.run_plan('请核验原表',[source,{'aligned':True,'requiresEvidence':True}]*3)
|
||||
with self.assertRaises(TaskUnavailable):
|
||||
self.run_plan('请核验原表',[plan_value(),review(False,False)]*3)
|
||||
|
||||
def test_explicit_cancellation_does_not_force_source_read(self):
|
||||
result,_=self.run_plan('不要核对原表,只看合格率',[plan_value(),review(False,False)])
|
||||
self.assertFalse(result['verifySource'])
|
||||
|
||||
def test_natural_first_turn_source_request_is_reviewed_even_when_plan_omits_flag(self):
|
||||
question='把上传的xlsx中的数值与系统记录逐项比对'
|
||||
result,call=self.run_plan(question,[plan_value(),review(True,False),plan_value(verifySource=True),review(True,False)])
|
||||
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
|
||||
|
||||
def test_previous_source_requirement_and_explicit_cancellation_follow_current_intent(self):
|
||||
history={'_serverQuestions':[{'question':'把上传的xlsx中的数值与系统记录逐项比对','status':'PARTIAL'}]}
|
||||
result,_=self.run_plan('按刚才要求继续,主要看国内还是海外',
|
||||
[plan_value(),review(True,False),plan_value(verifySource=True),review(True,False)],history)
|
||||
self.assertTrue(result['verifySource'])
|
||||
result,_=self.run_plan('不用核对原表,只看合格率',[plan_value(),review(False,False)],history)
|
||||
self.assertFalse(result['verifySource'])
|
||||
|
||||
def test_positive_optional_review_fields_are_never_ignored(self):
|
||||
history={'_serverQuestions':[{'question':'将上传的资料和系统的值一项项核一遍','status':'PARTIAL'}]}
|
||||
first=plan_value();corrected=plan_value(verifySource=True)
|
||||
with patch('nao_agent_task._model_json',side_effect=[first,review(True,False),corrected,review(True,False)]) as call:
|
||||
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
|
||||
question='按刚才要求继续,主要看国内还是海外',context=history,cancel_event=threading.Event(),semantic_review=True)
|
||||
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
|
||||
|
||||
|
||||
class ParentSourceRepairTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.task=normalize_task(task_value(kind='source_verify',dimensions=[],dimensionStages={},
|
||||
stages=['black','bright'],fields=[],period=CURRENT,comparisonPeriod=PREVIOUS,
|
||||
comparisonRequired=True,filterUpdates=FILTERS),'对照原表核验')
|
||||
self.context=bind_task({'releaseId':RELEASE},self.task)
|
||||
root=comparison();root['releaseId']=RELEASE;root['filterBinding']=FILTERS
|
||||
self.root={'mode':'compare','evidenceRef':'QBBBBBBBB','result':root}
|
||||
self.children=[source_document('black','汽车交通-国内'),source_document('bright','汽车交通-国内'),
|
||||
source_document('black','汽车交通-海外')]
|
||||
|
||||
def test_child_scopes_do_not_satisfy_parent_and_two_stages_cover_both_periods(self):
|
||||
docs=[self.root,*self.children]
|
||||
queries=plan_missing_source_queries(docs,self.task,self.context)
|
||||
self.assertEqual([q['stage'] for q in queries],['black','bright'])
|
||||
for q in queries:
|
||||
self.assertEqual(q['filters'],FILTERS);self.assertEqual(q['period'],CURRENT)
|
||||
self.assertEqual(q['comparisonPeriod'],PREVIOUS);self.assertEqual(q['action'],'source')
|
||||
self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
docs.append(source_document('black'));self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
self.assertEqual([q['stage'] for q in plan_missing_source_queries(docs,self.task,self.context)],['bright'])
|
||||
docs.append(source_document('bright'));self.assertTrue(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
|
||||
def test_missing_period_or_mismatched_source_stays_incomplete(self):
|
||||
for doc in (source_document('bright',bad=True),source_document('bright',missing_comparison=True)):
|
||||
docs=[self.root,source_document('black'),doc]
|
||||
self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
self.assertEqual([q['stage'] for q in plan_missing_source_queries(docs,self.task,self.context)],['bright'])
|
||||
|
||||
def test_runtime_executes_both_parent_reads_with_longer_bounded_timeout(self):
|
||||
docs=[self.root,*self.children];seen=[];api=MagicMock()
|
||||
def read(arguments):
|
||||
seen.append((copy.deepcopy(arguments),api.timeout));result=source_document(arguments['stage']);docs.append(result);return result
|
||||
request={'actorId':'actor','sessionId':'session','releaseId':RELEASE,'question':self.task['question'],'context':self.context}
|
||||
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
|
||||
patch('nao_agent_task.plan_repairs',side_effect=AssertionError('parent checks should be planned directly')),\
|
||||
patch.object(runtime,'DelegatedApi') as delegated,patch('nao_quality_tools.QualityTools') as tools,\
|
||||
patch('nao_runtime_policy.remaining_seconds',return_value=1000):
|
||||
delegated.return_value.login.return_value=api;tools.return_value.quality_data.side_effect=read
|
||||
runtime.complete_agent_evidence(task=self.task,request=request,actor_root=Path('/offline/session'),
|
||||
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
|
||||
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
|
||||
self.assertEqual([q['stage'] for q,_ in seen],['black','bright'])
|
||||
self.assertTrue(all(q['filters']==FILTERS and timeout==180 for q,timeout in seen))
|
||||
self.assertTrue(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
|
||||
def test_finalization_time_reserve_prevents_false_completion(self):
|
||||
docs=[self.root,*self.children];seen=[];api=MagicMock()
|
||||
request={'actorId':'actor','sessionId':'session','releaseId':RELEASE,'question':self.task['question'],'context':self.context}
|
||||
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
|
||||
patch.object(runtime,'DelegatedApi') as delegated,patch('nao_quality_tools.QualityTools') as tools,\
|
||||
patch('nao_runtime_policy.remaining_seconds',return_value=90):
|
||||
delegated.return_value.login.return_value=api;tools.return_value.quality_data.side_effect=lambda args:seen.append(args)
|
||||
runtime.complete_agent_evidence(task=self.task,request=request,actor_root=Path('/offline/session'),
|
||||
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
|
||||
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
|
||||
self.assertEqual(seen,[]);self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
|
||||
|
||||
def test_each_http_request_recalculates_source_budget(self):
|
||||
timeouts=[]
|
||||
class Opener:
|
||||
def open(self, request, timeout):
|
||||
timeouts.append(timeout);return io.BytesIO(b'{}')
|
||||
api=runtime.DelegatedApi('/offline/access',opener=Opener(),timeout=180)
|
||||
api.deadline_reserve_seconds=90
|
||||
access=SimpleNamespace(release_id=RELEASE,token='synthetic',query_url='http://127.0.0.1/offline')
|
||||
with patch.object(runtime,'load_access_file',return_value=access),\
|
||||
patch('nao_runtime_policy.remaining_seconds',side_effect=[250,100,90]):
|
||||
api.call('/api/options?releaseId='+RELEASE,method='GET')
|
||||
api.call('/api/options?releaseId='+RELEASE,method='GET')
|
||||
with self.assertRaises(runtime.RuntimeErrorSafe):api.call('/api/options?releaseId='+RELEASE,method='GET')
|
||||
self.assertEqual(timeouts,[160,10])
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Per-turn diagnostic evidence must stay private and independently readable."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_agent_task import TaskUnavailable
|
||||
from nao_quality_runtime import RuntimeErrorSafe, plan_agent_task
|
||||
|
||||
|
||||
class TaskDiagnosticHistoryTests(unittest.TestCase):
|
||||
def test_success_and_failure_are_distinct_private_redacted_records(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
private = Path(tmp) / 'private'
|
||||
private.mkdir()
|
||||
legacy = private / 'task-plan-failure.json'
|
||||
first, second = str(uuid.uuid4()), str(uuid.uuid4())
|
||||
|
||||
def success(**kwargs):
|
||||
kwargs['on_validation_error']({'attempt': 1, 'field': 'filterUpdates.product',
|
||||
'receivedType': 'dict', 'expected': 'password=private',
|
||||
'message': 'Bearer private-token', 'rawValue': 'secret-value'})
|
||||
return {'kind': 'metrics'}
|
||||
|
||||
def failure(**kwargs):
|
||||
kwargs['on_validation_error']({'attempt': 2, 'field': 'goal', 'receivedType': 'str',
|
||||
'message': 'api_key=private-secret'})
|
||||
raise TaskUnavailable('password=private-secret', field='goal', received={'token': 'raw'})
|
||||
|
||||
with patch('nao_agent_task.plan_task', side_effect=success):
|
||||
self.assertEqual(plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=first)['kind'], 'metrics')
|
||||
with patch('nao_agent_task.plan_task', side_effect=failure):
|
||||
with self.assertRaisesRegex(RuntimeErrorSafe, '尚未查询数据'):
|
||||
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=second)
|
||||
first_path = private / 'task-plan-turns' / (first + '.json')
|
||||
second_path = private / 'task-plan-turns' / (second + '.json')
|
||||
first_data, second_data = json.loads(first_path.read_text()), json.loads(second_path.read_text())
|
||||
self.assertEqual(first_data['status'], 'planned')
|
||||
self.assertEqual(first_data['attempts'][0]['field'], 'filterUpdates.product')
|
||||
self.assertEqual(second_data['status'], 'failed')
|
||||
self.assertEqual(second_data['failure']['field'], 'goal')
|
||||
for path in (first_path, second_path):
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
|
||||
for sensitive in ('private-secret', 'private-token', 'secret-value', 'password='):
|
||||
self.assertNotIn(sensitive, path.read_text())
|
||||
self.assertTrue(legacy.exists())
|
||||
self.assertTrue((private / 'task-plan-validation.json').exists())
|
||||
|
||||
def test_concurrent_turns_keep_independent_attempt_history(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
private = Path(tmp) / 'private'
|
||||
private.mkdir()
|
||||
legacy = private / 'task-plan-failure.json'
|
||||
barrier = threading.Barrier(2)
|
||||
ids = [str(uuid.uuid4()), str(uuid.uuid4())]
|
||||
errors = []
|
||||
|
||||
def planner(**kwargs):
|
||||
barrier.wait(timeout=5)
|
||||
kwargs['on_validation_error']({'attempt': kwargs['marker'], 'field': 'goal', 'receivedType': 'str'})
|
||||
return {'kind': 'metrics'}
|
||||
|
||||
def run(index):
|
||||
try:
|
||||
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=ids[index], marker=index + 1)
|
||||
except Exception as error:
|
||||
errors.append(error)
|
||||
|
||||
with patch('nao_agent_task.plan_task', side_effect=planner):
|
||||
threads = [threading.Thread(target=run, args=(i,)) for i in range(2)]
|
||||
for thread in threads: thread.start()
|
||||
for thread in threads: thread.join(timeout=10)
|
||||
self.assertFalse(errors)
|
||||
for index, turn_id in enumerate(ids):
|
||||
data = json.loads((private / 'task-plan-turns' / (turn_id + '.json')).read_text())
|
||||
self.assertEqual(data['status'], 'planned')
|
||||
self.assertEqual(data['attempts'][0]['attempt'], index + 1)
|
||||
|
||||
def test_invalid_turn_and_symlink_path_never_write_outside_private_directory(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
private = Path(tmp) / 'private'
|
||||
outside = Path(tmp) / 'outside'
|
||||
private.mkdir(); outside.mkdir()
|
||||
legacy = private / 'task-plan-failure.json'
|
||||
with patch('nao_agent_task.plan_task', return_value={'kind': 'metrics'}):
|
||||
with self.assertRaises(ValueError):
|
||||
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id='../outside/pwn')
|
||||
os.symlink(outside, private / 'task-plan-turns')
|
||||
with self.assertRaises(ValueError):
|
||||
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=str(uuid.uuid4()))
|
||||
self.assertEqual(list(outside.iterdir()), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,80 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from analysis_visuals import build_visuals
|
||||
from test_month_range_evidence import QUESTION, context, metric_doc
|
||||
|
||||
|
||||
class VisualStageReferenceTests(unittest.TestCase):
|
||||
def test_explicit_empty_denominators_stay_a_gap_instead_of_fallback_100(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
doc = metric_doc(period, range(1, 6))
|
||||
for row in doc['result']['trend']:
|
||||
row['black']['totalWeight'] = 100
|
||||
row['bright']['totalWeight'] = 100
|
||||
invalid = doc['result']['trend'][1]
|
||||
invalid.update(compositeRate=1)
|
||||
invalid['black']['totalWeight'] = 0
|
||||
invalid['bright']['totalWeight'] = None
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
|
||||
self.assertEqual(chart['series'][0]['values'], [.71, None, .73, .74, .75])
|
||||
|
||||
def test_unknown_source_counts_cannot_turn_into_a_100_percent_point(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
doc = metric_doc(period, range(1, 6))
|
||||
doc['result']['trend'][1].update(compositeRate=1, coverage={'blackRows': None, 'brightRows': None})
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
|
||||
self.assertIsNone(chart['series'][0]['values'][1])
|
||||
|
||||
def test_one_unknown_side_or_parent_source_count_stays_a_gap(self):
|
||||
for change_row in (True, False):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
doc = metric_doc(period, range(1, 6))
|
||||
doc['result']['coverage']['sourceMonths'][1]['brightRows'] = None
|
||||
if change_row:
|
||||
doc['result']['trend'][1]['coverage']['brightRows'] = None
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
|
||||
self.assertIsNone(chart['series'][0]['values'][1])
|
||||
|
||||
def test_stage_rate_zero_denominator_is_a_gap_even_with_positive_source_rows(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
frame = context(period)
|
||||
frame['_agentTask']['stages'] = ['black']
|
||||
doc = metric_doc(period, range(1, 6))
|
||||
for row in doc['result']['trend']:
|
||||
row['black']['totalWeight'] = 100
|
||||
doc['result']['trend'][1]['black'].update(rate=1, totalWeight=0)
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=frame)['visuals'][0]
|
||||
self.assertEqual(chart['metric'], 'black_rate')
|
||||
self.assertIsNone(chart['series'][0]['values'][1])
|
||||
|
||||
def test_scrap_rate_zero_denominator_is_a_gap_and_real_zero_is_kept(self):
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
|
||||
frame = context(period)
|
||||
frame['_agentTask'].update(stages=['black'], fields=['scrapRate'])
|
||||
doc = metric_doc(period, range(1, 6))
|
||||
for row in doc['result']['trend']:
|
||||
row['black'].update(totalWeight=100, scrapWeight=10, scrapRate=.1)
|
||||
doc['result']['trend'][1]['black'].update(totalWeight=0, scrapWeight=0, scrapRate=0)
|
||||
doc['result']['trend'][2]['black'].update(totalWeight=100, scrapWeight=0, scrapRate=0)
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=frame)['visuals'][0]
|
||||
self.assertEqual(chart['metric'], 'black_scrap_rate')
|
||||
self.assertEqual(chart['series'][0]['values'], [.1, None, 0, .1, .1])
|
||||
|
||||
def test_real_zero_with_one_positive_stage_is_preserved_in_both_directions(self):
|
||||
for stage, other in [('black', 'bright'), ('bright', 'black')]:
|
||||
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 3}
|
||||
doc = metric_doc(period, range(1, 4))
|
||||
for row in doc['result']['trend']:
|
||||
row['compositeRate'] = 0
|
||||
row[stage]['totalWeight'] = 100
|
||||
row[other]['totalWeight'] = None
|
||||
row['coverage'][other + 'Rows'] = 0
|
||||
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
|
||||
self.assertEqual(chart['series'][0]['values'], [0, 0, 0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,328 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_quality_tools import QualityTools, QualityToolError
|
||||
from test_nao_quality_tools import FakeApi, snapshot_period
|
||||
from nao_agent_task import normalize_task, bind_task
|
||||
from test_nao_agent_task import task_value
|
||||
from analysis_visuals import build_visuals
|
||||
from nao_evidence_finalizer import coverage_sufficient
|
||||
from nao_protocol_execution import execute
|
||||
from nao_weight_ranking import rank_response, ranking_chart, ranking_report, missing_ranking_visuals
|
||||
|
||||
|
||||
QUESTION = '做几张26年的亮片生产重量柏拉图,产品级、产业部级'
|
||||
ACTOR='11111111-1111-4111-8111-111111111111'
|
||||
SESSION='22222222-2222-4222-8222-222222222222'
|
||||
TURN='55555555-5555-4555-8555-555555555555'
|
||||
|
||||
|
||||
def signed_response(positive_rows=872, unavailable_rows=167, *, top_weight=None, metric='totalWeight', stage='black'):
|
||||
period={'year':2026,'monthStart':1,'monthEnd':9}
|
||||
total=sum([top_weight] if top_weight is not None else [])
|
||||
rows=[]
|
||||
for index in range(positive_rows):
|
||||
weight=(top_weight if index==0 and top_weight is not None else 1)
|
||||
total+=weight if not (index==0 and top_weight is not None) else 0
|
||||
rows.append({'key':f'product-{index:04d}','label':f'规格{index:04d}',
|
||||
stage:{metric:weight},'coverage':{stage+'Rows':1}})
|
||||
rows.append({'key':'mimo-negative','label':'mimo-negative',stage:{metric:-10.05},'coverage':{stage+'Rows':1}})
|
||||
for index in range(unavailable_rows):
|
||||
rows.append({'key':f'no-stage-{index:04d}','label':f'未覆盖{index:04d}',stage:{metric:None},
|
||||
'coverage':{stage+'Rows':0}})
|
||||
total-=10.05
|
||||
months=[{'year':2026,'month':month,'blackRows':1 if stage=='black' else 0,
|
||||
'brightRows':1 if stage=='bright' else 0} for month in range(1,10)]
|
||||
summary={stage:{metric:total}}
|
||||
return {'rows':rows,'context':{'rowCount':len(rows)},'summary':summary,
|
||||
'coverage':{'rawRows':len(rows),'includedRows':len(rows),'sourceMonths':months}},period
|
||||
|
||||
|
||||
class WeightApi(FakeApi):
|
||||
def __init__(self, *, partial_year=True, hole=False, mismatch=False):
|
||||
super().__init__()
|
||||
self.partial_year, self.hole, self.mismatch = partial_year, hole, mismatch
|
||||
self.metric_query = 0
|
||||
|
||||
def call(self, path, payload=None, method=None):
|
||||
result = super().call(path, payload, method)
|
||||
if path == '/api/analysis/tools/catalog':
|
||||
result['sourceFields'] = [{'key':'customer_code','label':'客户编码','type':'source_value',
|
||||
'stages':['bright'],'columns':{'bright':'客户编码'},'operators':['in']}]
|
||||
if path.startswith('/api/metrics?'):
|
||||
self.metric_query += 1
|
||||
params = parse_qs(urlsplit(path).query)
|
||||
for month in result['coverage']['sourceMonths']:month['brightRows']=12
|
||||
rows = []
|
||||
for label, weight, scrap in [('scrap-leader', 10, 9), ('production-leader', 80, 1), ('third', 60, 2)]:
|
||||
row = self._product_row(label, label, 1, scrap)
|
||||
row['bright']['totalWeight'] = weight
|
||||
rows.append(row)
|
||||
result['rows'] = rows
|
||||
result['context']['rowCount'] = 3
|
||||
result['summary']['bright']['totalWeight'] = 151 if self.mismatch else 150
|
||||
result['summary']['bright']['scrapWeight'] = 12
|
||||
if self.partial_year:
|
||||
result['coverage']['sourceMonths'] = result['coverage']['sourceMonths'][:9]
|
||||
result['coverage']['missingStages'] = [{'year':2026,'month':m,'stage':'bright'} for m in (10,11,12)]
|
||||
if self.hole:
|
||||
result['coverage']['sourceMonths'] = [m for m in result['coverage']['sourceMonths'] if m['month'] != 5]
|
||||
if 'monthStart' in params:
|
||||
result['context']['monthStart'] = int(params['monthStart'][0])
|
||||
result['context']['monthEnd'] = int(params['monthEnd'][0])
|
||||
if 'columnFilters' in params:
|
||||
result['filters'] = {'columnFilters':json.loads(params['columnFilters'][0])}
|
||||
result['protocolEvidence'] = {'queryId':f'rank-query-{self.metric_query}',
|
||||
'attemptId':'attempt','sessionId':'session','turnId':'turn','revision':1,
|
||||
'executionId':'execution','responseHash':'a'*64}
|
||||
return result
|
||||
|
||||
|
||||
class NegativeWeightApi(WeightApi):
|
||||
def call(self, path, payload=None, method=None):
|
||||
result=super().call(path,payload,method)
|
||||
if path.startswith('/api/metrics?'):
|
||||
params=parse_qs(urlsplit(path).query)
|
||||
if params.get('view',[''])[0]=='product':
|
||||
rows=[]
|
||||
for key,label,weight in (('large','大重量规格',100),('small','小重量规格',1),('mimo','mimo修正',-10.05)):
|
||||
row=self._product_row(key,label,1,1)
|
||||
row['black']['totalWeight']=weight
|
||||
rows.append(row)
|
||||
result['rows']=rows;result['context']['rowCount']=3
|
||||
result['summary']['black']['totalWeight']=90.95
|
||||
for month in result['coverage']['sourceMonths']:month['blackRows']=1
|
||||
result['protocolEvidence']={'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,
|
||||
'revision':3,'executionId':TURN,'queryId':'signed-rank-query','responseHash':'b'*64}
|
||||
return result
|
||||
|
||||
|
||||
def planned(**updates):
|
||||
value={'dimensions':['product','department'],'stages':['bright'],'fields':['totalWeight'],
|
||||
'period':{'year':2026},'visualPreference':'required','visualType':'pareto','periodCoverage':'available'}
|
||||
value.update(updates)
|
||||
return normalize_task(task_value(**value), QUESTION)
|
||||
|
||||
|
||||
class WeightRankingTests(unittest.TestCase):
|
||||
def run_rank(self, metric='totalWeight', **options):
|
||||
api = WeightApi(**options)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
QualityTools(api_factory=lambda:api, evidence_dir=Path(folder)).quality_data({
|
||||
'mode':'rank','releaseId':'release-a','period':{'year':2026},
|
||||
'dimension':'product','stage':'bright','metric':metric,'limit':2})
|
||||
doc = json.loads(next(Path(folder).glob('nao-quality-*.json')).read_text())
|
||||
return doc, api
|
||||
|
||||
def test_ranks_all_products_by_requested_stage_weight_before_top_cut(self):
|
||||
doc, api = self.run_rank()
|
||||
result = doc['result']
|
||||
self.assertEqual([row['label'] for row in result['rows']], ['production-leader','third'])
|
||||
self.assertEqual(result['totalWeight'], 150)
|
||||
self.assertEqual(result['rows'][0]['share'], 80/150)
|
||||
self.assertAlmostEqual(result['rows'][-1]['cumulativeShare'], 140/150)
|
||||
self.assertLess(result['rows'][-1]['cumulativeShare'], 1)
|
||||
self.assertEqual(result['rankingCoverage']['groups'], 3)
|
||||
self.assertFalse(any('rowLimit=' in path for _,path,_ in api.calls if path.startswith('/api/metrics')))
|
||||
|
||||
def test_black_total_weight_signed_reconciliation_keeps_all_absent_groups_auditable(self):
|
||||
response,period=signed_response()
|
||||
result=rank_response(response,{'id':'release-a','name':'A','status':'PUBLISHED'},period,{},
|
||||
'product','black','totalWeight',10)
|
||||
audit=result['rankingCoverage']
|
||||
self.assertEqual(response['context']['rowCount'],1040)
|
||||
self.assertEqual(audit['sourceGroups'],1040)
|
||||
self.assertEqual(audit['groups'],873)
|
||||
self.assertEqual(audit['unavailableGroups'],167)
|
||||
self.assertEqual(audit['positiveGroups'],872)
|
||||
self.assertEqual(audit['negativeGroups'],1)
|
||||
self.assertEqual(audit['negativeWeight'],-10.05)
|
||||
self.assertEqual(audit['negativeWeightMagnitude'],10.05)
|
||||
self.assertEqual(result['totalWeight'],861.95)
|
||||
self.assertEqual(result['rows'][0]['weight'],1)
|
||||
self.assertTrue(all(row['weight']>0 for row in result['rows']))
|
||||
self.assertAlmostEqual(result['rows'][0]['share'],1/861.95)
|
||||
doc={'evidenceRef':'QSIGNED10','result':result}
|
||||
task=planned(dimensions=['product'],stages=['black'],fields=['totalWeight'],period=period)
|
||||
chart=ranking_chart(doc,'signed-black-total',task)
|
||||
self.assertIsNotNone(chart)
|
||||
self.assertIn('1项负向投入重量修正',chart['note'])
|
||||
bad=copy.deepcopy(response);bad['summary']['black']['totalWeight']+=.1
|
||||
with self.assertRaisesRegex(QualityToolError,'合计与同范围正式总量不一致'):
|
||||
rank_response(bad,{'id':'release-a'},period,{},'product','black','totalWeight',10)
|
||||
|
||||
def test_negative_weights_remain_rejected_outside_black_total_weight(self):
|
||||
for stage,metric in (('black','scrapWeight'),('bright','totalWeight'),('bright','scrapWeight')):
|
||||
with self.subTest(stage=stage,metric=metric):
|
||||
response={'rows':[{'key':'negative','label':'负值',stage:{metric:-1},'coverage':{stage+'Rows':1}}],
|
||||
'context':{'rowCount':1},'summary':{stage:{metric:10}},
|
||||
'coverage':{'sourceMonths':[{'year':2026,'month':month,'blackRows':1,'brightRows':1}
|
||||
for month in range(1,10)]}}
|
||||
with self.assertRaisesRegex(QualityToolError,'重量不能为负'):
|
||||
rank_response(response,{'id':'release-a'},{'year':2026,'monthStart':1,'monthEnd':9},
|
||||
{},'product',stage,metric,10)
|
||||
|
||||
def test_signed_black_total_reports_correction_and_blocks_over_100_chart_as_partial(self):
|
||||
response,period=signed_response(positive_rows=2,unavailable_rows=0,top_weight=100)
|
||||
result=rank_response(response,{'id':'release-a','name':'A','status':'PUBLISHED'},period,{},
|
||||
'product','black','totalWeight',10)
|
||||
self.assertEqual(result['totalWeight'],90.95)
|
||||
self.assertEqual([row['label'] for row in result['rows']],['规格0000','规格0001'])
|
||||
self.assertGreater(result['rows'][-1]['cumulativeShare'],1)
|
||||
document={'evidenceRef':'QNEGATIVE1','result':result}
|
||||
task=planned(dimensions=['product'],stages=['black'],fields=['totalWeight'],period=period,
|
||||
visualPreference='required')
|
||||
context=bind_task({'releaseId':'release-a'},task)
|
||||
self.assertIsNone(ranking_chart(document,'negative-correction',task))
|
||||
gap=missing_ranking_visuals(task,[],[document])
|
||||
self.assertEqual(len(gap),1)
|
||||
self.assertIn('负向修正使正重量Top合计超过黑片净总重',gap[0])
|
||||
report=ranking_report([document],task,context)
|
||||
self.assertIn('1项负向投入重量修正,净额-10.05 kg仍计入正式净总重分母',report['facts'][0]['text'])
|
||||
section=report['sections'][0]
|
||||
self.assertGreater(section.index('排名表只展示正重量Top'),section.index('| 1 |'))
|
||||
|
||||
def test_protocol_execution_keeps_signed_denominator_but_returns_partial_without_over_100_chart(self):
|
||||
t=normalize_task(task_value(kind='analysis',dimensions=['product'],stages=['black'],
|
||||
dimensionStages={'product':['black']},fields=['totalWeight'],period={'year':2026},
|
||||
trendRequired=False,visualPreference='auto',visualType='pareto'),
|
||||
'按产品查看黑片投入重量排名',{'_protocol':{'version':1}})
|
||||
if t['filters'].get('columnFilters')==[]:t['filters'].pop('columnFilters')
|
||||
api=NegativeWeightApi()
|
||||
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={
|
||||
'answer':'已核验黑片修正重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}):
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
result=execute(request={'releaseId':'release-a','turnId':TURN,'question':t['question']},task=t,
|
||||
authority={'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN},
|
||||
actor_root=root,delegated=api,connection=('offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
doc=json.loads(next((root/'evidence').glob('nao-quality-*.json')).read_text())
|
||||
self.assertEqual(doc['result']['totalWeight'],90.95)
|
||||
self.assertEqual(doc['result']['rankingCoverage']['negativeGroups'],1)
|
||||
self.assertEqual(doc['result']['rankingCoverage']['negativeWeight'],-10.05)
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertTrue(any('负向修正使正重量Top合计超过黑片净总重' in item
|
||||
for item in result['evidenceCompletion']['missingEvidence']))
|
||||
self.assertFalse(result['visuals'])
|
||||
|
||||
def test_rank_retains_column_filter_month_range_and_execution_proof(self):
|
||||
api=WeightApi()
|
||||
rule={'field':'customer_code','stages':['bright'],'operator':'in','values':['0012']}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
QualityTools(api_factory=lambda:api,evidence_dir=Path(folder)).quality_data({
|
||||
'mode':'rank','releaseId':'release-a','period':{'year':2026,'monthStart':1,'monthEnd':9},
|
||||
'filters':{},'columnFilters':[rule],'dimension':'product','stage':'bright',
|
||||
'metric':'totalWeight','limit':2})
|
||||
doc=json.loads(next(Path(folder).glob('nao-quality-*.json')).read_text())
|
||||
result=doc['result']
|
||||
self.assertEqual(result['filters']['columnFilters'],[rule])
|
||||
self.assertEqual(result['period'],{'year':2026,'monthStart':1,'monthEnd':9})
|
||||
self.assertEqual(result['periodCoverage']['availableMonths'],list(range(1,10)))
|
||||
self.assertTrue(result['periodCoverage']['complete'])
|
||||
self.assertEqual(result['protocolEvidence']['queryId'],'rank-query-1')
|
||||
metrics_paths=[path for _,path,_ in api.calls if path.startswith('/api/metrics')]
|
||||
self.assertEqual(len(metrics_paths),1)
|
||||
self.assertIn('columnFilters=',metrics_paths[0])
|
||||
self.assertNotIn('rowLimit=',metrics_paths[0])
|
||||
self.assertNotIn('rowOffset=',metrics_paths[0])
|
||||
|
||||
def test_rank_only_allows_column_filter_parameter_and_other_modes_reject_it(self):
|
||||
api=WeightApi()
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tools=QualityTools(api_factory=lambda:api,evidence_dir=Path(folder))
|
||||
with self.assertRaisesRegex(QualityToolError,'query/compare/rank'):
|
||||
tools.quality_data({'mode':'changes','releaseId':'release-a','period':{'year':2026},
|
||||
'comparisonPeriod':{'year':2025},'dimension':'group','columnFilters':[]})
|
||||
self.assertEqual(api.calls,[])
|
||||
|
||||
def test_ranking_chart_scope_preserves_exact_continuous_month_range(self):
|
||||
from nao_weight_ranking import ranking_chart
|
||||
doc,_=self.run_rank()
|
||||
doc['result']['period']={'year':2026,'monthStart':3,'monthEnd':5}
|
||||
t=planned();t['period']=doc['result']['period']
|
||||
chart=ranking_chart(doc,'range-ranking-turn',t)
|
||||
self.assertIsNotNone(chart)
|
||||
self.assertEqual({key:chart['scope'][key] for key in ('year','monthStart','monthEnd')},t['period'])
|
||||
|
||||
def test_pareto_keeps_top_rows_but_binds_ratio_to_full_population(self):
|
||||
doc, _ = self.run_rank()
|
||||
task = planned()
|
||||
context = bind_task({'releaseId':'release-a'},task)
|
||||
output = build_visuals([doc],QUESTION,'turn-ranking',context=context)
|
||||
self.assertEqual(output['visualState'],'ready')
|
||||
chart = output['visuals'][0]
|
||||
self.assertEqual(chart['type'],'pareto')
|
||||
self.assertEqual(chart['coverage']['denominator'],150)
|
||||
self.assertEqual(chart['coverage']['shareBasis'],'all_scope_weight')
|
||||
self.assertIn('1—9月',chart['subtitle'])
|
||||
self.assertEqual(len(chart['categories']),2)
|
||||
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
|
||||
output=build_visuals([doc,department],QUESTION,'turn-ranking',context=context)
|
||||
self.assertEqual({v['dimension'] for v in output['visuals']},{'product','department'})
|
||||
from nao_weight_ranking import missing_ranking_visuals
|
||||
self.assertEqual(missing_ranking_visuals(task,output['visuals']),[])
|
||||
self.assertTrue(missing_ranking_visuals(task,output['visuals'][:1]))
|
||||
|
||||
def test_pareto_emits_each_requested_weight_field_for_the_same_stage(self):
|
||||
total_doc,_=self.run_rank()
|
||||
scrap_doc,_=self.run_rank(metric='scrapWeight')
|
||||
task=planned(fields=['totalWeight','scrapWeight'],dimensions=['product'],visualPreference='auto')
|
||||
context=bind_task({'releaseId':'release-a'},task)
|
||||
output=build_visuals([total_doc,scrap_doc],QUESTION,'two-field-ranking',context=context)
|
||||
self.assertEqual(output['visualState'],'ready')
|
||||
self.assertEqual({visual['metric'] for visual in output['visuals']},
|
||||
{'bright_total_weight','bright_scrap_weight'})
|
||||
from nao_weight_ranking import missing_ranking_visuals
|
||||
self.assertEqual(missing_ranking_visuals(task,output['visuals']),[])
|
||||
|
||||
def test_missing_dimension_is_not_completion_and_year_data_is_not_called_absent(self):
|
||||
doc, _ = self.run_rank()
|
||||
task = planned()
|
||||
context = bind_task({'releaseId':'release-a'},task)
|
||||
self.assertFalse(coverage_sufficient([doc],QUESTION,context))
|
||||
department = copy.deepcopy(doc)
|
||||
department['result']['dimension'] = 'department'
|
||||
department['evidenceRef'] = 'Q00000002'
|
||||
self.assertTrue(coverage_sufficient([doc,department],QUESTION,context))
|
||||
complete = planned(periodCoverage='complete')
|
||||
self.assertFalse(coverage_sufficient([doc,department],QUESTION,bind_task({'releaseId':'release-a'},complete)))
|
||||
|
||||
def test_internal_month_holes_and_total_mismatch_do_not_pass_as_complete(self):
|
||||
doc, _ = self.run_rank(hole=True)
|
||||
context = bind_task({'releaseId':'release-a'},planned())
|
||||
self.assertFalse(coverage_sufficient([doc],QUESTION,context))
|
||||
with self.assertRaises(QualityToolError):self.run_rank(mismatch=True)
|
||||
|
||||
def test_enum_strings_are_normalized_without_guessing_business_intent(self):
|
||||
task = normalize_task(task_value(dimensions='product,department',stages='bright',fields='totalWeight'),QUESTION)
|
||||
self.assertEqual(task['dimensions'],['product','department'])
|
||||
self.assertEqual(task['stages'],['bright'])
|
||||
|
||||
def test_each_ranking_dimension_uses_only_its_declared_stage(self):
|
||||
doc, _ = self.run_rank()
|
||||
department = copy.deepcopy(doc)
|
||||
department['result'].update(dimension='department', stage='black')
|
||||
department['evidenceRef'] = 'Q00000002'
|
||||
task = planned()
|
||||
task['stages'] = ['black', 'bright']
|
||||
task['dimensionStages'] = {'product': ['bright'], 'department': ['black']}
|
||||
context = bind_task({'releaseId': 'release-a'}, task)
|
||||
self.assertTrue(coverage_sufficient([doc, department], QUESTION, context))
|
||||
charts = build_visuals([doc, department], QUESTION, 'mixed-stage-ranking', context=context)['visuals']
|
||||
self.assertEqual({(v['dimension'], v['stage']) for v in charts}, {('product', 'bright'), ('department', 'black')})
|
||||
self.assertTrue(all(v['coverage']['denominator'] == 150 for v in charts))
|
||||
from nao_weight_ranking import missing_ranking_visuals
|
||||
self.assertEqual(missing_ranking_visuals(task, charts), [])
|
||||
wrong = copy.deepcopy(department); wrong['result']['stage'] = 'bright'
|
||||
self.assertFalse(coverage_sufficient([doc, wrong], QUESTION, context))
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { defectWeightRate } from '../web/src/trendMetrics.ts';
|
||||
|
||||
test('defect rate uses total stage weight rather than the sum of defect categories', () => {
|
||||
assert.equal(defectWeightRate(60, 3000), 0.02);
|
||||
assert.notEqual(defectWeightRate(60, 3000), 0.6);
|
||||
});
|
||||
test('zero and missing denominators are unavailable, while zero defects is zero', () => {
|
||||
assert.equal(defectWeightRate(0, 1000), 0);
|
||||
for (const total of [0, null, undefined, NaN]) assert.equal(defectWeightRate(60, total), null);
|
||||
assert.equal(defectWeightRate(null, 1000), null);
|
||||
});
|
||||
Reference in New Issue
Block a user