Files
quality_analyse/verification/test_dsh_agent_runtime.py
lyun bc19475b29 修复:DSH 自主查询、图片直显与报告稳定性
允许 Agent 在循环中查目录、修正查询和提问,保留实时权限检查及模型原始回答。
统一相同查询证据与图表资源;排名补齐阶段指标,修正 OR 重量筛选,并限制模型明细输入。
增加运行恢复、图表自适应坐标轴与两位小数,修复报告主图选择、重复图及边缘标签。

验证:独立副本通过 162 项 Python、73 项前端及 21 项后端测试,前端构建通过。
线上已用原 8 题复测,并追加两轮双期对比;TOP100 与组合筛选分别核对 100 行和 20 行。
2026-11-12 16:14:20 +08:00

352 lines
19 KiB
Python

"""Pinned-runtime tests use authored local SSE, never a model or business credential."""
import json
import hashlib
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
import dsh_agent_runtime as runtime
from nao_quality_runtime import ConversationAccumulator
from setup_dsh_runtime import inspect_runtime, runtime_node
class AuthoredModel:
"""A deterministic provider wire fixture which exercises the actual DSH loop."""
def __init__(self, *, stall=False, length=False, transient=False, malformed=False, repair=False):
self.calls, self.stall, self.length = [], stall, length
self.started = threading.Event()
self.release = threading.Event()
owner = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_args):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
owner.calls.append(body)
owner.started.set()
if transient and len(owner.calls) == 1:
encoded = b'{"error":{"message":"authored temporary failure"}}'
self.send_response(503)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
return
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.send_header('Connection', 'close')
self.end_headers()
def send(delta, finish=None):
value = {'id': 'authored-local-only', 'object': 'chat.completion.chunk', 'created': 0,
'model': 'authored-check', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]}
self.wfile.write(('data: ' + json.dumps(value, ensure_ascii=False) + '\n\n').encode())
self.wfile.flush()
try:
send({'role': 'assistant'})
if owner.stall:
owner.release.wait(20)
return
if not any(message.get('role') == 'tool' for message in body['messages']) or repair and len(owner.calls)<3:
arguments = ('{"mode":"query","period":{"year":2026},"filters":{"product":"'
+ ('18X15X4' if len(owner.calls)==1 else '18X15X4+T') + '"}}') if repair else '{"mode":"catalog"}'
send({'tool_calls': [{'index': 0, 'id': 'authored-quality-query-'+str(len(owner.calls)), 'type': 'function',
'function': {'name': runtime.PUBLIC_TOOL, 'arguments':
'{"mode":' if malformed and len(owner.calls) == 1 else arguments}}]})
send({}, 'tool_calls')
else:
send({'content': '已核实'})
time.sleep(0.4) # SDK notification polling must survive ordinary idle intervals.
send({'content': '质量指标。'})
send({}, 'length' if owner.length else 'stop')
usage = {'id': 'usage', 'choices': [], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'total_tokens': 110}}
self.wfile.write(('data: ' + json.dumps(usage) + '\n\ndata: [DONE]\n\n').encode())
self.wfile.flush()
except (OSError, BrokenPipeError):
pass
class Server(ThreadingHTTPServer):
daemon_threads = True
block_on_close = False
self.server = Server(('127.0.0.1', 0), Handler)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.connection = (f'http://127.0.0.1:{self.server.server_port}/v1', 'authored-test-only', 'authored-check')
def close(self):
self.release.set()
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=1)
class ProjectionTests(unittest.TestCase):
def test_failed_attempt_is_removed_before_retried_commit(self):
events = []
acc = ConversationAccumulator()
projection = runtime._Projection(acc, events.append)
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 1, 'turn': 1, 'step': 1})
projection.frame({'type': 'chunk', 'attemptId': 'a', 'revision': 2, 'index': 0,
'chunk': {'type': 'text-delta', 'index': 0, 'text': '失败草稿'}})
self.assertEqual(acc.answer, '失败草稿')
projection.event({'seq': 0, 'type': 'assistant/attempt', 'data': {'turn': 1, 'step': 1}})
projection.frame({'type': 'end', 'attemptId': 'a', 'revision': 3, 'index': 1,
'outcome': {'kind': 'committed', 'eventType': 'assistant/attempt', 'seq': 0}})
self.assertEqual(acc.answer, '')
self.assertFalse(any(entry.get('text') == '失败草稿' for entry in acc.timeline))
projection.frame({'type': 'start', 'attemptId': 'b', 'revision': 4, 'turn': 1, 'step': 1})
projection.frame({'type': 'chunk', 'attemptId': 'b', 'revision': 5, 'index': 0,
'chunk': {'type': 'text-delta', 'index': 0, 'text': '新答复'}})
projection.event({'seq': 1, 'type': 'assistant/message', 'data': {'turn': 1, 'step': 1,
'message': {'content': [{'type': 'text', 'text': '已提交答复'}]}}})
self.assertEqual(acc.answer, '已提交答复')
self.assertNotIn('失败草稿', json.dumps(acc.timeline, ensure_ascii=False))
self.assertNotIn('失败草稿', json.dumps(acc.timeline_segments, ensure_ascii=False))
def test_unknown_tool_and_stream_gap_fail_closed(self):
projection = runtime._Projection(ConversationAccumulator(), lambda _value: None)
with self.assertRaises(ValueError):
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 2, 'turn': 1, 'step': 1})
with self.assertRaises(ValueError):
projection.event({'seq': 0, 'type': 'tool/call', 'data': {'name': 'shell'}})
def test_identity_and_versions_are_pinned(self):
lock = json.loads((ROOT / 'runtime/dsh/vendor/python-sdk/source-lock.json').read_text())
self.assertEqual(lock['commit'], 'fcc0ad462172522aaf5c93a7b2e01f6b9f0b0159')
manifest = json.loads((ROOT / 'runtime/dsh/package.json').read_text())
self.assertEqual(manifest['dependencies']['@deepseek-ai/dsh'], '0.2.2-rc.1')
@unittest.skipUnless(inspect_runtime()['ready'], 'requires installed pinned runtime and macOS sandbox')
class RealDshTests(unittest.TestCase):
def setUp(self):
self.folder = tempfile.TemporaryDirectory(prefix='dsh-real-verification-')
self.addCleanup(self.folder.cleanup)
self.root = Path(self.folder.name).resolve()
self.events = []
self.tool_calls = []
self.authorizations = 0
self.cancel = threading.Event()
def run_agent(self, model, *, tool=None, deadline=None, authorization=None, initial=None):
self.addCleanup(model.close)
from nao_quality_tools import tool_definition
def authorize():
self.authorizations += 1
if authorization is not None:
authorization()
def query(arguments):
self.tool_calls.append(arguments)
return {'kind': 'quality_catalog', 'evidenceRef': 'QA1B2C3D4', 'message': '正式证据已核实'}
result = runtime.run_dsh_agent(actor_root=self.root, connection=model.connection,
question='/ordinary-user-input', context={'_agentTask': {}}, task={}, identity={'executionId': 'test-current'},
turn_id=str(uuid.uuid4()), cancel_event=self.cancel, emit=self.events.append,
tool_handler=tool or query, authorize=authorize, tool_definition=tool_definition(),
deadline=deadline or time.monotonic() + 40, initial_accumulator=initial)
self.summary = json.loads(next((self.root / 'private/dsh').glob('*/summary.json')).read_text())
return result
def test_real_loop_mcp_stream_commit_and_isolation_lifecycle(self):
model = AuthoredModel()
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(acc.answer, '已核实质量指标。')
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
self.assertGreaterEqual(self.authorizations, 5)
self.assertTrue(any(e.get('type') == 'answer' and e.get('answer') == '已核实' for e in self.events))
self.assertEqual(acc.tool_calls[0]['tool'], 'mcp_call')
self.assertEqual(acc.tool_calls[0]['sourceTool'], runtime.PUBLIC_TOOL)
self.assertTrue(acc.final_answer_ready)
self.assertEqual(len(model.calls), 2)
for request in model.calls:
self.assertEqual({t['function']['name'] for t in request.get('tools', [])}, {runtime.PUBLIC_TOOL})
self.assertNotIn('dsh_session_log', request)
self.assertEqual(self.summary['stopSucceeded'], True)
def test_initial_read_remains_in_the_real_native_event_stream(self):
seed=ConversationAccumulator()
seed.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'initial-fixture',
'input':{'tool':'quality_data','arguments':{'mode':'query'}}})
seed.feed({'type':'tool-output-available','toolName':'mcp_call','toolCallId':'initial-fixture',
'output':{'kind':'quality_metrics','summary':{'compositeRate':.9557}}})
acc,status,reason,stopped=self.run_agent(AuthoredModel(),initial=seed)
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
self.assertIs(acc,seed)
self.assertEqual(acc.answer,'已核实质量指标。')
self.assertEqual([call['input']['mode'] for call in acc.tool_calls],['query','catalog'])
self.assertTrue(all(call['status']=='completed' for call in acc.tool_calls))
self.assertEqual(len({event['sequence'] for event in acc.timeline}),len(acc.timeline))
def test_real_cancel_revokes_capability_and_reaps_process_tree(self):
model = AuthoredModel(stall=True)
def cancel_after_start():
model.started.wait(15)
self.cancel.set()
thread = threading.Thread(target=cancel_after_start, daemon=True)
thread.start()
began = time.monotonic()
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('CANCELLED', 'user_cancel', True), self.summary)
self.assertLess(time.monotonic() - began, 15)
self.assertEqual(self.summary['toolCalls'], 0)
def test_real_output_limit_cannot_be_success(self):
acc, status, reason, stopped = self.run_agent(AuthoredModel(length=True))
self.assertIn(status, ('PARTIAL', 'FAILED'))
self.assertNotEqual(reason, 'stop')
self.assertTrue(stopped)
def test_real_transient_failure_retries_once_without_duplicate_text(self):
model = AuthoredModel(transient=True)
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(len(model.calls), 3)
self.assertEqual(acc.answer, '已核实质量指标。')
self.assertGreater(self.summary['modelTokensCharged'], 220)
self.assertEqual(self.summary['modelRequests'], 3)
def test_malformed_tool_response_gets_one_native_retry_without_execution_or_rewriting(self):
model = AuthoredModel(malformed=True)
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(len(model.calls), 3)
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
self.assertEqual(acc.answer, '已核实质量指标。')
def test_real_deadline_stops_a_silent_model_independent_of_sdk_wait(self):
began = time.monotonic()
acc, status, reason, stopped = self.run_agent(AuthoredModel(stall=True), deadline=began + 3)
self.assertEqual((status, reason, stopped), ('FAILED', 'timeout', True), self.summary)
self.assertLess(time.monotonic() - began, 13)
def test_authorization_watchdog_stops_and_records_only_safe_failure_metadata(self):
model = AuthoredModel(stall=True)
def authorize():
if model.started.is_set():
raise RuntimeError('private-authorization-detail')
acc, status, reason, stopped = self.run_agent(model, authorization=authorize)
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_unavailable', True))
self.assertEqual(len(model.calls), 1)
self.assertEqual(self.summary['authorizationFailure'], {
'errorType': 'RuntimeError', 'httpStatus': None, 'retryable': False})
self.assertNotIn('private-authorization-detail', json.dumps(self.summary))
def test_authorization_rejection_stops_without_claiming_a_provider_failure(self):
from nao_quality_runtime import RuntimeErrorSafe
model = AuthoredModel(stall=True)
def authorize():
if model.started.is_set():
raise RuntimeErrorSafe('private-refusal-detail', http_status=403)
_, status, reason, stopped = self.run_agent(model, authorization=authorize)
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_rejected', True))
self.assertEqual(self.summary['authorizationFailure']['httpStatus'], 403)
self.assertNotIn('private-refusal-detail', json.dumps(self.summary))
def test_handled_tool_error_does_not_overturn_native_completion(self):
from nao_quality_tools import QualityToolError
def reject(_arguments):
raise QualityToolError('当前范围未授权。')
acc, status, reason, stopped = self.run_agent(AuthoredModel(), tool=reject)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertTrue(acc.tool_error)
self.assertEqual(acc.tool_calls[0]['status'], 'failed')
self.assertIn('当前范围未授权', acc.tool_calls[0]['outputSummary'])
def test_real_loop_corrects_a_rejected_product_argument_and_continues(self):
from nao_quality_tools import QualityToolError
seen=[]
def query(arguments):
seen.append(arguments['filters']['product'])
if seen[-1]=='18X15X4':
raise QualityToolError('名称未匹配,请使用完整规格18X15X4+T。',code='invalid_arguments')
return {'kind':'quality_metrics','message':'完整规格查询成功'}
acc,status,reason,stopped=self.run_agent(AuthoredModel(repair=True),tool=query)
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
self.assertEqual(seen,['18X15X4','18X15X4+T'])
self.assertEqual([call['status'] for call in acc.tool_calls],['failed','completed'])
self.assertEqual(acc.answer,'已核实质量指标。')
def test_no_data_is_a_normal_tool_outcome_without_zero_metrics(self):
acc,status,reason,stopped=self.run_agent(AuthoredModel(),tool=lambda args:{
'kind':'quality_no_data','status':'no_data','summary':None,'rows':[],
'message':'当前期间没有已发布来源数据。'})
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True))
self.assertFalse(acc.tool_error)
self.assertEqual(acc.tool_calls[0]['status'],'completed')
self.assertIn('暂无数据',acc.tool_calls[0]['outputSummary'])
def test_process_cleanup_refuses_a_changed_process_identity(self):
marker = uuid.uuid4().hex
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
path = self.root / 'process.json'
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': 'wrong'}))
self.assertFalse(runtime._stop_registered(path))
self.assertIsNone(process.poll())
finally:
process.kill()
process.wait(timeout=3)
def test_stop_all_reaps_only_a_matching_persisted_orphan_group(self):
marker = uuid.uuid4().hex
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
root = self.root / 'users'
path = root / 'actor/sessions/session/private/dsh' / marker / 'process.json'
path.parent.mkdir(parents=True)
fingerprint = runtime._process_fingerprint(process.pid)
self.assertIn(marker, fingerprint)
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker,
'fingerprint': hashlib.sha256(fingerprint.encode()).hexdigest()}))
result = runtime.stop_all_dsh(users_root=root)
self.assertEqual(result, {'stopped': 1, 'unconfirmed': 0})
process.wait(timeout=3)
self.assertEqual(json.loads(path.read_text())['state'], 'reaped')
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=3)
def test_os_sandbox_denies_business_files_other_ports_and_shell(self):
# Probe the actual generated OS policy, independent of model tool hiding.
secret = self.root / 'business-secret.txt'
secret.write_text('verification sentinel')
run = self.root / 'isolated'
run.mkdir()
for child in ('home', 'workspace', 'tmp'):
(run / child).mkdir()
policy = runtime._sandbox(run, 9)
code = '''
const fs=require('node:fs'); const cp=require('node:child_process');
let read=false,write=false;try{fs.readFileSync(process.argv[1]);read=true}catch{}
try{fs.writeFileSync(process.argv[1], 'bad');write=true}catch{}
const spawned=cp.spawnSync('/bin/sh',['-c','exit 0']);
let network=false;try{await fetch('http://127.0.0.1:10',{signal:AbortSignal.timeout(1000)});network=true}catch{}
console.log(JSON.stringify({read,write,shell:!spawned.error,network}));
'''
result = subprocess.run(['/usr/bin/sandbox-exec', '-f', str(policy), str(runtime_node()),
'--input-type=commonjs', '-e', '(async()=>{' + code + '})()', str(secret)],
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(json.loads(result.stdout), {'read': False, 'write': False, 'shell': False, 'network': False})
self.assertEqual(secret.read_text(), 'verification sentinel')
if __name__ == '__main__':
unittest.main()