允许 Agent 在循环中查目录、修正查询和提问,保留实时权限检查及模型原始回答。 统一相同查询证据与图表资源;排名补齐阶段指标,修正 OR 重量筛选,并限制模型明细输入。 增加运行恢复、图表自适应坐标轴与两位小数,修复报告主图选择、重复图及边缘标签。 验证:独立副本通过 162 项 Python、73 项前端及 21 项后端测试,前端构建通过。 线上已用原 8 题复测,并追加两轮双期对比;TOP100 与组合筛选分别核对 100 行和 20 行。
352 lines
19 KiB
Python
352 lines
19 KiB
Python
"""Pinned-runtime tests use authored local SSE, never a model or business credential."""
|
|
import json
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import dsh_agent_runtime as runtime
|
|
from nao_quality_runtime import ConversationAccumulator
|
|
from setup_dsh_runtime import inspect_runtime, runtime_node
|
|
|
|
|
|
class AuthoredModel:
|
|
"""A deterministic provider wire fixture which exercises the actual DSH loop."""
|
|
def __init__(self, *, stall=False, length=False, transient=False, malformed=False, repair=False):
|
|
self.calls, self.stall, self.length = [], stall, length
|
|
self.started = threading.Event()
|
|
self.release = threading.Event()
|
|
owner = self
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_args):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
owner.calls.append(body)
|
|
owner.started.set()
|
|
if transient and len(owner.calls) == 1:
|
|
encoded = b'{"error":{"message":"authored temporary failure"}}'
|
|
self.send_response(503)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', str(len(encoded)))
|
|
self.end_headers()
|
|
self.wfile.write(encoded)
|
|
return
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'text/event-stream')
|
|
self.send_header('Connection', 'close')
|
|
self.end_headers()
|
|
|
|
def send(delta, finish=None):
|
|
value = {'id': 'authored-local-only', 'object': 'chat.completion.chunk', 'created': 0,
|
|
'model': 'authored-check', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]}
|
|
self.wfile.write(('data: ' + json.dumps(value, ensure_ascii=False) + '\n\n').encode())
|
|
self.wfile.flush()
|
|
|
|
try:
|
|
send({'role': 'assistant'})
|
|
if owner.stall:
|
|
owner.release.wait(20)
|
|
return
|
|
if not any(message.get('role') == 'tool' for message in body['messages']) or repair and len(owner.calls)<3:
|
|
arguments = ('{"mode":"query","period":{"year":2026},"filters":{"product":"'
|
|
+ ('18X15X4' if len(owner.calls)==1 else '18X15X4+T') + '"}}') if repair else '{"mode":"catalog"}'
|
|
send({'tool_calls': [{'index': 0, 'id': 'authored-quality-query-'+str(len(owner.calls)), 'type': 'function',
|
|
'function': {'name': runtime.PUBLIC_TOOL, 'arguments':
|
|
'{"mode":' if malformed and len(owner.calls) == 1 else arguments}}]})
|
|
send({}, 'tool_calls')
|
|
else:
|
|
send({'content': '已核实'})
|
|
time.sleep(0.4) # SDK notification polling must survive ordinary idle intervals.
|
|
send({'content': '质量指标。'})
|
|
send({}, 'length' if owner.length else 'stop')
|
|
usage = {'id': 'usage', 'choices': [], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'total_tokens': 110}}
|
|
self.wfile.write(('data: ' + json.dumps(usage) + '\n\ndata: [DONE]\n\n').encode())
|
|
self.wfile.flush()
|
|
except (OSError, BrokenPipeError):
|
|
pass
|
|
|
|
class Server(ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
block_on_close = False
|
|
self.server = Server(('127.0.0.1', 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
self.connection = (f'http://127.0.0.1:{self.server.server_port}/v1', 'authored-test-only', 'authored-check')
|
|
|
|
def close(self):
|
|
self.release.set()
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join(timeout=1)
|
|
|
|
|
|
class ProjectionTests(unittest.TestCase):
|
|
def test_failed_attempt_is_removed_before_retried_commit(self):
|
|
events = []
|
|
acc = ConversationAccumulator()
|
|
projection = runtime._Projection(acc, events.append)
|
|
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 1, 'turn': 1, 'step': 1})
|
|
projection.frame({'type': 'chunk', 'attemptId': 'a', 'revision': 2, 'index': 0,
|
|
'chunk': {'type': 'text-delta', 'index': 0, 'text': '失败草稿'}})
|
|
self.assertEqual(acc.answer, '失败草稿')
|
|
projection.event({'seq': 0, 'type': 'assistant/attempt', 'data': {'turn': 1, 'step': 1}})
|
|
projection.frame({'type': 'end', 'attemptId': 'a', 'revision': 3, 'index': 1,
|
|
'outcome': {'kind': 'committed', 'eventType': 'assistant/attempt', 'seq': 0}})
|
|
self.assertEqual(acc.answer, '')
|
|
self.assertFalse(any(entry.get('text') == '失败草稿' for entry in acc.timeline))
|
|
projection.frame({'type': 'start', 'attemptId': 'b', 'revision': 4, 'turn': 1, 'step': 1})
|
|
projection.frame({'type': 'chunk', 'attemptId': 'b', 'revision': 5, 'index': 0,
|
|
'chunk': {'type': 'text-delta', 'index': 0, 'text': '新答复'}})
|
|
projection.event({'seq': 1, 'type': 'assistant/message', 'data': {'turn': 1, 'step': 1,
|
|
'message': {'content': [{'type': 'text', 'text': '已提交答复'}]}}})
|
|
self.assertEqual(acc.answer, '已提交答复')
|
|
self.assertNotIn('失败草稿', json.dumps(acc.timeline, ensure_ascii=False))
|
|
self.assertNotIn('失败草稿', json.dumps(acc.timeline_segments, ensure_ascii=False))
|
|
|
|
def test_unknown_tool_and_stream_gap_fail_closed(self):
|
|
projection = runtime._Projection(ConversationAccumulator(), lambda _value: None)
|
|
with self.assertRaises(ValueError):
|
|
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 2, 'turn': 1, 'step': 1})
|
|
with self.assertRaises(ValueError):
|
|
projection.event({'seq': 0, 'type': 'tool/call', 'data': {'name': 'shell'}})
|
|
|
|
def test_identity_and_versions_are_pinned(self):
|
|
lock = json.loads((ROOT / 'runtime/dsh/vendor/python-sdk/source-lock.json').read_text())
|
|
self.assertEqual(lock['commit'], 'fcc0ad462172522aaf5c93a7b2e01f6b9f0b0159')
|
|
manifest = json.loads((ROOT / 'runtime/dsh/package.json').read_text())
|
|
self.assertEqual(manifest['dependencies']['@deepseek-ai/dsh'], '0.2.2-rc.1')
|
|
|
|
|
|
@unittest.skipUnless(inspect_runtime()['ready'], 'requires installed pinned runtime and macOS sandbox')
|
|
class RealDshTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.folder = tempfile.TemporaryDirectory(prefix='dsh-real-verification-')
|
|
self.addCleanup(self.folder.cleanup)
|
|
self.root = Path(self.folder.name).resolve()
|
|
self.events = []
|
|
self.tool_calls = []
|
|
self.authorizations = 0
|
|
self.cancel = threading.Event()
|
|
|
|
def run_agent(self, model, *, tool=None, deadline=None, authorization=None, initial=None):
|
|
self.addCleanup(model.close)
|
|
from nao_quality_tools import tool_definition
|
|
def authorize():
|
|
self.authorizations += 1
|
|
if authorization is not None:
|
|
authorization()
|
|
def query(arguments):
|
|
self.tool_calls.append(arguments)
|
|
return {'kind': 'quality_catalog', 'evidenceRef': 'QA1B2C3D4', 'message': '正式证据已核实'}
|
|
result = runtime.run_dsh_agent(actor_root=self.root, connection=model.connection,
|
|
question='/ordinary-user-input', context={'_agentTask': {}}, task={}, identity={'executionId': 'test-current'},
|
|
turn_id=str(uuid.uuid4()), cancel_event=self.cancel, emit=self.events.append,
|
|
tool_handler=tool or query, authorize=authorize, tool_definition=tool_definition(),
|
|
deadline=deadline or time.monotonic() + 40, initial_accumulator=initial)
|
|
self.summary = json.loads(next((self.root / 'private/dsh').glob('*/summary.json')).read_text())
|
|
return result
|
|
|
|
def test_real_loop_mcp_stream_commit_and_isolation_lifecycle(self):
|
|
model = AuthoredModel()
|
|
acc, status, reason, stopped = self.run_agent(model)
|
|
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
|
self.assertEqual(acc.answer, '已核实质量指标。')
|
|
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
|
|
self.assertGreaterEqual(self.authorizations, 5)
|
|
self.assertTrue(any(e.get('type') == 'answer' and e.get('answer') == '已核实' for e in self.events))
|
|
self.assertEqual(acc.tool_calls[0]['tool'], 'mcp_call')
|
|
self.assertEqual(acc.tool_calls[0]['sourceTool'], runtime.PUBLIC_TOOL)
|
|
self.assertTrue(acc.final_answer_ready)
|
|
self.assertEqual(len(model.calls), 2)
|
|
for request in model.calls:
|
|
self.assertEqual({t['function']['name'] for t in request.get('tools', [])}, {runtime.PUBLIC_TOOL})
|
|
self.assertNotIn('dsh_session_log', request)
|
|
self.assertEqual(self.summary['stopSucceeded'], True)
|
|
|
|
def test_initial_read_remains_in_the_real_native_event_stream(self):
|
|
seed=ConversationAccumulator()
|
|
seed.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'initial-fixture',
|
|
'input':{'tool':'quality_data','arguments':{'mode':'query'}}})
|
|
seed.feed({'type':'tool-output-available','toolName':'mcp_call','toolCallId':'initial-fixture',
|
|
'output':{'kind':'quality_metrics','summary':{'compositeRate':.9557}}})
|
|
acc,status,reason,stopped=self.run_agent(AuthoredModel(),initial=seed)
|
|
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
|
|
self.assertIs(acc,seed)
|
|
self.assertEqual(acc.answer,'已核实质量指标。')
|
|
self.assertEqual([call['input']['mode'] for call in acc.tool_calls],['query','catalog'])
|
|
self.assertTrue(all(call['status']=='completed' for call in acc.tool_calls))
|
|
self.assertEqual(len({event['sequence'] for event in acc.timeline}),len(acc.timeline))
|
|
|
|
def test_real_cancel_revokes_capability_and_reaps_process_tree(self):
|
|
model = AuthoredModel(stall=True)
|
|
def cancel_after_start():
|
|
model.started.wait(15)
|
|
self.cancel.set()
|
|
thread = threading.Thread(target=cancel_after_start, daemon=True)
|
|
thread.start()
|
|
began = time.monotonic()
|
|
acc, status, reason, stopped = self.run_agent(model)
|
|
self.assertEqual((status, reason, stopped), ('CANCELLED', 'user_cancel', True), self.summary)
|
|
self.assertLess(time.monotonic() - began, 15)
|
|
self.assertEqual(self.summary['toolCalls'], 0)
|
|
|
|
def test_real_output_limit_cannot_be_success(self):
|
|
acc, status, reason, stopped = self.run_agent(AuthoredModel(length=True))
|
|
self.assertIn(status, ('PARTIAL', 'FAILED'))
|
|
self.assertNotEqual(reason, 'stop')
|
|
self.assertTrue(stopped)
|
|
|
|
def test_real_transient_failure_retries_once_without_duplicate_text(self):
|
|
model = AuthoredModel(transient=True)
|
|
acc, status, reason, stopped = self.run_agent(model)
|
|
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
|
self.assertEqual(len(model.calls), 3)
|
|
self.assertEqual(acc.answer, '已核实质量指标。')
|
|
self.assertGreater(self.summary['modelTokensCharged'], 220)
|
|
self.assertEqual(self.summary['modelRequests'], 3)
|
|
|
|
def test_malformed_tool_response_gets_one_native_retry_without_execution_or_rewriting(self):
|
|
model = AuthoredModel(malformed=True)
|
|
acc, status, reason, stopped = self.run_agent(model)
|
|
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
|
self.assertEqual(len(model.calls), 3)
|
|
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
|
|
self.assertEqual(acc.answer, '已核实质量指标。')
|
|
|
|
def test_real_deadline_stops_a_silent_model_independent_of_sdk_wait(self):
|
|
began = time.monotonic()
|
|
acc, status, reason, stopped = self.run_agent(AuthoredModel(stall=True), deadline=began + 3)
|
|
self.assertEqual((status, reason, stopped), ('FAILED', 'timeout', True), self.summary)
|
|
self.assertLess(time.monotonic() - began, 13)
|
|
|
|
def test_authorization_watchdog_stops_and_records_only_safe_failure_metadata(self):
|
|
model = AuthoredModel(stall=True)
|
|
def authorize():
|
|
if model.started.is_set():
|
|
raise RuntimeError('private-authorization-detail')
|
|
acc, status, reason, stopped = self.run_agent(model, authorization=authorize)
|
|
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_unavailable', True))
|
|
self.assertEqual(len(model.calls), 1)
|
|
self.assertEqual(self.summary['authorizationFailure'], {
|
|
'errorType': 'RuntimeError', 'httpStatus': None, 'retryable': False})
|
|
self.assertNotIn('private-authorization-detail', json.dumps(self.summary))
|
|
|
|
def test_authorization_rejection_stops_without_claiming_a_provider_failure(self):
|
|
from nao_quality_runtime import RuntimeErrorSafe
|
|
model = AuthoredModel(stall=True)
|
|
def authorize():
|
|
if model.started.is_set():
|
|
raise RuntimeErrorSafe('private-refusal-detail', http_status=403)
|
|
_, status, reason, stopped = self.run_agent(model, authorization=authorize)
|
|
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_rejected', True))
|
|
self.assertEqual(self.summary['authorizationFailure']['httpStatus'], 403)
|
|
self.assertNotIn('private-refusal-detail', json.dumps(self.summary))
|
|
|
|
def test_handled_tool_error_does_not_overturn_native_completion(self):
|
|
from nao_quality_tools import QualityToolError
|
|
def reject(_arguments):
|
|
raise QualityToolError('当前范围未授权。')
|
|
acc, status, reason, stopped = self.run_agent(AuthoredModel(), tool=reject)
|
|
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
|
self.assertTrue(acc.tool_error)
|
|
self.assertEqual(acc.tool_calls[0]['status'], 'failed')
|
|
self.assertIn('当前范围未授权', acc.tool_calls[0]['outputSummary'])
|
|
|
|
def test_real_loop_corrects_a_rejected_product_argument_and_continues(self):
|
|
from nao_quality_tools import QualityToolError
|
|
seen=[]
|
|
def query(arguments):
|
|
seen.append(arguments['filters']['product'])
|
|
if seen[-1]=='18X15X4':
|
|
raise QualityToolError('名称未匹配,请使用完整规格18X15X4+T。',code='invalid_arguments')
|
|
return {'kind':'quality_metrics','message':'完整规格查询成功'}
|
|
acc,status,reason,stopped=self.run_agent(AuthoredModel(repair=True),tool=query)
|
|
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
|
|
self.assertEqual(seen,['18X15X4','18X15X4+T'])
|
|
self.assertEqual([call['status'] for call in acc.tool_calls],['failed','completed'])
|
|
self.assertEqual(acc.answer,'已核实质量指标。')
|
|
|
|
def test_no_data_is_a_normal_tool_outcome_without_zero_metrics(self):
|
|
acc,status,reason,stopped=self.run_agent(AuthoredModel(),tool=lambda args:{
|
|
'kind':'quality_no_data','status':'no_data','summary':None,'rows':[],
|
|
'message':'当前期间没有已发布来源数据。'})
|
|
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True))
|
|
self.assertFalse(acc.tool_error)
|
|
self.assertEqual(acc.tool_calls[0]['status'],'completed')
|
|
self.assertIn('暂无数据',acc.tool_calls[0]['outputSummary'])
|
|
|
|
def test_process_cleanup_refuses_a_changed_process_identity(self):
|
|
marker = uuid.uuid4().hex
|
|
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
|
|
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
path = self.root / 'process.json'
|
|
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': 'wrong'}))
|
|
self.assertFalse(runtime._stop_registered(path))
|
|
self.assertIsNone(process.poll())
|
|
finally:
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
|
|
def test_stop_all_reaps_only_a_matching_persisted_orphan_group(self):
|
|
marker = uuid.uuid4().hex
|
|
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
|
|
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
root = self.root / 'users'
|
|
path = root / 'actor/sessions/session/private/dsh' / marker / 'process.json'
|
|
path.parent.mkdir(parents=True)
|
|
fingerprint = runtime._process_fingerprint(process.pid)
|
|
self.assertIn(marker, fingerprint)
|
|
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker,
|
|
'fingerprint': hashlib.sha256(fingerprint.encode()).hexdigest()}))
|
|
result = runtime.stop_all_dsh(users_root=root)
|
|
self.assertEqual(result, {'stopped': 1, 'unconfirmed': 0})
|
|
process.wait(timeout=3)
|
|
self.assertEqual(json.loads(path.read_text())['state'], 'reaped')
|
|
finally:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
|
|
def test_os_sandbox_denies_business_files_other_ports_and_shell(self):
|
|
# Probe the actual generated OS policy, independent of model tool hiding.
|
|
secret = self.root / 'business-secret.txt'
|
|
secret.write_text('verification sentinel')
|
|
run = self.root / 'isolated'
|
|
run.mkdir()
|
|
for child in ('home', 'workspace', 'tmp'):
|
|
(run / child).mkdir()
|
|
policy = runtime._sandbox(run, 9)
|
|
code = '''
|
|
const fs=require('node:fs'); const cp=require('node:child_process');
|
|
let read=false,write=false;try{fs.readFileSync(process.argv[1]);read=true}catch{}
|
|
try{fs.writeFileSync(process.argv[1], 'bad');write=true}catch{}
|
|
const spawned=cp.spawnSync('/bin/sh',['-c','exit 0']);
|
|
let network=false;try{await fetch('http://127.0.0.1:10',{signal:AbortSignal.timeout(1000)});network=true}catch{}
|
|
console.log(JSON.stringify({read,write,shell:!spawned.error,network}));
|
|
'''
|
|
result = subprocess.run(['/usr/bin/sandbox-exec', '-f', str(policy), str(runtime_node()),
|
|
'--input-type=commonjs', '-e', '(async()=>{' + code + '})()', str(secret)],
|
|
capture_output=True, text=True, timeout=10)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(json.loads(result.stdout), {'read': False, 'write': False, 'shell': False, 'network': False})
|
|
self.assertEqual(secret.read_text(), 'verification sentinel')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|