"""Pinned-runtime tests use authored local SSE, never a model or business credential.""" import json import hashlib import os from pathlib import Path import subprocess import sys import tempfile import threading import time import unittest import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'scripts')) import dsh_agent_runtime as runtime from nao_quality_runtime import ConversationAccumulator from setup_dsh_runtime import inspect_runtime, runtime_node class AuthoredModel: """A deterministic provider wire fixture which exercises the actual DSH loop.""" def __init__(self, *, stall=False, length=False, transient=False, malformed=False): self.calls, self.stall, self.length = [], stall, length self.started = threading.Event() self.release = threading.Event() owner = self class Handler(BaseHTTPRequestHandler): def log_message(self, *_args): pass def do_POST(self): body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) owner.calls.append(body) owner.started.set() if transient and len(owner.calls) == 1: encoded = b'{"error":{"message":"authored temporary failure"}}' self.send_response(503) self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(encoded))) self.end_headers() self.wfile.write(encoded) return self.send_response(200) self.send_header('Content-Type', 'text/event-stream') self.send_header('Connection', 'close') self.end_headers() def send(delta, finish=None): value = {'id': 'authored-local-only', 'object': 'chat.completion.chunk', 'created': 0, 'model': 'authored-check', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]} self.wfile.write(('data: ' + json.dumps(value, ensure_ascii=False) + '\n\n').encode()) self.wfile.flush() try: send({'role': 'assistant'}) if owner.stall: owner.release.wait(20) return if not any(message.get('role') == 'tool' for message in body['messages']): send({'tool_calls': [{'index': 0, 'id': 'authored-quality-query', 'type': 'function', 'function': {'name': runtime.PUBLIC_TOOL, 'arguments': '{"mode":' if malformed and len(owner.calls) == 1 else '{"mode":"catalog"}'}}]}) send({}, 'tool_calls') else: send({'content': '已核实'}) time.sleep(0.4) # SDK notification polling must survive ordinary idle intervals. send({'content': '质量指标。'}) send({}, 'length' if owner.length else 'stop') usage = {'id': 'usage', 'choices': [], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'total_tokens': 110}} self.wfile.write(('data: ' + json.dumps(usage) + '\n\ndata: [DONE]\n\n').encode()) self.wfile.flush() except (OSError, BrokenPipeError): pass class Server(ThreadingHTTPServer): daemon_threads = True block_on_close = False self.server = Server(('127.0.0.1', 0), Handler) self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) self.thread.start() self.connection = (f'http://127.0.0.1:{self.server.server_port}/v1', 'authored-test-only', 'authored-check') def close(self): self.release.set() self.server.shutdown() self.server.server_close() self.thread.join(timeout=1) class ProjectionTests(unittest.TestCase): def test_failed_attempt_is_removed_before_retried_commit(self): events = [] acc = ConversationAccumulator() projection = runtime._Projection(acc, events.append) projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 1, 'turn': 1, 'step': 1}) projection.frame({'type': 'chunk', 'attemptId': 'a', 'revision': 2, 'index': 0, 'chunk': {'type': 'text-delta', 'index': 0, 'text': '失败草稿'}}) self.assertEqual(acc.answer, '失败草稿') projection.event({'seq': 0, 'type': 'assistant/attempt', 'data': {'turn': 1, 'step': 1}}) projection.frame({'type': 'end', 'attemptId': 'a', 'revision': 3, 'index': 1, 'outcome': {'kind': 'committed', 'eventType': 'assistant/attempt', 'seq': 0}}) self.assertEqual(acc.answer, '') self.assertFalse(any(entry.get('text') == '失败草稿' for entry in acc.timeline)) projection.frame({'type': 'start', 'attemptId': 'b', 'revision': 4, 'turn': 1, 'step': 1}) projection.frame({'type': 'chunk', 'attemptId': 'b', 'revision': 5, 'index': 0, 'chunk': {'type': 'text-delta', 'index': 0, 'text': '新答复'}}) projection.event({'seq': 1, 'type': 'assistant/message', 'data': {'turn': 1, 'step': 1, 'message': {'content': [{'type': 'text', 'text': '已提交答复'}]}}}) self.assertEqual(acc.answer, '已提交答复') self.assertNotIn('失败草稿', json.dumps(acc.timeline, ensure_ascii=False)) self.assertNotIn('失败草稿', json.dumps(acc.timeline_segments, ensure_ascii=False)) def test_unknown_tool_and_stream_gap_fail_closed(self): projection = runtime._Projection(ConversationAccumulator(), lambda _value: None) with self.assertRaises(ValueError): projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 2, 'turn': 1, 'step': 1}) with self.assertRaises(ValueError): projection.event({'seq': 0, 'type': 'tool/call', 'data': {'name': 'shell'}}) def test_identity_and_versions_are_pinned(self): lock = json.loads((ROOT / 'runtime/dsh/vendor/python-sdk/source-lock.json').read_text()) self.assertEqual(lock['commit'], 'fcc0ad462172522aaf5c93a7b2e01f6b9f0b0159') manifest = json.loads((ROOT / 'runtime/dsh/package.json').read_text()) self.assertEqual(manifest['dependencies']['@deepseek-ai/dsh'], '0.2.2-rc.1') @unittest.skipUnless(inspect_runtime()['ready'], 'requires installed pinned runtime and macOS sandbox') class RealDshTests(unittest.TestCase): def setUp(self): self.folder = tempfile.TemporaryDirectory(prefix='dsh-real-verification-') self.addCleanup(self.folder.cleanup) self.root = Path(self.folder.name).resolve() self.events = [] self.tool_calls = [] self.authorizations = 0 self.cancel = threading.Event() def run_agent(self, model, *, tool=None, deadline=None, authorization=None, initial=None): self.addCleanup(model.close) from nao_quality_tools import tool_definition def authorize(): self.authorizations += 1 if authorization is not None: authorization() def query(arguments): self.tool_calls.append(arguments) return {'kind': 'quality_catalog', 'evidenceRef': 'QA1B2C3D4', 'message': '正式证据已核实'} result = runtime.run_dsh_agent(actor_root=self.root, connection=model.connection, question='/ordinary-user-input', context={'_agentTask': {}}, task={}, identity={'executionId': 'test-current'}, turn_id=str(uuid.uuid4()), cancel_event=self.cancel, emit=self.events.append, tool_handler=tool or query, authorize=authorize, tool_definition=tool_definition(), deadline=deadline or time.monotonic() + 40, initial_accumulator=initial) self.summary = json.loads(next((self.root / 'private/dsh').glob('*/summary.json')).read_text()) return result def test_real_loop_mcp_stream_commit_and_isolation_lifecycle(self): model = AuthoredModel() acc, status, reason, stopped = self.run_agent(model) self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary) self.assertEqual(acc.answer, '已核实质量指标。') self.assertEqual(self.tool_calls, [{'mode': 'catalog'}]) self.assertGreaterEqual(self.authorizations, 5) self.assertTrue(any(e.get('type') == 'answer' and e.get('answer') == '已核实' for e in self.events)) self.assertEqual(acc.tool_calls[0]['tool'], 'mcp_call') self.assertEqual(acc.tool_calls[0]['sourceTool'], runtime.PUBLIC_TOOL) self.assertTrue(acc.final_answer_ready) self.assertEqual(len(model.calls), 2) for request in model.calls: self.assertEqual({t['function']['name'] for t in request.get('tools', [])}, {runtime.PUBLIC_TOOL}) self.assertNotIn('dsh_session_log', request) self.assertEqual(self.summary['stopSucceeded'], True) def test_initial_read_remains_in_the_real_native_event_stream(self): seed=ConversationAccumulator() seed.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'initial-fixture', 'input':{'tool':'quality_data','arguments':{'mode':'query'}}}) seed.feed({'type':'tool-output-available','toolName':'mcp_call','toolCallId':'initial-fixture', 'output':{'kind':'quality_metrics','summary':{'compositeRate':.9557}}}) acc,status,reason,stopped=self.run_agent(AuthoredModel(),initial=seed) self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary) self.assertIs(acc,seed) self.assertEqual(acc.answer,'已核实质量指标。') self.assertEqual([call['input']['mode'] for call in acc.tool_calls],['query','catalog']) self.assertTrue(all(call['status']=='completed' for call in acc.tool_calls)) self.assertEqual(len({event['sequence'] for event in acc.timeline}),len(acc.timeline)) def test_real_cancel_revokes_capability_and_reaps_process_tree(self): model = AuthoredModel(stall=True) def cancel_after_start(): model.started.wait(15) self.cancel.set() thread = threading.Thread(target=cancel_after_start, daemon=True) thread.start() began = time.monotonic() acc, status, reason, stopped = self.run_agent(model) self.assertEqual((status, reason, stopped), ('CANCELLED', 'user_cancel', True), self.summary) self.assertLess(time.monotonic() - began, 15) self.assertEqual(self.summary['toolCalls'], 0) def test_real_output_limit_cannot_be_success(self): acc, status, reason, stopped = self.run_agent(AuthoredModel(length=True)) self.assertIn(status, ('PARTIAL', 'FAILED')) self.assertNotEqual(reason, 'stop') self.assertTrue(stopped) def test_real_transient_failure_retries_once_without_duplicate_text(self): model = AuthoredModel(transient=True) acc, status, reason, stopped = self.run_agent(model) self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary) self.assertEqual(len(model.calls), 3) self.assertEqual(acc.answer, '已核实质量指标。') self.assertGreater(self.summary['modelTokensCharged'], 220) self.assertEqual(self.summary['modelRequests'], 3) def test_malformed_tool_response_gets_one_native_retry_without_execution_or_rewriting(self): model = AuthoredModel(malformed=True) acc, status, reason, stopped = self.run_agent(model) self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary) self.assertEqual(len(model.calls), 3) self.assertEqual(self.tool_calls, [{'mode': 'catalog'}]) self.assertEqual(acc.answer, '已核实质量指标。') def test_real_deadline_stops_a_silent_model_independent_of_sdk_wait(self): began = time.monotonic() acc, status, reason, stopped = self.run_agent(AuthoredModel(stall=True), deadline=began + 3) self.assertEqual((status, reason, stopped), ('FAILED', 'timeout', True), self.summary) self.assertLess(time.monotonic() - began, 13) def test_authorization_watchdog_stops_and_records_only_safe_failure_metadata(self): model = AuthoredModel(stall=True) def authorize(): if model.started.is_set(): raise RuntimeError('private-authorization-detail') acc, status, reason, stopped = self.run_agent(model, authorization=authorize) self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_unavailable', True)) self.assertEqual(len(model.calls), 1) self.assertEqual(self.summary['authorizationFailure'], { 'errorType': 'RuntimeError', 'httpStatus': None, 'retryable': False}) self.assertNotIn('private-authorization-detail', json.dumps(self.summary)) def test_authorization_rejection_stops_without_claiming_a_provider_failure(self): from nao_quality_runtime import RuntimeErrorSafe model = AuthoredModel(stall=True) def authorize(): if model.started.is_set(): raise RuntimeErrorSafe('private-refusal-detail', http_status=403) _, status, reason, stopped = self.run_agent(model, authorization=authorize) self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_rejected', True)) self.assertEqual(self.summary['authorizationFailure']['httpStatus'], 403) self.assertNotIn('private-refusal-detail', json.dumps(self.summary)) def test_handled_tool_error_does_not_overturn_native_completion(self): from nao_quality_tools import QualityToolError def reject(_arguments): raise QualityToolError('当前范围未授权。') acc, status, reason, stopped = self.run_agent(AuthoredModel(), tool=reject) self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary) self.assertTrue(acc.tool_error) self.assertEqual(acc.tool_calls[0]['status'], 'failed') self.assertIn('当前范围未授权', acc.tool_calls[0]['outputSummary']) def test_no_data_is_a_normal_tool_outcome_without_zero_metrics(self): acc,status,reason,stopped=self.run_agent(AuthoredModel(),tool=lambda args:{ 'kind':'quality_no_data','status':'no_data','summary':None,'rows':[], 'message':'当前期间没有已发布来源数据。'}) self.assertEqual((status,reason,stopped),('COMPLETED','stop',True)) self.assertFalse(acc.tool_error) self.assertEqual(acc.tool_calls[0]['status'],'completed') self.assertIn('暂无数据',acc.tool_calls[0]['outputSummary']) def test_process_cleanup_refuses_a_changed_process_identity(self): marker = uuid.uuid4().hex process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker], start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: path = self.root / 'process.json' path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': 'wrong'})) self.assertFalse(runtime._stop_registered(path)) self.assertIsNone(process.poll()) finally: process.kill() process.wait(timeout=3) def test_stop_all_reaps_only_a_matching_persisted_orphan_group(self): marker = uuid.uuid4().hex process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker], start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: root = self.root / 'users' path = root / 'actor/sessions/session/private/dsh' / marker / 'process.json' path.parent.mkdir(parents=True) fingerprint = runtime._process_fingerprint(process.pid) self.assertIn(marker, fingerprint) path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': hashlib.sha256(fingerprint.encode()).hexdigest()})) result = runtime.stop_all_dsh(users_root=root) self.assertEqual(result, {'stopped': 1, 'unconfirmed': 0}) process.wait(timeout=3) self.assertEqual(json.loads(path.read_text())['state'], 'reaped') finally: if process.poll() is None: process.kill() process.wait(timeout=3) def test_os_sandbox_denies_business_files_other_ports_and_shell(self): # Probe the actual generated OS policy, independent of model tool hiding. secret = self.root / 'business-secret.txt' secret.write_text('verification sentinel') run = self.root / 'isolated' run.mkdir() for child in ('home', 'workspace', 'tmp'): (run / child).mkdir() policy = runtime._sandbox(run, 9) code = ''' const fs=require('node:fs'); const cp=require('node:child_process'); let read=false,write=false;try{fs.readFileSync(process.argv[1]);read=true}catch{} try{fs.writeFileSync(process.argv[1], 'bad');write=true}catch{} const spawned=cp.spawnSync('/bin/sh',['-c','exit 0']); let network=false;try{await fetch('http://127.0.0.1:10',{signal:AbortSignal.timeout(1000)});network=true}catch{} console.log(JSON.stringify({read,write,shell:!spawned.error,network})); ''' result = subprocess.run(['/usr/bin/sandbox-exec', '-f', str(policy), str(runtime_node()), '--input-type=commonjs', '-e', '(async()=>{' + code + '})()', str(secret)], capture_output=True, text=True, timeout=10) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(json.loads(result.stdout), {'read': False, 'write': False, 'shell': False, 'network': False}) self.assertEqual(secret.read_text(), 'verification sentinel') if __name__ == '__main__': unittest.main()