"""Authored SSE exercises the real observer and installed SDK without network I/O.""" import json from pathlib import Path import subprocess import sys import unittest ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'scripts')) from nao_thinking_stream import ProviderStream from nao_tool_stream import ToolStreamNormalizer, ToolStreamProtocolError TOOL = 'mcp__quality__quality_data' def frame(*calls, choice=0, finish=None, **delta): if calls: delta['tool_calls'] = list(calls) return {'choices': [{'index': choice, 'delta': delta, 'finish_reason': finish}]} def call(index, identity=None, arguments=None, name=None): value = {'index': index} if identity is not None: value.update(id=identity, type='function') function = {} if name is not None: function['name'] = name if arguments is not None: function['arguments'] = arguments if function: value['function'] = function return value def wire(rows, *, done=True, newline=b'\n'): result = b''.join(b'data: ' + json.dumps(row, ensure_ascii=False).encode() + newline * 2 for row in rows) return result + (b'data: [DONE]' + newline * 2 if done else b'') def collision_rows(): return [frame(call(0, 'a', '{"mode":"catalog"}', TOOL)), frame(call(1, 'b', '{"mode":"compare"}', TOOL)), frame(call(1, 'c', '{"mode":', TOOL)), frame(call(1, arguments='"changes"}')), frame(call(2, 'd', '{"mode":"metrics"}', TOOL)), frame(finish='tool_calls')] class ToolStreamTests(unittest.TestCase): def normalize(self, data, pieces=None): normalizer = ToolStreamNormalizer({TOOL}) output = b'' for piece in pieces if pieces is not None else [data]: output += normalizer.feed(piece) return output + normalizer.finish() def completion(self, data): observer = ProviderStream(lambda _text: None) observer.feed(data) observer.finish() return observer.completion()['choices'] def test_raw_collision_is_a_red_capable_real_observer_fixture(self): tools = self.completion(wire(collision_rows()))[0]['message']['tool_calls'] self.assertEqual(len(tools), 3) self.assertEqual(tools[1]['function']['name'], TOOL + TOOL) with self.assertRaises(ValueError): json.loads(tools[1]['function']['arguments']) def test_collision_preserves_three_calls_and_following_wire_index_cannot_collide(self): result = self.completion(self.normalize(wire(collision_rows())))[0]['message']['tool_calls'] self.assertEqual([value['id'] for value in result], ['a', 'b', 'c', 'd']) self.assertEqual([value['function']['name'] for value in result], [TOOL] * 4) self.assertEqual([json.loads(value['function']['arguments'])['mode'] for value in result], ['catalog', 'compare', 'changes', 'metrics']) rows = [json.loads(line[5:]) for line in self.normalize(wire(collision_rows())).splitlines() if line.startswith(b'data:') and b'[DONE]' not in line] self.assertEqual([item['index'] for row in rows for choice in row['choices'] for item in choice['delta'].get('tool_calls', [])], [0, 1, 2, 2, 3]) def test_legal_parallel_interleaved_fragments_preserve_exact_wire(self): rows = [frame(call(0, 'a', '{"value":', TOOL), call(1, 'b', '{"value":', TOOL)), frame(call(1, arguments='"亮片🙂"}')), frame(call(0, arguments='"黑片"}')), frame(finish='tool_calls')] data = wire(rows, newline=b'\r\n') self.assertEqual(self.normalize(data), data) result = self.completion(data)[0]['message']['tool_calls'] self.assertEqual([json.loads(item['function']['arguments'])['value'] for item in result], ['黑片', '亮片🙂']) def test_every_byte_boundary_and_headers_heartbeats_are_preserved(self): data = b': heartbeat\n\nevent: message\n' + wire(collision_rows()) expected = self.normalize(data) self.assertEqual(self.normalize(data, [bytes([byte]) for byte in data]), expected) self.assertTrue(expected.startswith(b': heartbeat\n\nevent: message\n')) def test_reindexed_frame_preserves_name_id_argument_string_and_other_fields(self): arguments = ' {"quoted":"a\\\"b","path":"x\\\\y","unicode":"亮片🙂"} ' original = frame(call(0, 'identity/亮片', arguments, TOOL), content='原文🙂') original.update(id='completion-original', extra={'unchanged': True}) data = wire([frame(call(0, 'first', '{}', TOOL)), original, frame(finish='tool_calls')]) normalized = self.normalize(data) rows = [json.loads(line[5:]) for line in normalized.splitlines() if line.startswith(b'data:') and b'[DONE]' not in line] expected = json.loads(json.dumps(original)) expected['choices'][0]['delta']['tool_calls'][0]['index'] = 1 self.assertEqual(rows[1], expected) self.assertEqual(rows[1]['choices'][0]['delta']['tool_calls'][0]['function']['arguments'], arguments) def test_feed_returns_valid_events_before_the_rest_of_the_stream_arrives(self): normalizer = ToolStreamNormalizer({TOOL}) first = wire([frame(call(0, 'a', '{"mode":', TOOL))], done=False) self.assertEqual(normalizer.feed(first), first) final = wire([frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')]) self.assertEqual(normalizer.feed(final) + normalizer.finish(), final) def test_fragmented_name_and_delayed_initial_id_remain_one_call(self): rows = [frame(call(0, arguments='{', name='mcp__quality__')), frame(call(0, 'a', arguments='"mode":', name='quality_data')), frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')] data = wire(rows) self.assertEqual(self.normalize(data), data) self.assertEqual(self.completion(data)[0]['message']['tool_calls'][0]['id'], 'a') def test_message_snapshot_and_non_sse_json_are_not_rewritten(self): snapshot = {'choices': [{'index': 0, 'message': {'tool_calls': [call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)]}, 'finish_reason': 'tool_calls'}]} data = wire([snapshot]) self.assertEqual(self.normalize(data), data) ordinary = json.dumps(snapshot).encode() self.assertEqual(self.normalize(ordinary), ordinary) def test_choices_have_independent_index_and_id_namespaces(self): rows = [frame(call(0, 'a', '{}', TOOL), choice=0), frame(call(0, 'a', '{}', TOOL), choice=1), frame(call(0, 'b', '{}', TOOL), choice=0), frame(call(1, 'b', '{}', TOOL), choice=1), frame(choice=0, finish='tool_calls'), frame(choice=1, finish='tool_calls')] choices = self.completion(self.normalize(wire(rows))) self.assertEqual([[item['id'] for item in choice['message']['tool_calls']] for choice in choices], [['a', 'b'], ['a', 'b']]) def test_repeated_active_id_continues_but_repeated_name_is_not_silently_deduplicated(self): data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(call(0, 'a', '"catalog"}')), frame(finish='tool_calls')]) self.assertEqual(self.normalize(data), data) self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'a', '', TOOL))]) def reject(self, rows, *, code=None, done=False): with self.assertRaises(ToolStreamProtocolError) as caught: self.normalize(wire(rows, done=done)) if code: self.assertEqual(caught.exception.code, code) self.assertNotIn('sensitive-fixture', str(caught.exception)) def test_argument_errors_are_preserved_for_sdk_without_aborting_later_calls(self): for old in ('', '{"mode":', '[]', 'null', '42', '{}{}', '{"x":NaN}', '{"x":Infinity}', '{"x":1,"x":2}'): with self.subTest(old=old): result = self.completion(self.normalize(wire([ frame(call(0, 'a', old, TOOL)), frame(call(0, 'b', '{}', TOOL)), frame(finish='tool_calls')])))[0]['message']['tool_calls'] self.assertEqual([item['id'] for item in result], ['a', 'b']) self.assertEqual([item['function']['arguments'] for item in result], [old, '{}']) def test_reuse_inside_one_event_keeps_each_original_call(self): data = wire([frame(call(0, 'a', '{}', TOOL), call(0, 'b', '{}', TOOL), call(0, 'c', '{}', TOOL)), frame(finish='tool_calls')]) result = self.completion(self.normalize(data))[0]['message']['tool_calls'] self.assertEqual([item['id'] for item in result], ['a', 'b', 'c']) def test_retired_call_is_not_recovered_from_its_complete_arguments_or_name(self): self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"x":', TOOL)), frame(call(0, 'a', '1}'))], code='reused_call_id') def test_malformed_frame_and_invalid_tool_payload_do_not_leak_input(self): for data in (b'data: {sensitive-fixture}\n\n', b'data: []\n\n', b'data: {"choices":[],"choices":[]}\n\n'): with self.assertRaises(ToolStreamProtocolError) as caught: self.normalize(data) self.assertNotIn('sensitive-fixture', str(caught.exception)) for changed in (call(0, 'a', '{}', TOOL) | {'type': 'custom'}, call(0, 'a', '{}', TOOL) | {'function': {'name': TOOL, 'arguments': {}}}): self.reject([frame(changed)]) def test_partial_name_never_authorizes_a_rotation(self): self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(call(0, 'b', '{}', TOOL))], code='incomplete_previous_call') def test_unknown_tool_prefix_and_final_incomplete_name_fail_closed(self): self.reject([frame(call(0, 'a', '{}', 'sensitive-fixture'))], code='unadvertised_tool') self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(finish='tool_calls')], code='unadvertised_tool') self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', 'shell'))], code='unadvertised_tool') def test_retired_id_cannot_return_or_move_to_another_wire_index(self): first = [frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', TOOL))] for last in (call(0, 'a', '', None), call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)): with self.subTest(last=last): self.reject(first + [frame(last)], code='reused_call_id') def test_empty_or_invalid_id_and_missing_or_invalid_index_fail_closed(self): base = call(0, 'a', '{}', TOOL) for changed in ({**base, 'id': ''}, {**base, 'id': 3}, {**base, 'index': True}, {**base, 'index': -1}, {**base, 'index': 0.5}, {key: value for key, value in base.items() if key != 'index'}): with self.subTest(changed=changed): self.reject([frame(changed)]) def test_mimo_null_identity_and_name_are_continuations_of_an_identified_call(self): rows = [frame(call(0, 'call_mimo', '', TOOL)), frame({'index': 0, 'id': None, 'type': 'function', 'function': {'name': None, 'arguments': '{"mode": '}}), frame({'index': 0, 'id': None, 'type': 'function', 'function': {'name': None, 'arguments': '"catalog"}'}}), frame(finish='tool_calls')] result = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls'] self.assertEqual(len(result), 1) self.assertEqual(result[0]['id'], 'call_mimo') self.assertEqual(result[0]['function']['name'], TOOL) self.assertEqual(json.loads(result[0]['function']['arguments']), {'mode': 'catalog'}) def test_mimo_observed_namespace_alias_keeps_call_identity_and_arguments(self): # Captured with the real task input: MiMo Flash omitted only "mcp__". arguments = '{"mode": "query"}' for fragments in (['quality__quality_data'], ['quality__', 'quality_data'], ['q', 'uality__quality_data']): with self.subTest(fragments=fragments): rows = [frame(call(0, 'mimo_alias', '', fragments[0]))] rows += [frame(call(0, name=part)) for part in fragments[1:]] rows += [frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': arguments}}), frame(finish='tool_calls')] tools = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls'] self.assertEqual(len(tools), 1) self.assertEqual(tools[0]['id'], 'mimo_alias') self.assertEqual(tools[0]['function'], {'name': TOOL, 'arguments': arguments}) def test_namespace_alias_only_resolves_to_an_offered_tool(self): for name in ('quality_data', 'quality__shell', 'quality__quality_data_extra', 'functions.mcp__quality__quality_data'): self.reject([frame(call(0, 'a', '{}', name))], code='unadvertised_tool') normalizer = ToolStreamNormalizer({'other_tool'}) with self.assertRaises(ToolStreamProtocolError): normalizer.feed(wire([frame(call(0, 'a', '{}', 'quality__quality_data'))])) def test_null_identity_cannot_start_a_call_or_change_its_tool(self): self.reject([frame({'index': 0, 'id': None, 'function': {'name': TOOL, 'arguments': '{}'}})], code='invalid_call_id') self.reject([frame(call(0, 'a', '', TOOL)), frame({'index': 0, 'id': None, 'function': {'name': 'shell', 'arguments': '{}'}})], code='unadvertised_tool') def test_calls_after_choice_finish_or_done_fail_closed(self): self.reject([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'), frame(call(1, 'b', '{}', TOOL))], code='call_after_finish') normalizer = ToolStreamNormalizer({TOOL}) normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls')])) with self.assertRaises(ToolStreamProtocolError): normalizer.feed(wire([frame(call(1, 'b', '{}', TOOL))])) def test_missing_finish_and_failed_normalizer_cannot_be_resumed(self): normalizer = ToolStreamNormalizer({TOOL}) normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL))], done=False)) with self.assertRaises(ToolStreamProtocolError): normalizer.finish() with self.assertRaises(ToolStreamProtocolError): normalizer.feed(wire([frame(finish='tool_calls')])) def test_final_line_without_newline_and_usage_tail_are_preserved(self): data = wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'), {'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2}}]).rstrip(b'\n') self.assertEqual(self.normalize(data), data) def test_sparse_wire_indices_remain_unchanged_until_they_would_collide(self): data = wire([frame(call(4, 'a', '{}', TOOL)), frame(call(9, 'b', '{}', TOOL)), frame(finish='tool_calls')]) self.assertEqual(self.normalize(data), data) def test_one_failed_feed_cannot_return_an_earlier_valid_line_or_be_resumed(self): normalizer = ToolStreamNormalizer({TOOL}) data = wire([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"sensitive-fixture":', 'shell'))], done=False) with self.assertRaises(ToolStreamProtocolError): normalizer.feed(data) with self.assertRaises(ToolStreamProtocolError): normalizer.finish() def test_oversized_unterminated_frame_fails_closed(self): normalizer = ToolStreamNormalizer({TOOL}) normalizer.MAX_FRAME_BYTES = 32 with self.assertRaises(ToolStreamProtocolError) as caught: normalizer.feed(b'data: ' + b'x' * 33) self.assertEqual(caught.exception.code, 'frame_too_large') @unittest.skipUnless((ROOT / 'runtime/dsh/bin/node').is_file() and (ROOT / 'runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js').is_file(), 'requires the installed pinned DSH SDK') class InstalledSdkTests(unittest.TestCase): def sdk(self, data): script = r''' import fs from 'node:fs'; import { stream } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js'; import { normalizeContext } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/utils/transcript.js'; const wire = fs.readFileSync(0, 'utf8'); const model = {id:'m', name:'m', api:'openai-completions', provider:'test', baseUrl:'http://offline.invalid/v1', reasoning:false, input:['text'], cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:1024}; const context = normalizeContext({messages:[{role:'user',content:'authored offline fixture',timestamp:0}]}); let fetches = 0; const events = stream(model,context,{apiKey:'authored-no-network',fetch:async()=>{ fetches++; return new Response(wire,{status:200,headers:{'Content-Type':'text/event-stream'}}); }}); const calls = []; let terminal; for await (const event of events) { if (event.type === 'toolcall_end') calls.push({id:event.toolCall.id,name:event.toolCall.name, arguments:event.toolCall.arguments,invalid:event.toolCall.argumentsError !== undefined}); if (event.type === 'done' || event.type === 'error') terminal = event.type; } console.log(JSON.stringify({calls,terminal,fetches})); ''' result = subprocess.run([str(ROOT / 'runtime/dsh/bin/node'), '--input-type=module', '-e', script], cwd=ROOT, input=data, capture_output=True, timeout=15, check=True) return json.loads(result.stdout) def test_real_sdk_raw_collision_fails_and_normalized_calls_keep_ids_and_arguments(self): data = wire(collision_rows()) bad = self.sdk(data) self.assertEqual(len(bad['calls']), 3) self.assertTrue(bad['calls'][1]['invalid']) good = self.sdk(ToolStreamTests().normalize(data)) self.assertEqual(good['terminal'], 'done') self.assertEqual(good['fetches'], 1) self.assertEqual([value['id'] for value in good['calls']], ['a', 'b', 'c', 'd']) self.assertTrue(all(value['name'] == TOOL and not value['invalid'] for value in good['calls'])) self.assertEqual([value['arguments']['mode'] for value in good['calls']], ['catalog', 'compare', 'changes', 'metrics']) def test_real_sdk_still_rejects_a_final_incomplete_argument_object(self): data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(finish='tool_calls')]) result = self.sdk(ToolStreamTests().normalize(data)) self.assertTrue(result['calls'][0]['invalid']) def test_real_sdk_keeps_argument_error_and_subsequent_independent_call(self): data = wire([frame(call(0, 'bad', '{"mode":', TOOL)), frame(call(0, 'next', '{"mode":"query"}', TOOL)), frame(finish='tool_calls')]) result = self.sdk(ToolStreamTests().normalize(data)) self.assertEqual(result['terminal'], 'done') self.assertEqual([item['id'] for item in result['calls']], ['bad', 'next']) self.assertTrue(result['calls'][0]['invalid']) self.assertFalse(result['calls'][1]['invalid']) self.assertEqual(result['calls'][1]['arguments'], {'mode': 'query'}) def test_real_sdk_receives_the_advertised_name_for_observed_mimo_alias(self): data = wire([frame(call(0, 'mimo_alias', '', 'quality__quality_data')), frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': '{"mode":"query"}'}}), frame(finish='tool_calls')]) result = self.sdk(ToolStreamTests().normalize(data)) self.assertEqual(result['terminal'], 'done') self.assertEqual(result['calls'], [{'id': 'mimo_alias', 'name': TOOL, 'arguments': {'mode': 'query'}, 'invalid': False}]) if __name__ == '__main__': unittest.main()