接入 DSH 专用运行时和对话界面,保留模型原始回答,支持会话分页、连续追问与显式报告。 补齐组合筛选、各维度 Top100、分组查询到排名图的转换,图表故障静默省略并基于保存数据自动恢复。 优化报告风格、图表顺序和表格换行,更新接入说明及构建输入。 验证:独立目录前后端构建通过;118 项 Java、102 项前端、102 项图表、14 项 DSH 应用和 13 项组合筛选检查通过。 仅提交相关源码、锁定依赖与测试;不包含凭据、会话、业务数据或本地产物。
363 lines
20 KiB
Python
363 lines
20 KiB
Python
"""Authored SSE exercises the real observer and installed SDK without network I/O."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from nao_thinking_stream import ProviderStream
|
|
from nao_tool_stream import ToolStreamNormalizer, ToolStreamProtocolError
|
|
|
|
TOOL = 'mcp__quality__quality_data'
|
|
|
|
|
|
def frame(*calls, choice=0, finish=None, **delta):
|
|
if calls:
|
|
delta['tool_calls'] = list(calls)
|
|
return {'choices': [{'index': choice, 'delta': delta, 'finish_reason': finish}]}
|
|
|
|
|
|
def call(index, identity=None, arguments=None, name=None):
|
|
value = {'index': index}
|
|
if identity is not None:
|
|
value.update(id=identity, type='function')
|
|
function = {}
|
|
if name is not None:
|
|
function['name'] = name
|
|
if arguments is not None:
|
|
function['arguments'] = arguments
|
|
if function:
|
|
value['function'] = function
|
|
return value
|
|
|
|
|
|
def wire(rows, *, done=True, newline=b'\n'):
|
|
result = b''.join(b'data: ' + json.dumps(row, ensure_ascii=False).encode() + newline * 2 for row in rows)
|
|
return result + (b'data: [DONE]' + newline * 2 if done else b'')
|
|
|
|
|
|
def collision_rows():
|
|
return [frame(call(0, 'a', '{"mode":"catalog"}', TOOL)),
|
|
frame(call(1, 'b', '{"mode":"compare"}', TOOL)),
|
|
frame(call(1, 'c', '{"mode":', TOOL)),
|
|
frame(call(1, arguments='"changes"}')),
|
|
frame(call(2, 'd', '{"mode":"metrics"}', TOOL)),
|
|
frame(finish='tool_calls')]
|
|
|
|
|
|
class ToolStreamTests(unittest.TestCase):
|
|
def normalize(self, data, pieces=None):
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
output = b''
|
|
for piece in pieces if pieces is not None else [data]:
|
|
output += normalizer.feed(piece)
|
|
return output + normalizer.finish()
|
|
|
|
def completion(self, data):
|
|
observer = ProviderStream(lambda _text: None)
|
|
observer.feed(data)
|
|
observer.finish()
|
|
return observer.completion()['choices']
|
|
|
|
def test_raw_collision_is_a_red_capable_real_observer_fixture(self):
|
|
tools = self.completion(wire(collision_rows()))[0]['message']['tool_calls']
|
|
self.assertEqual(len(tools), 3)
|
|
self.assertEqual(tools[1]['function']['name'], TOOL + TOOL)
|
|
with self.assertRaises(ValueError):
|
|
json.loads(tools[1]['function']['arguments'])
|
|
|
|
def test_collision_preserves_three_calls_and_following_wire_index_cannot_collide(self):
|
|
result = self.completion(self.normalize(wire(collision_rows())))[0]['message']['tool_calls']
|
|
self.assertEqual([value['id'] for value in result], ['a', 'b', 'c', 'd'])
|
|
self.assertEqual([value['function']['name'] for value in result], [TOOL] * 4)
|
|
self.assertEqual([json.loads(value['function']['arguments'])['mode'] for value in result],
|
|
['catalog', 'compare', 'changes', 'metrics'])
|
|
rows = [json.loads(line[5:]) for line in self.normalize(wire(collision_rows())).splitlines()
|
|
if line.startswith(b'data:') and b'[DONE]' not in line]
|
|
self.assertEqual([item['index'] for row in rows for choice in row['choices']
|
|
for item in choice['delta'].get('tool_calls', [])], [0, 1, 2, 2, 3])
|
|
|
|
def test_legal_parallel_interleaved_fragments_preserve_exact_wire(self):
|
|
rows = [frame(call(0, 'a', '{"value":', TOOL), call(1, 'b', '{"value":', TOOL)),
|
|
frame(call(1, arguments='"亮片🙂"}')),
|
|
frame(call(0, arguments='"黑片"}')), frame(finish='tool_calls')]
|
|
data = wire(rows, newline=b'\r\n')
|
|
self.assertEqual(self.normalize(data), data)
|
|
result = self.completion(data)[0]['message']['tool_calls']
|
|
self.assertEqual([json.loads(item['function']['arguments'])['value'] for item in result], ['黑片', '亮片🙂'])
|
|
|
|
def test_every_byte_boundary_and_headers_heartbeats_are_preserved(self):
|
|
data = b': heartbeat\n\nevent: message\n' + wire(collision_rows())
|
|
expected = self.normalize(data)
|
|
self.assertEqual(self.normalize(data, [bytes([byte]) for byte in data]), expected)
|
|
self.assertTrue(expected.startswith(b': heartbeat\n\nevent: message\n'))
|
|
|
|
def test_reindexed_frame_preserves_name_id_argument_string_and_other_fields(self):
|
|
arguments = ' {"quoted":"a\\\"b","path":"x\\\\y","unicode":"亮片🙂"} '
|
|
original = frame(call(0, 'identity/亮片', arguments, TOOL), content='原文🙂')
|
|
original.update(id='completion-original', extra={'unchanged': True})
|
|
data = wire([frame(call(0, 'first', '{}', TOOL)), original, frame(finish='tool_calls')])
|
|
normalized = self.normalize(data)
|
|
rows = [json.loads(line[5:]) for line in normalized.splitlines()
|
|
if line.startswith(b'data:') and b'[DONE]' not in line]
|
|
expected = json.loads(json.dumps(original))
|
|
expected['choices'][0]['delta']['tool_calls'][0]['index'] = 1
|
|
self.assertEqual(rows[1], expected)
|
|
self.assertEqual(rows[1]['choices'][0]['delta']['tool_calls'][0]['function']['arguments'], arguments)
|
|
|
|
def test_feed_returns_valid_events_before_the_rest_of_the_stream_arrives(self):
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
first = wire([frame(call(0, 'a', '{"mode":', TOOL))], done=False)
|
|
self.assertEqual(normalizer.feed(first), first)
|
|
final = wire([frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')])
|
|
self.assertEqual(normalizer.feed(final) + normalizer.finish(), final)
|
|
|
|
def test_fragmented_name_and_delayed_initial_id_remain_one_call(self):
|
|
rows = [frame(call(0, arguments='{', name='mcp__quality__')),
|
|
frame(call(0, 'a', arguments='"mode":', name='quality_data')),
|
|
frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')]
|
|
data = wire(rows)
|
|
self.assertEqual(self.normalize(data), data)
|
|
self.assertEqual(self.completion(data)[0]['message']['tool_calls'][0]['id'], 'a')
|
|
|
|
def test_message_snapshot_and_non_sse_json_are_not_rewritten(self):
|
|
snapshot = {'choices': [{'index': 0, 'message': {'tool_calls': [call(1, 'a', '{}', TOOL),
|
|
call(1, 'b', '{}', TOOL)]},
|
|
'finish_reason': 'tool_calls'}]}
|
|
data = wire([snapshot])
|
|
self.assertEqual(self.normalize(data), data)
|
|
ordinary = json.dumps(snapshot).encode()
|
|
self.assertEqual(self.normalize(ordinary), ordinary)
|
|
|
|
def test_choices_have_independent_index_and_id_namespaces(self):
|
|
rows = [frame(call(0, 'a', '{}', TOOL), choice=0), frame(call(0, 'a', '{}', TOOL), choice=1),
|
|
frame(call(0, 'b', '{}', TOOL), choice=0), frame(call(1, 'b', '{}', TOOL), choice=1),
|
|
frame(choice=0, finish='tool_calls'), frame(choice=1, finish='tool_calls')]
|
|
choices = self.completion(self.normalize(wire(rows)))
|
|
self.assertEqual([[item['id'] for item in choice['message']['tool_calls']] for choice in choices],
|
|
[['a', 'b'], ['a', 'b']])
|
|
|
|
def test_repeated_active_id_continues_but_repeated_name_is_not_silently_deduplicated(self):
|
|
data = wire([frame(call(0, 'a', '{"mode":', TOOL)),
|
|
frame(call(0, 'a', '"catalog"}')), frame(finish='tool_calls')])
|
|
self.assertEqual(self.normalize(data), data)
|
|
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'a', '', TOOL))])
|
|
|
|
def reject(self, rows, *, code=None, done=False):
|
|
with self.assertRaises(ToolStreamProtocolError) as caught:
|
|
self.normalize(wire(rows, done=done))
|
|
if code:
|
|
self.assertEqual(caught.exception.code, code)
|
|
self.assertNotIn('sensitive-fixture', str(caught.exception))
|
|
|
|
def test_argument_errors_are_preserved_for_sdk_without_aborting_later_calls(self):
|
|
for old in ('', '{"mode":', '[]', 'null', '42', '{}{}', '{"x":NaN}', '{"x":Infinity}',
|
|
'{"x":1,"x":2}'):
|
|
with self.subTest(old=old):
|
|
result = self.completion(self.normalize(wire([
|
|
frame(call(0, 'a', old, TOOL)), frame(call(0, 'b', '{}', TOOL)),
|
|
frame(finish='tool_calls')])))[0]['message']['tool_calls']
|
|
self.assertEqual([item['id'] for item in result], ['a', 'b'])
|
|
self.assertEqual([item['function']['arguments'] for item in result], [old, '{}'])
|
|
|
|
def test_reuse_inside_one_event_keeps_each_original_call(self):
|
|
data = wire([frame(call(0, 'a', '{}', TOOL), call(0, 'b', '{}', TOOL), call(0, 'c', '{}', TOOL)),
|
|
frame(finish='tool_calls')])
|
|
result = self.completion(self.normalize(data))[0]['message']['tool_calls']
|
|
self.assertEqual([item['id'] for item in result], ['a', 'b', 'c'])
|
|
|
|
def test_retired_call_is_not_recovered_from_its_complete_arguments_or_name(self):
|
|
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"x":', TOOL)),
|
|
frame(call(0, 'a', '1}'))], code='reused_call_id')
|
|
|
|
def test_malformed_frame_and_invalid_tool_payload_do_not_leak_input(self):
|
|
for data in (b'data: {sensitive-fixture}\n\n', b'data: []\n\n',
|
|
b'data: {"choices":[],"choices":[]}\n\n'):
|
|
with self.assertRaises(ToolStreamProtocolError) as caught:
|
|
self.normalize(data)
|
|
self.assertNotIn('sensitive-fixture', str(caught.exception))
|
|
for changed in (call(0, 'a', '{}', TOOL) | {'type': 'custom'},
|
|
call(0, 'a', '{}', TOOL) | {'function': {'name': TOOL, 'arguments': {}}}):
|
|
self.reject([frame(changed)])
|
|
|
|
def test_partial_name_never_authorizes_a_rotation(self):
|
|
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(call(0, 'b', '{}', TOOL))],
|
|
code='incomplete_previous_call')
|
|
|
|
def test_unknown_tool_prefix_and_final_incomplete_name_fail_closed(self):
|
|
self.reject([frame(call(0, 'a', '{}', 'sensitive-fixture'))], code='unadvertised_tool')
|
|
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(finish='tool_calls')],
|
|
code='unadvertised_tool')
|
|
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', 'shell'))],
|
|
code='unadvertised_tool')
|
|
|
|
def test_retired_id_cannot_return_or_move_to_another_wire_index(self):
|
|
first = [frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', TOOL))]
|
|
for last in (call(0, 'a', '', None), call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)):
|
|
with self.subTest(last=last):
|
|
self.reject(first + [frame(last)], code='reused_call_id')
|
|
|
|
def test_empty_or_invalid_id_and_missing_or_invalid_index_fail_closed(self):
|
|
base = call(0, 'a', '{}', TOOL)
|
|
for changed in ({**base, 'id': ''}, {**base, 'id': 3}, {**base, 'index': True},
|
|
{**base, 'index': -1}, {**base, 'index': 0.5},
|
|
{key: value for key, value in base.items() if key != 'index'}):
|
|
with self.subTest(changed=changed):
|
|
self.reject([frame(changed)])
|
|
|
|
def test_mimo_null_identity_and_name_are_continuations_of_an_identified_call(self):
|
|
rows = [frame(call(0, 'call_mimo', '', TOOL)),
|
|
frame({'index': 0, 'id': None, 'type': 'function',
|
|
'function': {'name': None, 'arguments': '{"mode": '}}),
|
|
frame({'index': 0, 'id': None, 'type': 'function',
|
|
'function': {'name': None, 'arguments': '"catalog"}'}}),
|
|
frame(finish='tool_calls')]
|
|
result = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0]['id'], 'call_mimo')
|
|
self.assertEqual(result[0]['function']['name'], TOOL)
|
|
self.assertEqual(json.loads(result[0]['function']['arguments']), {'mode': 'catalog'})
|
|
|
|
def test_mimo_observed_namespace_alias_keeps_call_identity_and_arguments(self):
|
|
# Captured with the real task input: MiMo Flash omitted only "mcp__".
|
|
arguments = '{"mode": "query"}'
|
|
for fragments in (['quality__quality_data'], ['quality__', 'quality_data'],
|
|
['q', 'uality__quality_data']):
|
|
with self.subTest(fragments=fragments):
|
|
rows = [frame(call(0, 'mimo_alias', '', fragments[0]))]
|
|
rows += [frame(call(0, name=part)) for part in fragments[1:]]
|
|
rows += [frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': arguments}}),
|
|
frame(finish='tool_calls')]
|
|
tools = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
|
|
self.assertEqual(len(tools), 1)
|
|
self.assertEqual(tools[0]['id'], 'mimo_alias')
|
|
self.assertEqual(tools[0]['function'], {'name': TOOL, 'arguments': arguments})
|
|
|
|
def test_namespace_alias_only_resolves_to_an_offered_tool(self):
|
|
for name in ('quality_data', 'quality__shell', 'quality__quality_data_extra',
|
|
'functions.mcp__quality__quality_data'):
|
|
self.reject([frame(call(0, 'a', '{}', name))], code='unadvertised_tool')
|
|
normalizer = ToolStreamNormalizer({'other_tool'})
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.feed(wire([frame(call(0, 'a', '{}', 'quality__quality_data'))]))
|
|
|
|
def test_null_identity_cannot_start_a_call_or_change_its_tool(self):
|
|
self.reject([frame({'index': 0, 'id': None, 'function': {'name': TOOL, 'arguments': '{}'}})],
|
|
code='invalid_call_id')
|
|
self.reject([frame(call(0, 'a', '', TOOL)),
|
|
frame({'index': 0, 'id': None, 'function': {'name': 'shell', 'arguments': '{}'}})],
|
|
code='unadvertised_tool')
|
|
|
|
def test_calls_after_choice_finish_or_done_fail_closed(self):
|
|
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
|
|
frame(call(1, 'b', '{}', TOOL))], code='call_after_finish')
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls')]))
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.feed(wire([frame(call(1, 'b', '{}', TOOL))]))
|
|
|
|
def test_missing_finish_and_failed_normalizer_cannot_be_resumed(self):
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL))], done=False))
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.finish()
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.feed(wire([frame(finish='tool_calls')]))
|
|
|
|
def test_final_line_without_newline_and_usage_tail_are_preserved(self):
|
|
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
|
|
{'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2}}]).rstrip(b'\n')
|
|
self.assertEqual(self.normalize(data), data)
|
|
|
|
def test_sparse_wire_indices_remain_unchanged_until_they_would_collide(self):
|
|
data = wire([frame(call(4, 'a', '{}', TOOL)), frame(call(9, 'b', '{}', TOOL)), frame(finish='tool_calls')])
|
|
self.assertEqual(self.normalize(data), data)
|
|
|
|
def test_one_failed_feed_cannot_return_an_earlier_valid_line_or_be_resumed(self):
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"sensitive-fixture":', 'shell'))], done=False)
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.feed(data)
|
|
with self.assertRaises(ToolStreamProtocolError):
|
|
normalizer.finish()
|
|
|
|
def test_oversized_unterminated_frame_fails_closed(self):
|
|
normalizer = ToolStreamNormalizer({TOOL})
|
|
normalizer.MAX_FRAME_BYTES = 32
|
|
with self.assertRaises(ToolStreamProtocolError) as caught:
|
|
normalizer.feed(b'data: ' + b'x' * 33)
|
|
self.assertEqual(caught.exception.code, 'frame_too_large')
|
|
|
|
|
|
@unittest.skipUnless((ROOT / 'runtime/dsh/bin/node').is_file()
|
|
and (ROOT / 'runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js').is_file(),
|
|
'requires the installed pinned DSH SDK')
|
|
class InstalledSdkTests(unittest.TestCase):
|
|
def sdk(self, data):
|
|
script = r'''
|
|
import fs from 'node:fs';
|
|
import { stream } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js';
|
|
import { normalizeContext } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/utils/transcript.js';
|
|
const wire = fs.readFileSync(0, 'utf8');
|
|
const model = {id:'m', name:'m', api:'openai-completions', provider:'test', baseUrl:'http://offline.invalid/v1',
|
|
reasoning:false, input:['text'], cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:1024};
|
|
const context = normalizeContext({messages:[{role:'user',content:'authored offline fixture',timestamp:0}]});
|
|
let fetches = 0;
|
|
const events = stream(model,context,{apiKey:'authored-no-network',fetch:async()=>{
|
|
fetches++; return new Response(wire,{status:200,headers:{'Content-Type':'text/event-stream'}});
|
|
}});
|
|
const calls = []; let terminal;
|
|
for await (const event of events) {
|
|
if (event.type === 'toolcall_end') calls.push({id:event.toolCall.id,name:event.toolCall.name,
|
|
arguments:event.toolCall.arguments,invalid:event.toolCall.argumentsError !== undefined});
|
|
if (event.type === 'done' || event.type === 'error') terminal = event.type;
|
|
}
|
|
console.log(JSON.stringify({calls,terminal,fetches}));
|
|
'''
|
|
result = subprocess.run([str(ROOT / 'runtime/dsh/bin/node'), '--input-type=module', '-e', script],
|
|
cwd=ROOT, input=data, capture_output=True, timeout=15, check=True)
|
|
return json.loads(result.stdout)
|
|
|
|
def test_real_sdk_raw_collision_fails_and_normalized_calls_keep_ids_and_arguments(self):
|
|
data = wire(collision_rows())
|
|
bad = self.sdk(data)
|
|
self.assertEqual(len(bad['calls']), 3)
|
|
self.assertTrue(bad['calls'][1]['invalid'])
|
|
good = self.sdk(ToolStreamTests().normalize(data))
|
|
self.assertEqual(good['terminal'], 'done')
|
|
self.assertEqual(good['fetches'], 1)
|
|
self.assertEqual([value['id'] for value in good['calls']], ['a', 'b', 'c', 'd'])
|
|
self.assertTrue(all(value['name'] == TOOL and not value['invalid'] for value in good['calls']))
|
|
self.assertEqual([value['arguments']['mode'] for value in good['calls']],
|
|
['catalog', 'compare', 'changes', 'metrics'])
|
|
|
|
def test_real_sdk_still_rejects_a_final_incomplete_argument_object(self):
|
|
data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(finish='tool_calls')])
|
|
result = self.sdk(ToolStreamTests().normalize(data))
|
|
self.assertTrue(result['calls'][0]['invalid'])
|
|
|
|
def test_real_sdk_keeps_argument_error_and_subsequent_independent_call(self):
|
|
data = wire([frame(call(0, 'bad', '{"mode":', TOOL)),
|
|
frame(call(0, 'next', '{"mode":"query"}', TOOL)), frame(finish='tool_calls')])
|
|
result = self.sdk(ToolStreamTests().normalize(data))
|
|
self.assertEqual(result['terminal'], 'done')
|
|
self.assertEqual([item['id'] for item in result['calls']], ['bad', 'next'])
|
|
self.assertTrue(result['calls'][0]['invalid'])
|
|
self.assertFalse(result['calls'][1]['invalid'])
|
|
self.assertEqual(result['calls'][1]['arguments'], {'mode': 'query'})
|
|
|
|
def test_real_sdk_receives_the_advertised_name_for_observed_mimo_alias(self):
|
|
data = wire([frame(call(0, 'mimo_alias', '', 'quality__quality_data')),
|
|
frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': '{"mode":"query"}'}}),
|
|
frame(finish='tool_calls')])
|
|
result = self.sdk(ToolStreamTests().normalize(data))
|
|
self.assertEqual(result['terminal'], 'done')
|
|
self.assertEqual(result['calls'], [{'id': 'mimo_alias', 'name': TOOL,
|
|
'arguments': {'mode': 'query'}, 'invalid': False}])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|