Files
quality_analyse/verification/test_nao_tool_stream.py
T
lyun f1a2fbd494 统一:DSH 智能分析、Top100 与图表静默恢复
接入 DSH 专用运行时和对话界面,保留模型原始回答,支持会话分页、连续追问与显式报告。
补齐组合筛选、各维度 Top100、分组查询到排名图的转换,图表故障静默省略并基于保存数据自动恢复。
优化报告风格、图表顺序和表格换行,更新接入说明及构建输入。

验证:独立目录前后端构建通过;118 项 Java、102 项前端、102 项图表、14 项 DSH 应用和 13 项组合筛选检查通过。
仅提交相关源码、锁定依赖与测试;不包含凭据、会话、业务数据或本地产物。
2026-11-12 11:57:41 +08:00

363 lines
20 KiB
Python

"""Authored SSE exercises the real observer and installed SDK without network I/O."""
import json
from pathlib import Path
import subprocess
import sys
import unittest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
from nao_thinking_stream import ProviderStream
from nao_tool_stream import ToolStreamNormalizer, ToolStreamProtocolError
TOOL = 'mcp__quality__quality_data'
def frame(*calls, choice=0, finish=None, **delta):
if calls:
delta['tool_calls'] = list(calls)
return {'choices': [{'index': choice, 'delta': delta, 'finish_reason': finish}]}
def call(index, identity=None, arguments=None, name=None):
value = {'index': index}
if identity is not None:
value.update(id=identity, type='function')
function = {}
if name is not None:
function['name'] = name
if arguments is not None:
function['arguments'] = arguments
if function:
value['function'] = function
return value
def wire(rows, *, done=True, newline=b'\n'):
result = b''.join(b'data: ' + json.dumps(row, ensure_ascii=False).encode() + newline * 2 for row in rows)
return result + (b'data: [DONE]' + newline * 2 if done else b'')
def collision_rows():
return [frame(call(0, 'a', '{"mode":"catalog"}', TOOL)),
frame(call(1, 'b', '{"mode":"compare"}', TOOL)),
frame(call(1, 'c', '{"mode":', TOOL)),
frame(call(1, arguments='"changes"}')),
frame(call(2, 'd', '{"mode":"metrics"}', TOOL)),
frame(finish='tool_calls')]
class ToolStreamTests(unittest.TestCase):
def normalize(self, data, pieces=None):
normalizer = ToolStreamNormalizer({TOOL})
output = b''
for piece in pieces if pieces is not None else [data]:
output += normalizer.feed(piece)
return output + normalizer.finish()
def completion(self, data):
observer = ProviderStream(lambda _text: None)
observer.feed(data)
observer.finish()
return observer.completion()['choices']
def test_raw_collision_is_a_red_capable_real_observer_fixture(self):
tools = self.completion(wire(collision_rows()))[0]['message']['tool_calls']
self.assertEqual(len(tools), 3)
self.assertEqual(tools[1]['function']['name'], TOOL + TOOL)
with self.assertRaises(ValueError):
json.loads(tools[1]['function']['arguments'])
def test_collision_preserves_three_calls_and_following_wire_index_cannot_collide(self):
result = self.completion(self.normalize(wire(collision_rows())))[0]['message']['tool_calls']
self.assertEqual([value['id'] for value in result], ['a', 'b', 'c', 'd'])
self.assertEqual([value['function']['name'] for value in result], [TOOL] * 4)
self.assertEqual([json.loads(value['function']['arguments'])['mode'] for value in result],
['catalog', 'compare', 'changes', 'metrics'])
rows = [json.loads(line[5:]) for line in self.normalize(wire(collision_rows())).splitlines()
if line.startswith(b'data:') and b'[DONE]' not in line]
self.assertEqual([item['index'] for row in rows for choice in row['choices']
for item in choice['delta'].get('tool_calls', [])], [0, 1, 2, 2, 3])
def test_legal_parallel_interleaved_fragments_preserve_exact_wire(self):
rows = [frame(call(0, 'a', '{"value":', TOOL), call(1, 'b', '{"value":', TOOL)),
frame(call(1, arguments='"亮片🙂"}')),
frame(call(0, arguments='"黑片"}')), frame(finish='tool_calls')]
data = wire(rows, newline=b'\r\n')
self.assertEqual(self.normalize(data), data)
result = self.completion(data)[0]['message']['tool_calls']
self.assertEqual([json.loads(item['function']['arguments'])['value'] for item in result], ['黑片', '亮片🙂'])
def test_every_byte_boundary_and_headers_heartbeats_are_preserved(self):
data = b': heartbeat\n\nevent: message\n' + wire(collision_rows())
expected = self.normalize(data)
self.assertEqual(self.normalize(data, [bytes([byte]) for byte in data]), expected)
self.assertTrue(expected.startswith(b': heartbeat\n\nevent: message\n'))
def test_reindexed_frame_preserves_name_id_argument_string_and_other_fields(self):
arguments = ' {"quoted":"a\\\"b","path":"x\\\\y","unicode":"亮片🙂"} '
original = frame(call(0, 'identity/亮片', arguments, TOOL), content='原文🙂')
original.update(id='completion-original', extra={'unchanged': True})
data = wire([frame(call(0, 'first', '{}', TOOL)), original, frame(finish='tool_calls')])
normalized = self.normalize(data)
rows = [json.loads(line[5:]) for line in normalized.splitlines()
if line.startswith(b'data:') and b'[DONE]' not in line]
expected = json.loads(json.dumps(original))
expected['choices'][0]['delta']['tool_calls'][0]['index'] = 1
self.assertEqual(rows[1], expected)
self.assertEqual(rows[1]['choices'][0]['delta']['tool_calls'][0]['function']['arguments'], arguments)
def test_feed_returns_valid_events_before_the_rest_of_the_stream_arrives(self):
normalizer = ToolStreamNormalizer({TOOL})
first = wire([frame(call(0, 'a', '{"mode":', TOOL))], done=False)
self.assertEqual(normalizer.feed(first), first)
final = wire([frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')])
self.assertEqual(normalizer.feed(final) + normalizer.finish(), final)
def test_fragmented_name_and_delayed_initial_id_remain_one_call(self):
rows = [frame(call(0, arguments='{', name='mcp__quality__')),
frame(call(0, 'a', arguments='"mode":', name='quality_data')),
frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')]
data = wire(rows)
self.assertEqual(self.normalize(data), data)
self.assertEqual(self.completion(data)[0]['message']['tool_calls'][0]['id'], 'a')
def test_message_snapshot_and_non_sse_json_are_not_rewritten(self):
snapshot = {'choices': [{'index': 0, 'message': {'tool_calls': [call(1, 'a', '{}', TOOL),
call(1, 'b', '{}', TOOL)]},
'finish_reason': 'tool_calls'}]}
data = wire([snapshot])
self.assertEqual(self.normalize(data), data)
ordinary = json.dumps(snapshot).encode()
self.assertEqual(self.normalize(ordinary), ordinary)
def test_choices_have_independent_index_and_id_namespaces(self):
rows = [frame(call(0, 'a', '{}', TOOL), choice=0), frame(call(0, 'a', '{}', TOOL), choice=1),
frame(call(0, 'b', '{}', TOOL), choice=0), frame(call(1, 'b', '{}', TOOL), choice=1),
frame(choice=0, finish='tool_calls'), frame(choice=1, finish='tool_calls')]
choices = self.completion(self.normalize(wire(rows)))
self.assertEqual([[item['id'] for item in choice['message']['tool_calls']] for choice in choices],
[['a', 'b'], ['a', 'b']])
def test_repeated_active_id_continues_but_repeated_name_is_not_silently_deduplicated(self):
data = wire([frame(call(0, 'a', '{"mode":', TOOL)),
frame(call(0, 'a', '"catalog"}')), frame(finish='tool_calls')])
self.assertEqual(self.normalize(data), data)
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'a', '', TOOL))])
def reject(self, rows, *, code=None, done=False):
with self.assertRaises(ToolStreamProtocolError) as caught:
self.normalize(wire(rows, done=done))
if code:
self.assertEqual(caught.exception.code, code)
self.assertNotIn('sensitive-fixture', str(caught.exception))
def test_argument_errors_are_preserved_for_sdk_without_aborting_later_calls(self):
for old in ('', '{"mode":', '[]', 'null', '42', '{}{}', '{"x":NaN}', '{"x":Infinity}',
'{"x":1,"x":2}'):
with self.subTest(old=old):
result = self.completion(self.normalize(wire([
frame(call(0, 'a', old, TOOL)), frame(call(0, 'b', '{}', TOOL)),
frame(finish='tool_calls')])))[0]['message']['tool_calls']
self.assertEqual([item['id'] for item in result], ['a', 'b'])
self.assertEqual([item['function']['arguments'] for item in result], [old, '{}'])
def test_reuse_inside_one_event_keeps_each_original_call(self):
data = wire([frame(call(0, 'a', '{}', TOOL), call(0, 'b', '{}', TOOL), call(0, 'c', '{}', TOOL)),
frame(finish='tool_calls')])
result = self.completion(self.normalize(data))[0]['message']['tool_calls']
self.assertEqual([item['id'] for item in result], ['a', 'b', 'c'])
def test_retired_call_is_not_recovered_from_its_complete_arguments_or_name(self):
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"x":', TOOL)),
frame(call(0, 'a', '1}'))], code='reused_call_id')
def test_malformed_frame_and_invalid_tool_payload_do_not_leak_input(self):
for data in (b'data: {sensitive-fixture}\n\n', b'data: []\n\n',
b'data: {"choices":[],"choices":[]}\n\n'):
with self.assertRaises(ToolStreamProtocolError) as caught:
self.normalize(data)
self.assertNotIn('sensitive-fixture', str(caught.exception))
for changed in (call(0, 'a', '{}', TOOL) | {'type': 'custom'},
call(0, 'a', '{}', TOOL) | {'function': {'name': TOOL, 'arguments': {}}}):
self.reject([frame(changed)])
def test_partial_name_never_authorizes_a_rotation(self):
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(call(0, 'b', '{}', TOOL))],
code='incomplete_previous_call')
def test_unknown_tool_prefix_and_final_incomplete_name_fail_closed(self):
self.reject([frame(call(0, 'a', '{}', 'sensitive-fixture'))], code='unadvertised_tool')
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(finish='tool_calls')],
code='unadvertised_tool')
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', 'shell'))],
code='unadvertised_tool')
def test_retired_id_cannot_return_or_move_to_another_wire_index(self):
first = [frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', TOOL))]
for last in (call(0, 'a', '', None), call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)):
with self.subTest(last=last):
self.reject(first + [frame(last)], code='reused_call_id')
def test_empty_or_invalid_id_and_missing_or_invalid_index_fail_closed(self):
base = call(0, 'a', '{}', TOOL)
for changed in ({**base, 'id': ''}, {**base, 'id': 3}, {**base, 'index': True},
{**base, 'index': -1}, {**base, 'index': 0.5},
{key: value for key, value in base.items() if key != 'index'}):
with self.subTest(changed=changed):
self.reject([frame(changed)])
def test_mimo_null_identity_and_name_are_continuations_of_an_identified_call(self):
rows = [frame(call(0, 'call_mimo', '', TOOL)),
frame({'index': 0, 'id': None, 'type': 'function',
'function': {'name': None, 'arguments': '{"mode": '}}),
frame({'index': 0, 'id': None, 'type': 'function',
'function': {'name': None, 'arguments': '"catalog"}'}}),
frame(finish='tool_calls')]
result = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['id'], 'call_mimo')
self.assertEqual(result[0]['function']['name'], TOOL)
self.assertEqual(json.loads(result[0]['function']['arguments']), {'mode': 'catalog'})
def test_mimo_observed_namespace_alias_keeps_call_identity_and_arguments(self):
# Captured with the real task input: MiMo Flash omitted only "mcp__".
arguments = '{"mode": "query"}'
for fragments in (['quality__quality_data'], ['quality__', 'quality_data'],
['q', 'uality__quality_data']):
with self.subTest(fragments=fragments):
rows = [frame(call(0, 'mimo_alias', '', fragments[0]))]
rows += [frame(call(0, name=part)) for part in fragments[1:]]
rows += [frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': arguments}}),
frame(finish='tool_calls')]
tools = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
self.assertEqual(len(tools), 1)
self.assertEqual(tools[0]['id'], 'mimo_alias')
self.assertEqual(tools[0]['function'], {'name': TOOL, 'arguments': arguments})
def test_namespace_alias_only_resolves_to_an_offered_tool(self):
for name in ('quality_data', 'quality__shell', 'quality__quality_data_extra',
'functions.mcp__quality__quality_data'):
self.reject([frame(call(0, 'a', '{}', name))], code='unadvertised_tool')
normalizer = ToolStreamNormalizer({'other_tool'})
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(call(0, 'a', '{}', 'quality__quality_data'))]))
def test_null_identity_cannot_start_a_call_or_change_its_tool(self):
self.reject([frame({'index': 0, 'id': None, 'function': {'name': TOOL, 'arguments': '{}'}})],
code='invalid_call_id')
self.reject([frame(call(0, 'a', '', TOOL)),
frame({'index': 0, 'id': None, 'function': {'name': 'shell', 'arguments': '{}'}})],
code='unadvertised_tool')
def test_calls_after_choice_finish_or_done_fail_closed(self):
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
frame(call(1, 'b', '{}', TOOL))], code='call_after_finish')
normalizer = ToolStreamNormalizer({TOOL})
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls')]))
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(call(1, 'b', '{}', TOOL))]))
def test_missing_finish_and_failed_normalizer_cannot_be_resumed(self):
normalizer = ToolStreamNormalizer({TOOL})
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL))], done=False))
with self.assertRaises(ToolStreamProtocolError):
normalizer.finish()
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(finish='tool_calls')]))
def test_final_line_without_newline_and_usage_tail_are_preserved(self):
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
{'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2}}]).rstrip(b'\n')
self.assertEqual(self.normalize(data), data)
def test_sparse_wire_indices_remain_unchanged_until_they_would_collide(self):
data = wire([frame(call(4, 'a', '{}', TOOL)), frame(call(9, 'b', '{}', TOOL)), frame(finish='tool_calls')])
self.assertEqual(self.normalize(data), data)
def test_one_failed_feed_cannot_return_an_earlier_valid_line_or_be_resumed(self):
normalizer = ToolStreamNormalizer({TOOL})
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"sensitive-fixture":', 'shell'))], done=False)
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(data)
with self.assertRaises(ToolStreamProtocolError):
normalizer.finish()
def test_oversized_unterminated_frame_fails_closed(self):
normalizer = ToolStreamNormalizer({TOOL})
normalizer.MAX_FRAME_BYTES = 32
with self.assertRaises(ToolStreamProtocolError) as caught:
normalizer.feed(b'data: ' + b'x' * 33)
self.assertEqual(caught.exception.code, 'frame_too_large')
@unittest.skipUnless((ROOT / 'runtime/dsh/bin/node').is_file()
and (ROOT / 'runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js').is_file(),
'requires the installed pinned DSH SDK')
class InstalledSdkTests(unittest.TestCase):
def sdk(self, data):
script = r'''
import fs from 'node:fs';
import { stream } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js';
import { normalizeContext } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/utils/transcript.js';
const wire = fs.readFileSync(0, 'utf8');
const model = {id:'m', name:'m', api:'openai-completions', provider:'test', baseUrl:'http://offline.invalid/v1',
reasoning:false, input:['text'], cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:1024};
const context = normalizeContext({messages:[{role:'user',content:'authored offline fixture',timestamp:0}]});
let fetches = 0;
const events = stream(model,context,{apiKey:'authored-no-network',fetch:async()=>{
fetches++; return new Response(wire,{status:200,headers:{'Content-Type':'text/event-stream'}});
}});
const calls = []; let terminal;
for await (const event of events) {
if (event.type === 'toolcall_end') calls.push({id:event.toolCall.id,name:event.toolCall.name,
arguments:event.toolCall.arguments,invalid:event.toolCall.argumentsError !== undefined});
if (event.type === 'done' || event.type === 'error') terminal = event.type;
}
console.log(JSON.stringify({calls,terminal,fetches}));
'''
result = subprocess.run([str(ROOT / 'runtime/dsh/bin/node'), '--input-type=module', '-e', script],
cwd=ROOT, input=data, capture_output=True, timeout=15, check=True)
return json.loads(result.stdout)
def test_real_sdk_raw_collision_fails_and_normalized_calls_keep_ids_and_arguments(self):
data = wire(collision_rows())
bad = self.sdk(data)
self.assertEqual(len(bad['calls']), 3)
self.assertTrue(bad['calls'][1]['invalid'])
good = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(good['terminal'], 'done')
self.assertEqual(good['fetches'], 1)
self.assertEqual([value['id'] for value in good['calls']], ['a', 'b', 'c', 'd'])
self.assertTrue(all(value['name'] == TOOL and not value['invalid'] for value in good['calls']))
self.assertEqual([value['arguments']['mode'] for value in good['calls']],
['catalog', 'compare', 'changes', 'metrics'])
def test_real_sdk_still_rejects_a_final_incomplete_argument_object(self):
data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertTrue(result['calls'][0]['invalid'])
def test_real_sdk_keeps_argument_error_and_subsequent_independent_call(self):
data = wire([frame(call(0, 'bad', '{"mode":', TOOL)),
frame(call(0, 'next', '{"mode":"query"}', TOOL)), frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(result['terminal'], 'done')
self.assertEqual([item['id'] for item in result['calls']], ['bad', 'next'])
self.assertTrue(result['calls'][0]['invalid'])
self.assertFalse(result['calls'][1]['invalid'])
self.assertEqual(result['calls'][1]['arguments'], {'mode': 'query'})
def test_real_sdk_receives_the_advertised_name_for_observed_mimo_alias(self):
data = wire([frame(call(0, 'mimo_alias', '', 'quality__quality_data')),
frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': '{"mode":"query"}'}}),
frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(result['terminal'], 'done')
self.assertEqual(result['calls'], [{'id': 'mimo_alias', 'name': TOOL,
'arguments': {'mode': 'query'}, 'invalid': False}])
if __name__ == '__main__':
unittest.main()