统一:DSH 智能分析、Top100 与图表静默恢复

接入 DSH 专用运行时和对话界面,保留模型原始回答,支持会话分页、连续追问与显式报告。
补齐组合筛选、各维度 Top100、分组查询到排名图的转换,图表故障静默省略并基于保存数据自动恢复。
优化报告风格、图表顺序和表格换行,更新接入说明及构建输入。

验证:独立目录前后端构建通过;118 项 Java、102 项前端、102 项图表、14 项 DSH 应用和 13 项组合筛选检查通过。
仅提交相关源码、锁定依赖与测试;不包含凭据、会话、业务数据或本地产物。
This commit is contained in:
2026-11-12 11:57:41 +08:00
parent cc8832c8e5
commit f1a2fbd494
166 changed files with 26281 additions and 2248 deletions
+274
View File
@@ -0,0 +1,274 @@
"""Synthetic scale benchmark in a disposable database and a separate loopback API.
The live release and data are never modified. Replicas retain the sample's 15
source periods and full JSON column widths; they are not real annual business data.
"""
from __future__ import annotations
import argparse
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import http.cookiejar
import json
import math
import os
import re
from pathlib import Path
import secrets
import signal
import socket
import subprocess
import sys
import time
from urllib.parse import urlencode
from urllib.request import Request, build_opener, HTTPCookieProcessor
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup
import project
import managed_test_resources as test_resources
OUT = ROOT / 'output/completion'
def ident(value):
if not value.replace('_', '').isalnum():
raise ValueError('Invalid isolated database identifier')
return '`' + value + '`'
def sql(statement, database=None, empty=False):
create = re.fullmatch(r'CREATE DATABASE `?([a-z0-9_]+)`? CHARACTER SET [a-z0-9_]+(?: COLLATE [a-z0-9_]+)?;?', statement.strip(), flags=re.I)
if create:
return test_resources.create(ROOT, create[1], backup.client, statement, empty=empty)
drop = re.fullmatch(r'DROP DATABASE(?: IF EXISTS)? `?([a-z0-9_]+)`?;?', statement.strip(), flags=re.I)
if drop:
return test_resources.drop(ROOT, drop[1], backup.client)
result = backup.client(statement, database)
user = re.fullmatch(r"DROP USER(?: IF EXISTS)? '([a-z0-9_]+)'@'127\.0\.0\.1';?", statement.strip(), flags=re.I)
if user:
test_resources.user_dropped(ROOT, user[1])
return result
def clone_database(source, target):
tables = sql('SHOW TABLES;', source).splitlines()
for table in tables:
sql(f'CREATE TABLE {ident(target)}.{ident(table)} LIKE {ident(source)}.{ident(table)};')
if table not in ('fact_black', 'fact_bright'):
sql(f'INSERT INTO {ident(target)}.{ident(table)} SELECT * FROM {ident(source)}.{ident(table)};')
# A copied database must never resume the live installation's pending jobs.
for table in ('task', 'import_run'):
if table in tables:
sql(f"UPDATE {ident(table)} SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING','RETRY_WAIT','CANCEL_REQUESTED');", target)
for table in ('backup_plan', 'quality_inspection_strategy'):
if table in tables:
sql(f'UPDATE {ident(table)} SET active=FALSE;', target)
if 'analysis_turn' in tables and 'status' in [row.split('\t')[0] for row in sql('SHOW COLUMNS FROM analysis_turn;', target).splitlines()]:
sql("UPDATE analysis_turn SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING');", target)
return tables
def seed_facts(source, target, stage, count):
table = 'fact_' + stage
columns = [line.split('\t')[0] for line in sql(f'SHOW COLUMNS FROM {ident(table)};', source).splitlines()]
columns.remove('id')
# Replicate only the unique builds in the selected current release. Including
# older transform builds would inflate table size without exercising the
# promised number of query-visible records.
selected = "EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id)"
seed_count = int(sql(f'SELECT COUNT(*) FROM {ident(table)} f WHERE {selected};', source).strip())
if not seed_count:
raise RuntimeError('Source stage has no seed rows')
offset = int(sql(f'SELECT MAX(physical_row_no)+1 FROM {ident(table)};', source).strip())
repeats = math.ceil(count / seed_count)
if offset * (repeats + 1) > 2_147_483_647:
raise ValueError('Synthetic row identity exceeds the supported integer range')
sql('CREATE TEMPORARY TABLE scale_numbers (n INT PRIMARY KEY);' +
'INSERT INTO scale_numbers VALUES ' + ','.join(f'({i})' for i in range(repeats)) + ';' +
f'INSERT INTO {ident(table)} (' + ','.join(ident(c) for c in columns) + ') SELECT ' +
','.join(f'f.{ident(c)} + n.n*{offset}' if c == 'physical_row_no' else f'f.{ident(c)}' for c in columns) +
f' FROM {ident(source)}.{ident(table)} f CROSS JOIN scale_numbers n WHERE ' +
selected.replace('FROM release_member', 'FROM ' + ident(source) + '.release_member').replace('JOIN current_release', 'JOIN ' + ident(source) + '.current_release') +
f' ORDER BY n.n,f.id LIMIT {int(count)};', target)
return int(sql(f'SELECT COUNT(*) FROM {ident(table)};', target).strip())
class BenchApi:
def __init__(self, port, config):
self.base = f'http://127.0.0.1:{port}'
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
token = self.get('/api/auth/csrf')['token']
request = Request(self.base + '/api/auth/login',
data=json.dumps({'username': config['adminUsername'], 'password': config['adminPassword']}).encode(),
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
with self.opener.open(request, timeout=30) as response:
response.read()
def get(self, path, params=None):
request = Request(self.base + path + ('?' + urlencode(params) if params else ''),
headers={'Accept': 'application/json'})
with self.opener.open(request, timeout=180) as response:
return json.load(response)
def percentile(values, fraction):
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None
def run(args):
if not args.run:
print('Plan only. Use --run to create an isolated synthetic database, benchmark, and remove it.')
return 0
if args.black < 1 or args.bright < 1 or args.black + args.bright > 2_000_000:
raise ValueError('Synthetic scale must be between two rows and two million rows')
if args.port == 5281 or not 1 <= args.concurrency <= 20:
raise ValueError('Use a separate loopback port and at most twenty workers')
with socket.socket() as probe:
if probe.connect_ex(('127.0.0.1', args.port)) == 0:
raise RuntimeError('The requested benchmark port is already in use')
OUT.mkdir(parents=True, exist_ok=True)
config = project.config()
tag = datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
database = 'quality_scale_' + tag
account = 'qscale_' + uuid.uuid4().hex[:12]
password = secrets.token_urlsafe(32)
report = {'status': 'RUNNING', 'kind': 'SYNTHETIC_SAMPLE_REPLICATION',
'sourcePeriods': '15 provided black/bright periods retained; not a full real year',
'requestedRows': {'black': args.black, 'bright': args.bright},
'concurrency': args.concurrency, 'database': database,
'liveDatabaseModified': False, 'cleanup': {'database': False, 'account': False, 'process': False}}
result_path = OUT / 'scale-benchmark.json'
process = None
created_db = created_user = False
started = time.monotonic()
try:
report['livePointerBefore'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
sql(f'CREATE DATABASE {ident(database)} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
created_db = True
clone_database(config['mysqlDatabase'], database)
report['rows'] = {stage: seed_facts(config['mysqlDatabase'], database, stage, count)
for stage, count in (('black', args.black), ('bright', args.bright))}
report['currentReleaseRows'] = {stage: int(sql("SELECT COUNT(*) FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id);", database).strip()) for stage in ('black', 'bright')}
if report['currentReleaseRows'] != report['rows']:
raise RuntimeError('Not all synthetic rows are visible in the benchmark release')
existing_tables = set(sql('SHOW TABLES;', database).splitlines())
# A copied 750-row materialization must never stand in for the 800k facts.
for materialization in ('metric_rollup', 'metric_rollup_build'):
if materialization in existing_tables:
sql('DELETE FROM ' + ident(materialization) + ';', database)
sql(f"CREATE USER '{account}'@'127.0.0.1' IDENTIFIED BY '{password}';")
created_user = True
test_resources.attach_user(ROOT, database, account)
sql(f"GRANT ALL PRIVILEGES ON {ident(database)}.* TO '{account}'@'127.0.0.1';")
jars = [p for p in (ROOT / 'backend/target').glob('*.jar') if not p.name.endswith('.original')]
if not jars:
raise RuntimeError('Build the backend before benchmarking')
jar = max(jars, key=lambda p: p.stat().st_mtime)
env = project.environment()
env.update({'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{config["mysqlPort"]}/{database}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': account, 'MYSQL_PASSWORD': password, 'PORT': str(args.port), 'SERVER_PORT': str(args.port),
'QUALITY_GOVERNANCE_BACKGROUND_ENABLED': 'false', 'QUALITY_BACKUP_DATABASE': database})
with (OUT / 'scale-backend.log').open('wb') as log:
process = subprocess.Popen([str(project.RUNTIME / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx768m', '-jar', str(jar)],
cwd=ROOT, env=env, stdout=log, stderr=log, start_new_session=True)
test_resources.attach_child(ROOT, database, process.pid)
preparing = time.monotonic()
project.wait_http(f'http://127.0.0.1:{args.port}/actuator/health', seconds=90)
if 'metric_rollup_build' in set(sql('SHOW TABLES;', database).splitlines()):
deadline = time.monotonic() + 300
while True:
status = sql("SELECT COUNT(DISTINCT rm.transform_build_id),COUNT(DISTINCT rb.build_id),COALESCE(SUM(rb.source_row_count),0) FROM current_release cr JOIN release_member rm ON rm.release_id=cr.release_id LEFT JOIN metric_rollup_build rb ON rb.build_id=rm.transform_build_id WHERE cr.dataset_id='quality';", database).strip().split('\t')
visible, rolled, rolled_rows = map(int, status)
if visible == rolled and rolled_rows == args.black + args.bright:
report['rollupPreparation'] = {'visibleBuilds':visible,'rolledBuilds':rolled,'sourceRows':rolled_rows,'seconds':round(time.monotonic()-preparing,4)}
break
if process.poll() is not None or time.monotonic() >= deadline:
raise RuntimeError('Synthetic rollup preparation did not cover all query-visible facts')
time.sleep(.25)
clients = [BenchApi(args.port, config) for _ in range(args.concurrency)]
common = {'year': 2026, 'releaseId': report['livePointerBefore'], 'includeUnassigned': 'true'}
cases = [('/api/metrics', {**common, 'view': view}) for view in ('overall', 'department', 'group', 'product')]
cases.extend(('/api/analytics/top', {**common, 'stage': stage, 'dimension': dimension, 'limit': 20})
for stage in ('black', 'bright') for dimension in ('category', 'product'))
def verify_visible_counts(result):
if 'summary' in result:
coverage = result.get('summary', {}).get('coverage', {})
valid = coverage.get('blackRows') == args.black and coverage.get('brightRows') == args.bright
else:
months = result.get('coverage', {}).get('sourceMonths', [])
valid = (sum(row.get('blackRows', 0) for row in months) == args.black
and sum(row.get('brightRows', 0) for row in months) == args.bright
and result.get('cumulativeThresholdApplied') is False and bool(result.get('resultId')))
if not valid: raise RuntimeError('The benchmark did not aggregate all requested synthetic records')
cold = []
for path, params in cases:
tick = time.monotonic()
result = clients[0].get(path, params)
if 'summary' not in result and 'resultId' not in result:
raise RuntimeError('Benchmark endpoint did not return metric data')
verify_visible_counts(result)
cold.append({'view': params.get('view', 'TOP_' + params.get('stage','') + '_' + params.get('dimension','')),
'seconds': round(time.monotonic() - tick, 4), 'rows': len(result.get('rows', []))})
report['firstQueryMeasurements'] = cold
# MySQL process/buffer state is shared with the local installation, so this
# is a first API query, not a claim that the physical cache was cold.
def worker(index):
timings = []
for n in range(args.rounds):
path, params = cases[(index + n) % len(cases)]
tick = time.monotonic()
result = clients[index].get(path, params)
if 'summary' not in result and 'resultId' not in result:
raise RuntimeError('Concurrent query returned no summary')
verify_visible_counts(result)
timings.append(time.monotonic() - tick)
return timings
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
timings = [item for group in pool.map(worker, range(args.concurrency)) for item in group]
report['concurrentSummary'] = {'requests': len(timings), 'p50Seconds': round(percentile(timings, .5), 4),
'p95Seconds': round(percentile(timings, .95), 4), 'maxSeconds': round(max(timings), 4),
'initialTwoSecondTargetMet': percentile(timings, .95) <= 2}
report['firstQueryTwoSecondTargetMet'] = all(item['seconds'] <= 2 for item in cold)
report['status'] = 'PASS' if report['concurrentSummary']['initialTwoSecondTargetMet'] and report['firstQueryTwoSecondTargetMet'] else 'PERFORMANCE_TARGET_NOT_MET'
report['limits'] = ['Synthetic replicas do not prove real annual data quality', 'No mixed AI/import concurrency in this run',
'The host MySQL buffer pool was not flushed', 'Production environment and business acceptance remain separate']
except Exception as failure:
report['status'] = 'ERROR'
report['errorType'] = type(failure).__name__
# Database/HTTP clients can include credentials in exception details.
report['error'] = 'Isolated benchmark failed; inspect the private local log without publishing credentials.'
finally:
if process is not None:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try: process.wait(timeout=30)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
report['cleanup']['process'] = True
if created_db:
sql(f'DROP DATABASE {ident(database)};')
report['cleanup']['database'] = True
if created_user:
sql(f"DROP USER '{account}'@'127.0.0.1';")
report['cleanup']['account'] = True
report['livePointerAfter'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
report['livePointerUnchanged'] = report.get('livePointerBefore') == report['livePointerAfter']
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
result_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(json.dumps({key: report.get(key) for key in ('status', 'rows', 'concurrentSummary', 'cleanup', 'livePointerUnchanged', 'elapsedSeconds')}, ensure_ascii=False))
return 0 if report['status'] == 'PASS' and report['livePointerUnchanged'] else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--run', action='store_true')
parser.add_argument('--black', type=int, default=500_000)
parser.add_argument('--bright', type=int, default=300_000)
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--rounds', type=int, default=3)
parser.add_argument('--port', type=int, default=5283)
sys.exit(run(parser.parse_args()))
+152
View File
@@ -0,0 +1,152 @@
"""Private, disposable local API runtime for integration acceptance tests."""
from contextlib import contextmanager
import http.cookiejar
import json
import os
from pathlib import Path
import secrets
import signal
import socket
import subprocess
import sys
import uuid
from urllib.error import HTTPError
from urllib.request import Request, build_opener, HTTPCookieProcessor
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
import project
import backup
import managed_test_resources as test_resources
sys.path.insert(0, str(ROOT / 'verification'))
from benchmark_scale import clone_database, ident, sql
class LocalApi:
def __init__(self, port, username=None, password=None):
if port not in (5281, 5283, 5284, 5285, 5287):
raise ValueError('Only fixed project loopback ports are allowed')
self.base = f'http://127.0.0.1:{port}'
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
self.csrf = None
if username is not None:
status, response, _ = self.call('/api/auth/csrf')
if status != 200: raise RuntimeError('CSRF initialization failed')
self.csrf = response['token']
status, _, _ = self.call('/api/auth/login', {'username': username, 'password': password})
if status != 200: raise RuntimeError('Local test authentication failed')
_, response, _ = self.call('/api/auth/csrf')
self.csrf = response['token']
def call(self, path, payload=None, method=None, binary=False):
if not path.startswith('/api/') or '://' in path:
raise ValueError('Only local API paths are allowed')
headers = {'Accept': 'application/json'}
if self.csrf: headers['X-XSRF-TOKEN'] = self.csrf
if payload is not None: headers['Content-Type'] = 'application/json'
request = Request(self.base + path, method=method or ('POST' if payload is not None else 'GET'), headers=headers,
data=json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None)
try:
with self.opener.open(request, timeout=180) as response:
data = response.read()
return response.status, data if binary else json.loads(data) if data else {}, dict(response.headers)
except HTTPError as failure:
raw = failure.read()
try: body = json.loads(raw)
except (ValueError, UnicodeDecodeError): body = {'detail': 'Non-JSON error response'}
return failure.code, body, dict(failure.headers)
@contextmanager
def isolated_api(port=5284, fresh=False, evidence_dir=None, current_release_only=False, product_scope=None,
env_overrides=None, auth_credentials=None):
with socket.socket() as probe:
if probe.connect_ex(('127.0.0.1', port)) == 0:
raise RuntimeError('The integration port is already in use')
if product_scope is not None and (not isinstance(product_scope,str) or not product_scope.strip() or len(product_scope)>256):
raise ValueError('Invalid isolated product scope')
allowed_overrides = {'QUALITY_NAO_MODEL_CONFIG_FILE', 'QUALITY_MODEL_CONFIG', 'QUALITY_ARCHIVE_ROOT',
'QUALITY_ANALYSIS_MODEL_CONFIG_FILE', 'QUALITY_ANALYSIS_RUNTIME_SCRIPT',
'QUALITY_ANALYSIS_STATE_DIR',
'SERVER_SERVLET_SESSION_COOKIE_NAME',
'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED', 'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'}
if env_overrides is not None and (not isinstance(env_overrides, dict) or not set(env_overrides).issubset(allowed_overrides)
or any(not isinstance(key, str) or not isinstance(value, str) for key, value in env_overrides.items())):
raise ValueError('Invalid isolated runtime environment override')
if auth_credentials is not None and (not isinstance(auth_credentials, tuple) or len(auth_credentials) != 2
or any(not isinstance(value, str) or not value for value in auth_credentials)):
raise ValueError('Invalid isolated test login credentials')
config = project.config()
database = 'quality_verify_' + uuid.uuid4().hex[:16]
account = 'qverify_' + uuid.uuid4().hex[:12]
password = secrets.token_urlsafe(32)
process = None
db_created = user_created = False
evidence = {'database': database, 'port': port, 'freshInstallation': fresh,
'factScope': ('currentReleaseProduct' if current_release_only else 'product') if product_scope is not None else 'currentRelease' if current_release_only else 'all', 'cleanup': {}}
out = Path(evidence_dir) if evidence_dir is not None else ROOT / 'output/completion'
out.mkdir(parents=True, exist_ok=True)
if evidence_dir is not None:
out.chmod(0o700)
try:
evidence['livePointerBefore'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
sql(f'CREATE DATABASE {ident(database)} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', empty=fresh)
db_created = True
if not fresh:
clone_database(config['mysqlDatabase'], database)
for table in ('fact_black', 'fact_bright'):
predicate = (f" WHERE EXISTS (SELECT 1 FROM {ident(config['mysqlDatabase'])}.release_member rm "
f"JOIN {ident(config['mysqlDatabase'])}.current_release cr ON cr.release_id=rm.release_id "
"AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id)"
if current_release_only else '')
if product_scope is not None:
# Hex encodes a text literal without interpolating caller-controlled SQL.
predicate += (' AND ' if predicate else ' WHERE ') + f"f.product COLLATE utf8mb4_0900_as_cs=CONVERT(0x{product_scope.encode('utf-8').hex()} USING utf8mb4) COLLATE utf8mb4_0900_as_cs"
sql(f'INSERT INTO {ident(database)}.{ident(table)} SELECT f.* FROM {ident(config["mysqlDatabase"])}.{ident(table)} f{predicate};')
sql(f"CREATE USER '{account}'@'127.0.0.1' IDENTIFIED BY '{password}';")
user_created = True
test_resources.attach_user(ROOT, database, account)
sql(f"GRANT ALL PRIVILEGES ON {ident(database)}.* TO '{account}'@'127.0.0.1';")
jar = max((ROOT / 'backend/target').glob('*.jar'), key=lambda p: p.stat().st_mtime)
jar = project.backend_snapshot(jar)
evidence['jarSha256'] = jar.stem
env = project.environment()
env.update({'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{config["mysqlPort"]}/{database}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': account, 'MYSQL_PASSWORD': password, 'PORT': str(port), 'SERVER_PORT': str(port),
'QUALITY_BACKUP_DATABASE': database,
'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(port)})
env.update(env_overrides or {})
login_user, login_password = auth_credentials or (config['adminUsername'], config['adminPassword'])
if auth_credentials is not None:
env.update({'QUALITY_BOOTSTRAP_ADMIN_USERNAME': login_user,
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': login_password})
with (out / ('fresh-install-backend.log' if fresh else 'acceptance-backend.log')).open('wb') as log:
os.chmod(out / ('fresh-install-backend.log' if fresh else 'acceptance-backend.log'), 0o600)
process = subprocess.Popen([str(project.RUNTIME / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx768m', '-jar', str(jar)],
cwd=ROOT, env=env, stdout=log, stderr=log, start_new_session=True)
test_resources.attach_child(ROOT, database, process.pid)
project.wait_http(f'http://127.0.0.1:{port}/actuator/health', seconds=90)
if fresh: test_resources.seal(ROOT, database, backup.client)
api = LocalApi(port, login_user, login_password)
evidence['flywayVersions'] = sql('SELECT version FROM flyway_schema_history WHERE success=1 ORDER BY installed_rank;', database).splitlines()
yield api, database, evidence
finally:
if process is not None:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try: process.wait(timeout=30)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
evidence['cleanup']['process'] = True
if db_created:
sql(f'DROP DATABASE {ident(database)};')
evidence['cleanup']['database'] = True
if user_created:
sql(f"DROP USER '{account}'@'127.0.0.1';")
evidence['cleanup']['account'] = True
evidence['livePointerAfter'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
evidence['livePointerUnchanged'] = evidence.get('livePointerBefore') == evidence['livePointerAfter']
evidence_path = out / ('fresh-install-isolation.json' if fresh else 'acceptance-isolation.json')
evidence_path.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
evidence_path.chmod(0o600)
+1 -1
View File
@@ -209,7 +209,7 @@ class MixedHistoryReportTest(unittest.TestCase):
]
report = build_agent_report([comparison, *investigations], context, question)
rendered = '\n'.join(report['sections'])
self.assertIn('原表值全部匹配且核验完整', rendered)
self.assertIn('所选原表值与已存储原始值全部匹配且核验完整', rendered)
self.assertIn('已核验205行/应核验205行', rendered)
self.assertIn('共12条', rendered)
self.assertIn('差异记录', rendered)
@@ -0,0 +1,125 @@
"""Engine pinning, provider tool boundaries and shared formal completion."""
import io
import json
import sys
import tempfile
import types
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT/'scripts'), str(ROOT/'verification')]
from quality_analysis_runtime import engine_for_request, run_turn
from nao_agent_contract import apply_platform_tools
import test_native_run as native_fixture
class EngineSelectionTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
self.addCleanup(self.directory.cleanup)
self.settings = Path(self.directory.name)/'model.json'
def test_old_config_defaults_to_nao_and_new_config_can_select_dsh(self):
self.assertEqual(engine_for_request({}, settings_file=self.settings), 'NAO')
self.settings.write_text(json.dumps({'runtimeEngine':'DSH'}))
self.assertEqual(engine_for_request({}, settings_file=self.settings), 'DSH')
def test_queued_engine_survives_later_config_change(self):
self.settings.write_text(json.dumps({'runtimeEngine':'NAO'}))
self.assertEqual(engine_for_request({'runtimeEngine':'DSH'}, settings_file=self.settings), 'DSH')
self.assertEqual(engine_for_request({'context':{'_runtimeEngine':'DSH'}}, settings_file=self.settings), 'DSH')
def test_conflicting_or_arbitrary_engine_is_not_an_executable(self):
for request in ({'runtimeEngine':'DSH','context':{'_runtimeEngine':'NAO'}},
{'runtimeEngine':'dsh'}, {'runtimeEngine':'/bin/sh'},
{'runtimeEngine':True}):
with self.subTest(request=request), self.assertRaises(ValueError):
engine_for_request(request, settings_file=self.settings)
def test_symlinked_settings_are_rejected(self):
target = self.settings.with_name('target.json'); target.write_text('{}')
self.settings.symlink_to(target)
with self.assertRaises(ValueError):engine_for_request({}, settings_file=self.settings)
def test_route_keeps_platform_protocol_and_never_silently_falls_back(self):
request = {'runtimeEngine':'DSH','question':'fixture'}
with patch('nao_quality_runtime.run_turn', side_effect=RuntimeError('DSH unavailable')) as platform:
with self.assertRaises(RuntimeError):run_turn(request, stdout=io.StringIO(), stdin=io.StringIO())
self.assertEqual(platform.call_count,1)
self.assertEqual(platform.call_args.kwargs['runtime_engine'],'DSH')
def test_dsh_provider_cannot_see_shell_fs_story_or_nao_tools(self):
names=['mcp__quality__quality_data','bash','read','mcp_call','story',
'mcp__other__quality_data','mcp__quality__write','working_directory']
payload={'tools':[{'type':'function','function':{'name':name}} for name in names],
'tool_choice':{'type':'function','function':{'name':'bash'}}}
value=apply_platform_tools(payload, native_agent=True, runtime_engine='DSH')
self.assertEqual([x['function']['name'] for x in value['tools']],['mcp__quality__quality_data'])
self.assertNotIn('tool_choice',value)
class DshFormalCompletionTests(unittest.TestCase):
# Reuse the established signed-evidence fixture, not the NAO implementation.
setUp = native_fixture.NativeRunTests.setUp
invoke = native_fixture.NativeRunTests.invoke
native = native_fixture.NativeRunTests.native
def configure_dsh(self):
self.request['runtimeEngine']='DSH'
self.dsh=Mock(side_effect=self.native)
module=types.ModuleType('dsh_agent_runtime'); module.run_dsh_agent=self.dsh
self.enterContext(patch.dict(sys.modules,{'dsh_agent_runtime':module}))
self.synthesis.side_effect=None
self.synthesis.return_value={'answer':'正式证据支持的答复。','evidenceSufficient':True,
'finishReason':'stop','missingEvidence':[],'titleRequired':False}
self.exploration=self.enterContext(patch('nao_native_exploration.NativeExplorationTools'))
def test_dsh_never_starts_nao_and_keeps_java_proof_and_visual_completion(self):
self.configure_dsh(); result=self.invoke()
self.manager.ensure_instance.assert_not_called(); self.auth.assert_not_called()
self.sse.assert_not_called(); self.dsh.assert_called_once()
self.assertEqual(result['engine'],'DSH')
self.assertEqual(result['status'],'COMPLETED')
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'verified_composition')
self.assertEqual(result['conclusionProtocolQueryIds'],['query-current','query-compare','query-group'])
self.assertEqual(result['answer'],'正式证据支持的答复。')
self.assertEqual(self.exploration.call_args.kwargs['identity']['releaseId'],self.request['releaseId'])
def test_loop_stop_does_not_publish_a_rejected_narrative_as_completed(self):
self.configure_dsh()
self.synthesis.return_value={'answer':'保留已核实数据,部分结论待核查。',
'evidenceSufficient':False,'finishReason':'stop','missingEvidence':['正文与明细不一致']}
result=self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertFalse(result['answerQuality']['narrativeComplete'])
self.assertIn('正文与明细不一致',result['answerQuality']['missingEvidence'])
def test_cancel_during_loop_has_no_composition_or_late_public_evidence(self):
self.configure_dsh(); self.cancel_during_native=True
result=self.invoke()
self.assertEqual(result['status'],'CANCELLED')
self.synthesis.assert_not_called()
self.assertEqual(result['answer'],''); self.assertEqual(result['queryEvidenceRefs'],[])
def test_synthesis_failure_preserves_partial_state_without_publishing_draft(self):
from nao_evidence_finalizer import FinalizationUnavailable
self.configure_dsh(); self.synthesis.side_effect=FinalizationUnavailable('fixture')
result=self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'rejected')
self.assertNotEqual(result['answer'],self.accumulator.answer)
def test_no_signed_evidence_cannot_publish_an_unchecked_dsh_draft(self):
self.configure_dsh(); self.documents=[]
self.accumulator.answer='TOP5全部属于缺角,合格率99%。'
result=self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'rejected')
self.assertNotIn('99%',result['answer'])
self.assertNotIn('全部属于缺角',result['answer'])
self.synthesis.assert_not_called()
if __name__=='__main__':unittest.main()
+8 -8
View File
@@ -400,8 +400,8 @@ class AnalysisVisualsTest(unittest.TestCase):
chart = build_visuals([evidence(result)], "哪些产品报废最多", "turn-top") ["visuals"][0]
self.assertEqual(chart["metric"], "current_scrap_weight")
self.assertEqual(chart["requiredFields"], ["black_scrap_weight"])
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(10)])
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(12)])
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 12, "truncated": False})
self.assertNotIn("share", json.dumps(chart, ensure_ascii=False))
def test_quality_top_categories_require_stage_canonical_source_fields(self):
@@ -642,7 +642,7 @@ class AnalysisVisualsTest(unittest.TestCase):
paired_totals["pairedRankingComplete"] = True
self.assertEqual(build_visuals([evidence(paired_totals)], "画图比较亮片来源具体缺陷增量", "turn-mixed")["visualState"], "unavailable")
def test_fine_defect_chart_limits_to_ten_and_authorizes_only_displayed_fields(self):
def test_fine_defect_chart_keeps_requested_rows_and_authorizes_only_displayed_fields(self):
keys = ["责废尺寸重量", "责废缺角重量", "责废垂直度重量", "责废刀丝重量", "责废烂毛重量",
"责废开裂重量", "责废其他重量", "料废尺寸重量", "料废缺角重量", "料废开裂重量",
"料废烂毛重量", "料废其他重量"]
@@ -656,8 +656,8 @@ class AnalysisVisualsTest(unittest.TestCase):
"totalRows": len(keys), "truncated": False, "rankingComplete": True,
}
chart = build_visuals([evidence(result)], "画图看黑片来源具体缺陷排名", "turn-ten")["visuals"][0]
self.assertEqual(len(chart["categories"]), 10)
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
self.assertEqual(len(chart["categories"]), 12)
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 12, "truncated": False})
self.assertEqual(chart["requiredFields"], [item["key"] for item in chart["categories"]])
def test_multi_query_trend_requires_same_release_and_filters_and_keeps_gaps_null(self):
@@ -910,9 +910,9 @@ class AnalysisVisualsTest(unittest.TestCase):
for index in range(15)
], totalRows=15, returnedRows=15, truncated=False)
chart = build_visuals([evidence(result)], "画产品报废增量排名", "turn-13")["visuals"][0]
self.assertEqual(len(chart["categories"]), 10)
self.assertTrue(chart["coverage"]["truncated"])
self.assertEqual(chart["series"][0]["values"], list(map(float, range(10))))
self.assertEqual(len(chart["categories"]), 15)
self.assertFalse(chart["coverage"]["truncated"])
self.assertEqual(chart["series"][0]["values"], list(map(float, range(15))))
if __name__ == "__main__":
+335
View File
@@ -0,0 +1,335 @@
"""Pinned-runtime tests use authored local SSE, never a model or business credential."""
import json
import hashlib
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
import dsh_agent_runtime as runtime
from nao_quality_runtime import ConversationAccumulator
from setup_dsh_runtime import inspect_runtime, runtime_node
class AuthoredModel:
"""A deterministic provider wire fixture which exercises the actual DSH loop."""
def __init__(self, *, stall=False, length=False, transient=False, malformed=False):
self.calls, self.stall, self.length = [], stall, length
self.started = threading.Event()
self.release = threading.Event()
owner = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_args):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
owner.calls.append(body)
owner.started.set()
if transient and len(owner.calls) == 1:
encoded = b'{"error":{"message":"authored temporary failure"}}'
self.send_response(503)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
return
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.send_header('Connection', 'close')
self.end_headers()
def send(delta, finish=None):
value = {'id': 'authored-local-only', 'object': 'chat.completion.chunk', 'created': 0,
'model': 'authored-check', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]}
self.wfile.write(('data: ' + json.dumps(value, ensure_ascii=False) + '\n\n').encode())
self.wfile.flush()
try:
send({'role': 'assistant'})
if owner.stall:
owner.release.wait(20)
return
if not any(message.get('role') == 'tool' for message in body['messages']):
send({'tool_calls': [{'index': 0, 'id': 'authored-quality-query', 'type': 'function',
'function': {'name': runtime.PUBLIC_TOOL, 'arguments':
'{"mode":' if malformed and len(owner.calls) == 1 else '{"mode":"catalog"}'}}]})
send({}, 'tool_calls')
else:
send({'content': '已核实'})
time.sleep(0.4) # SDK notification polling must survive ordinary idle intervals.
send({'content': '质量指标。'})
send({}, 'length' if owner.length else 'stop')
usage = {'id': 'usage', 'choices': [], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'total_tokens': 110}}
self.wfile.write(('data: ' + json.dumps(usage) + '\n\ndata: [DONE]\n\n').encode())
self.wfile.flush()
except (OSError, BrokenPipeError):
pass
class Server(ThreadingHTTPServer):
daemon_threads = True
block_on_close = False
self.server = Server(('127.0.0.1', 0), Handler)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.connection = (f'http://127.0.0.1:{self.server.server_port}/v1', 'authored-test-only', 'authored-check')
def close(self):
self.release.set()
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=1)
class ProjectionTests(unittest.TestCase):
def test_failed_attempt_is_removed_before_retried_commit(self):
events = []
acc = ConversationAccumulator()
projection = runtime._Projection(acc, events.append)
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 1, 'turn': 1, 'step': 1})
projection.frame({'type': 'chunk', 'attemptId': 'a', 'revision': 2, 'index': 0,
'chunk': {'type': 'text-delta', 'index': 0, 'text': '失败草稿'}})
self.assertEqual(acc.answer, '失败草稿')
projection.event({'seq': 0, 'type': 'assistant/attempt', 'data': {'turn': 1, 'step': 1}})
projection.frame({'type': 'end', 'attemptId': 'a', 'revision': 3, 'index': 1,
'outcome': {'kind': 'committed', 'eventType': 'assistant/attempt', 'seq': 0}})
self.assertEqual(acc.answer, '')
self.assertFalse(any(entry.get('text') == '失败草稿' for entry in acc.timeline))
projection.frame({'type': 'start', 'attemptId': 'b', 'revision': 4, 'turn': 1, 'step': 1})
projection.frame({'type': 'chunk', 'attemptId': 'b', 'revision': 5, 'index': 0,
'chunk': {'type': 'text-delta', 'index': 0, 'text': '新答复'}})
projection.event({'seq': 1, 'type': 'assistant/message', 'data': {'turn': 1, 'step': 1,
'message': {'content': [{'type': 'text', 'text': '已提交答复'}]}}})
self.assertEqual(acc.answer, '已提交答复')
self.assertNotIn('失败草稿', json.dumps(acc.timeline, ensure_ascii=False))
self.assertNotIn('失败草稿', json.dumps(acc.timeline_segments, ensure_ascii=False))
def test_unknown_tool_and_stream_gap_fail_closed(self):
projection = runtime._Projection(ConversationAccumulator(), lambda _value: None)
with self.assertRaises(ValueError):
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 2, 'turn': 1, 'step': 1})
with self.assertRaises(ValueError):
projection.event({'seq': 0, 'type': 'tool/call', 'data': {'name': 'shell'}})
def test_identity_and_versions_are_pinned(self):
lock = json.loads((ROOT / 'runtime/dsh/vendor/python-sdk/source-lock.json').read_text())
self.assertEqual(lock['commit'], 'fcc0ad462172522aaf5c93a7b2e01f6b9f0b0159')
manifest = json.loads((ROOT / 'runtime/dsh/package.json').read_text())
self.assertEqual(manifest['dependencies']['@deepseek-ai/dsh'], '0.2.2-rc.1')
@unittest.skipUnless(inspect_runtime()['ready'], 'requires installed pinned runtime and macOS sandbox')
class RealDshTests(unittest.TestCase):
def setUp(self):
self.folder = tempfile.TemporaryDirectory(prefix='dsh-real-verification-')
self.addCleanup(self.folder.cleanup)
self.root = Path(self.folder.name).resolve()
self.events = []
self.tool_calls = []
self.authorizations = 0
self.cancel = threading.Event()
def run_agent(self, model, *, tool=None, deadline=None, authorization=None, initial=None):
self.addCleanup(model.close)
from nao_quality_tools import tool_definition
def authorize():
self.authorizations += 1
if authorization is not None:
authorization()
def query(arguments):
self.tool_calls.append(arguments)
return {'kind': 'quality_catalog', 'evidenceRef': 'QA1B2C3D4', 'message': '正式证据已核实'}
result = runtime.run_dsh_agent(actor_root=self.root, connection=model.connection,
question='/ordinary-user-input', context={'_agentTask': {}}, task={}, identity={'executionId': 'test-current'},
turn_id=str(uuid.uuid4()), cancel_event=self.cancel, emit=self.events.append,
tool_handler=tool or query, authorize=authorize, tool_definition=tool_definition(),
deadline=deadline or time.monotonic() + 40, initial_accumulator=initial)
self.summary = json.loads(next((self.root / 'private/dsh').glob('*/summary.json')).read_text())
return result
def test_real_loop_mcp_stream_commit_and_isolation_lifecycle(self):
model = AuthoredModel()
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(acc.answer, '已核实质量指标。')
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
self.assertGreaterEqual(self.authorizations, 5)
self.assertTrue(any(e.get('type') == 'answer' and e.get('answer') == '已核实' for e in self.events))
self.assertEqual(acc.tool_calls[0]['tool'], 'mcp_call')
self.assertEqual(acc.tool_calls[0]['sourceTool'], runtime.PUBLIC_TOOL)
self.assertTrue(acc.final_answer_ready)
self.assertEqual(len(model.calls), 2)
for request in model.calls:
self.assertEqual({t['function']['name'] for t in request.get('tools', [])}, {runtime.PUBLIC_TOOL})
self.assertNotIn('dsh_session_log', request)
self.assertEqual(self.summary['stopSucceeded'], True)
def test_initial_read_remains_in_the_real_native_event_stream(self):
seed=ConversationAccumulator()
seed.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'initial-fixture',
'input':{'tool':'quality_data','arguments':{'mode':'query'}}})
seed.feed({'type':'tool-output-available','toolName':'mcp_call','toolCallId':'initial-fixture',
'output':{'kind':'quality_metrics','summary':{'compositeRate':.9557}}})
acc,status,reason,stopped=self.run_agent(AuthoredModel(),initial=seed)
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
self.assertIs(acc,seed)
self.assertEqual(acc.answer,'已核实质量指标。')
self.assertEqual([call['input']['mode'] for call in acc.tool_calls],['query','catalog'])
self.assertTrue(all(call['status']=='completed' for call in acc.tool_calls))
self.assertEqual(len({event['sequence'] for event in acc.timeline}),len(acc.timeline))
def test_real_cancel_revokes_capability_and_reaps_process_tree(self):
model = AuthoredModel(stall=True)
def cancel_after_start():
model.started.wait(15)
self.cancel.set()
thread = threading.Thread(target=cancel_after_start, daemon=True)
thread.start()
began = time.monotonic()
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('CANCELLED', 'user_cancel', True), self.summary)
self.assertLess(time.monotonic() - began, 15)
self.assertEqual(self.summary['toolCalls'], 0)
def test_real_output_limit_cannot_be_success(self):
acc, status, reason, stopped = self.run_agent(AuthoredModel(length=True))
self.assertIn(status, ('PARTIAL', 'FAILED'))
self.assertNotEqual(reason, 'stop')
self.assertTrue(stopped)
def test_real_transient_failure_retries_once_without_duplicate_text(self):
model = AuthoredModel(transient=True)
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(len(model.calls), 3)
self.assertEqual(acc.answer, '已核实质量指标。')
self.assertGreater(self.summary['modelTokensCharged'], 220)
self.assertEqual(self.summary['modelRequests'], 3)
def test_malformed_tool_response_gets_one_native_retry_without_execution_or_rewriting(self):
model = AuthoredModel(malformed=True)
acc, status, reason, stopped = self.run_agent(model)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertEqual(len(model.calls), 3)
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
self.assertEqual(acc.answer, '已核实质量指标。')
def test_real_deadline_stops_a_silent_model_independent_of_sdk_wait(self):
began = time.monotonic()
acc, status, reason, stopped = self.run_agent(AuthoredModel(stall=True), deadline=began + 3)
self.assertEqual((status, reason, stopped), ('FAILED', 'timeout', True), self.summary)
self.assertLess(time.monotonic() - began, 13)
def test_authorization_watchdog_stops_and_records_only_safe_failure_metadata(self):
model = AuthoredModel(stall=True)
def authorize():
if model.started.is_set():
raise RuntimeError('private-authorization-detail')
acc, status, reason, stopped = self.run_agent(model, authorization=authorize)
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_unavailable', True))
self.assertEqual(len(model.calls), 1)
self.assertEqual(self.summary['authorizationFailure'], {
'errorType': 'RuntimeError', 'httpStatus': None, 'retryable': False})
self.assertNotIn('private-authorization-detail', json.dumps(self.summary))
def test_authorization_rejection_stops_without_claiming_a_provider_failure(self):
from nao_quality_runtime import RuntimeErrorSafe
model = AuthoredModel(stall=True)
def authorize():
if model.started.is_set():
raise RuntimeErrorSafe('private-refusal-detail', http_status=403)
_, status, reason, stopped = self.run_agent(model, authorization=authorize)
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_rejected', True))
self.assertEqual(self.summary['authorizationFailure']['httpStatus'], 403)
self.assertNotIn('private-refusal-detail', json.dumps(self.summary))
def test_handled_tool_error_does_not_overturn_native_completion(self):
from nao_quality_tools import QualityToolError
def reject(_arguments):
raise QualityToolError('当前范围未授权。')
acc, status, reason, stopped = self.run_agent(AuthoredModel(), tool=reject)
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
self.assertTrue(acc.tool_error)
self.assertEqual(acc.tool_calls[0]['status'], 'failed')
self.assertIn('当前范围未授权', acc.tool_calls[0]['outputSummary'])
def test_no_data_is_a_normal_tool_outcome_without_zero_metrics(self):
acc,status,reason,stopped=self.run_agent(AuthoredModel(),tool=lambda args:{
'kind':'quality_no_data','status':'no_data','summary':None,'rows':[],
'message':'当前期间没有已发布来源数据。'})
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True))
self.assertFalse(acc.tool_error)
self.assertEqual(acc.tool_calls[0]['status'],'completed')
self.assertIn('暂无数据',acc.tool_calls[0]['outputSummary'])
def test_process_cleanup_refuses_a_changed_process_identity(self):
marker = uuid.uuid4().hex
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
path = self.root / 'process.json'
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': 'wrong'}))
self.assertFalse(runtime._stop_registered(path))
self.assertIsNone(process.poll())
finally:
process.kill()
process.wait(timeout=3)
def test_stop_all_reaps_only_a_matching_persisted_orphan_group(self):
marker = uuid.uuid4().hex
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
root = self.root / 'users'
path = root / 'actor/sessions/session/private/dsh' / marker / 'process.json'
path.parent.mkdir(parents=True)
fingerprint = runtime._process_fingerprint(process.pid)
self.assertIn(marker, fingerprint)
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker,
'fingerprint': hashlib.sha256(fingerprint.encode()).hexdigest()}))
result = runtime.stop_all_dsh(users_root=root)
self.assertEqual(result, {'stopped': 1, 'unconfirmed': 0})
process.wait(timeout=3)
self.assertEqual(json.loads(path.read_text())['state'], 'reaped')
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=3)
def test_os_sandbox_denies_business_files_other_ports_and_shell(self):
# Probe the actual generated OS policy, independent of model tool hiding.
secret = self.root / 'business-secret.txt'
secret.write_text('verification sentinel')
run = self.root / 'isolated'
run.mkdir()
for child in ('home', 'workspace', 'tmp'):
(run / child).mkdir()
policy = runtime._sandbox(run, 9)
code = '''
const fs=require('node:fs'); const cp=require('node:child_process');
let read=false,write=false;try{fs.readFileSync(process.argv[1]);read=true}catch{}
try{fs.writeFileSync(process.argv[1], 'bad');write=true}catch{}
const spawned=cp.spawnSync('/bin/sh',['-c','exit 0']);
let network=false;try{await fetch('http://127.0.0.1:10',{signal:AbortSignal.timeout(1000)});network=true}catch{}
console.log(JSON.stringify({read,write,shell:!spawned.error,network}));
'''
result = subprocess.run(['/usr/bin/sandbox-exec', '-f', str(policy), str(runtime_node()),
'--input-type=commonjs', '-e', '(async()=>{' + code + '})()', str(secret)],
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(json.loads(result.stdout), {'read': False, 'write': False, 'shell': False, 'network': False})
self.assertEqual(secret.read_text(), 'verification sentinel')
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,195 @@
import copy
import json
import sys
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT/'scripts'), str(Path(__file__).parent)]
from dsh_row_selection import selection_hint, select_rows, ranking_chart, validate_selection
from dsh_model_input import canonical_proposal, planning_input, compact_tool_result
from nao_agent_task import normalize_task, plan_task
from nao_native_exploration import prepare_step
from nao_quality_tools import QualityToolError, QualityTools, tool_definition
from test_nao_agent_task import task_value
from test_nao_quality_tools import RELEASE
from test_native_exploration import root
QUESTION = '帮我筛选,10月份合格率最低TOP20产品。且重量大于1000KG,并生成报告'
def selection(limit=20, stage='any'):
value = selection_hint(QUESTION)
value['limit'] = limit
value['weightFilter']['stage'] = stage
return value
def population():
rows = [{'key': f'P{i:03d}', 'label': f'产品{i:03d}', 'compositeRate': .5 + i/1000,
'black': {'rate': .99-i/10000, 'totalWeight': 1001 if i % 2 else 10},
'bright': {'rate': .98, 'totalWeight': 1001 if not i % 2 else 10}} for i in range(150)]
rows.insert(0, {'key': 'LOW_WEIGHT', 'compositeRate': .01, 'black': {'totalWeight': 1000}, 'bright': {'totalWeight': 1000}})
return {'kind': 'quality_metrics', 'releaseId': RELEASE, 'view': 'product',
'period': {'year': 2026, 'month': 10}, 'filters': {},
'rows': rows, 'rowCount': len(rows), 'rowsReturned': len(rows), 'rowsTruncated': False}
class AggregateSelectionTests(unittest.TestCase):
def test_captured_failure_is_bound_as_aggregate_condition_not_source_filter(self):
captured = task_value(period={'year':2026,'month':10}, fields=['rate'], stages=['black'], dimensions=['product'], filterUpdates={'extraFilters':[{'key':'totalWeight','operator':'gt','value':1000}]})
captured.update(needsClarification=True, clarification='请确认黑片、亮片还是任一阶段投入重量?')
before = copy.deepcopy(captured)
value = canonical_proposal(captured, question=QUESTION)
task = normalize_task(value, QUESTION, {'_businessCatalog': {'years': [2026]}})
self.assertTrue(task['needsClarification'])
self.assertEqual(task['aggregateSelection']['weightFilter']['value'], 1000)
self.assertNotIn('extraFilters', task['filters'])
self.assertEqual(captured, before)
def test_clarification_can_resume_same_month_top_and_report_request(self):
previous = root()
previous.update(needsClarification=True, aggregateSelection=selection(stage=None), period={'year': 2026, 'month': 10})
context = {'_agentTask': previous}
for answer, expected in [('黑片or亮片', 'any'), ('我的选择:任一阶段重量大于1000KG', 'any'), ('两阶段均满足', 'all'), ('黑片和亮片都要大于1000', 'all'), ('亮片', 'bright'), ('黑片', 'black')]:
fixed = canonical_proposal(task_value(), question=answer, context=context)
self.assertEqual(fixed['aggregateSelection']['weightFilter']['stage'], expected)
self.assertEqual(fixed['aggregateSelection']['requestQuestion'], QUESTION)
self.assertEqual(fixed['period'], {'year': 2026, 'month': 10})
self.assertFalse(fixed['needsClarification'])
hint = planning_input({'question': QUESTION})['aggregateSelectionHint']
self.assertIsNone(hint['weightFilter']['stage'])
def test_weight_only_model_alias_uses_existing_rank_contract_without_losing_conditions(self):
question='2026年10月,按亮片投入重量由高到低列出产品Top100,不生成报告。'
value=task_value(fields=['totalWeight'],stages=['bright'],dimensions=['product'],
aggregateSelection={'dimension':'product','metric':'totalWeight','rateStage':'bright','direction':'desc','limit':100,'requestQuestion':question})
fixed=canonical_proposal(value,question=question)
self.assertNotIn('aggregateSelection',fixed)
self.assertEqual(fixed['fields'],['totalWeight'])
self.assertEqual(fixed['stages'],['bright'])
predicate=copy.deepcopy(value);predicate['aggregateSelection']['weightFilter']={'metric':'totalWeight','operator':'gt','value':1000,'stage':'bright'}
self.assertIn('aggregateSelection',canonical_proposal(predicate,question=question))
def test_filter_precedes_sort_and_top_and_does_not_recalculate_composite(self):
source = population(); before = copy.deepcopy(source)
selected = select_rows(source, selection())
self.assertEqual(selected['eligibleRows'], 150)
self.assertEqual([row['key'] for row in selected['rows']], [f'P{i:03d}' for i in range(20)])
self.assertEqual(selected['rows'][0]['selectedRate'], .5)
self.assertEqual(source, before)
all_stages = select_rows(source, selection(stage='all'))
self.assertEqual(all_stages['returnedRows'], 0)
def test_top100_and_chart_panels_keep_every_rank(self):
source = population(); task = {'aggregateSelection': selection(100)}
view = compact_tool_result(source, task=task)
self.assertEqual(len(view['rows']), 100)
self.assertEqual(view['rowCount'], 151)
self.assertEqual(view['selectionDisplay']['eligibleRows'], 150)
charts = ranking_chart({'result': source, 'evidenceRef': 'Q12345678'}, task, 'test-turn')
self.assertEqual(len(charts), 4)
self.assertEqual(sum(len(chart['categories']) for chart in charts), 100)
self.assertTrue(all(chart['series'][0]['key'] == chart['metric'] for chart in charts))
self.assertEqual(charts[-1]['categories'][-1]['label'], '100. 产品099')
def test_missing_invalid_rows_and_ties_do_not_become_zero_or_fake_rates(self):
source = population()
source['rows'][1]['compositeRate'] = None
source['rows'][2]['compositeRate'] = float('nan')
source['rows'][3]['compositeRate'] = source['rows'][4]['compositeRate']
selected = select_rows(source, selection())
self.assertEqual(selected['missingRateRows'], 2)
self.assertEqual([row['key'] for row in selected['rows'][:2]], ['P002', 'P003'])
source['rowsTruncated'] = True
with self.assertRaises(ValueError): select_rows(source, selection())
def test_limit_bounds_and_batch_scope_keep_root_filters(self):
for count in (0, 101):
with self.assertRaises(ValueError): validate_selection(selection(count))
task = root(); task.update(aggregateSelection=selection(100), comparisonPeriod=None)
args, proposal = prepare_step({'mode': 'query', 'view': 'product'}, task, {'releaseId': RELEASE, 'scopes': []})
self.assertEqual(proposal['node']['dimension'], 'product')
self.assertEqual(args['limit'], 500)
self.assertEqual(args['period'], task['period'])
schema = tool_definition(native_protocol=True, task=task, release_id=RELEASE)['inputSchema']
self.assertIn('product', schema['properties']['view']['enum'])
task.update(stages=['black'], fields=['totalWeight'], dimensions=['product'])
args, proposal = prepare_step({'mode': 'rank', 'dimension': 'product', 'metric': 'totalWeight', 'stage': 'black', 'limit': 100}, task, {'releaseId': RELEASE, 'scopes': []})
self.assertEqual(args['limit'], 100)
with self.assertRaises(QualityToolError): prepare_step({**args, 'limit': 101}, task, {'releaseId': RELEASE, 'scopes': []})
def test_model_clarification_cannot_enter_native_execution(self):
value = task_value(executionMode='native_agent', needsClarification=True,
clarification='重量筛选用黑片、亮片、任一阶段还是两阶段都满足?', clarificationRequest=['黑片','亮片','任一阶段','两阶段均满足'], period={'year': 2026, 'month': 10}, dimensions=['product'])
context = {'_protocol': {'autonomousEnabled': True}, '_businessCatalog': {'years': [2026]}}
with patch('nao_agent_task._model_json', return_value=value):
task = plan_task(base_url='http://test', api_key='test', model='test', question=QUESTION,
context=context, cancel_event=threading.Event(), checks_enabled=False)
self.assertTrue(task['needsClarification'])
self.assertNotEqual(task.get('executionMode'), 'native_agent')
self.assertEqual(task['clarification'], value['clarification'])
self.assertEqual([entry['value'] for entry in task['clarificationRequest']['options']], value['clarificationRequest'])
def test_unrelated_new_tasks_and_additional_ambiguity_are_not_swallowed(self):
previous = root(); previous.update(needsClarification=True, aggregateSelection=selection(stage=None))
context = {'_agentTask': previous}
self.assertIsNone(selection_hint('新任务,查询综合率', context))
value = task_value(needsClarification=True, clarification='产品有多个候选,请选择具体规格。')
fixed = canonical_proposal(value, question='亮片', context=context)
self.assertTrue(fixed['needsClarification'])
self.assertEqual(fixed['clarification'], value['clarification'])
self.assertEqual(selection_hint('改成Top100,亮片', context)['limit'], 100)
def test_unpaginated_metrics_do_not_require_pagination_only_rowcount(self):
raw = {'summary': {'key':'overall','compositeRate': .95}, 'context': {},
'rows':[{'key':'P1','compositeRate':.9}], 'coverage': {}, 'trend':[]}
tools = object.__new__(QualityTools)
with patch.object(tools, '_metrics', return_value=raw) as read:
result = tools._query({'id': RELEASE}, {'year':2026,'month':10}, {}, 'product', 500)
self.assertEqual(result['rowCount'], 1)
self.assertFalse(result['rowsTruncated'])
self.assertIsNone(read.call_args.kwargs['row_limit'])
raw['context']['rowCount'] = 2
with patch.object(tools, '_metrics', return_value=raw):
with self.assertRaises(QualityToolError): tools._query({'id':RELEASE}, {'year':2026,'month':10}, {}, 'product', 500)
def test_product_batch_charts_accept_only_exact_signed_dimension_scope(self):
from analysis_visual_evidence import protocol_visual_result
value = population()
value['protocolEvidence'] = {'nodeId':'products', 'scopeId':'scope-products','queryId':'q-products'}
document = {'mode':'query','releaseId':RELEASE,'result':value,'evidenceRef':'Q12345678'}
task = root(); task.update(period=value['period'], comparisonPeriod=None, comparisonRequired=False,
aggregateSelection=selection(100), dimensions=['product'], fields=['rate','totalWeight'], stages=['black','bright'])
scope = {'nodeId':'products','scopeId':'scope-products','purpose':'metrics','task':{**task,'kind':'metrics'}}
result = protocol_visual_result([document],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':task})
self.assertEqual(result['visualState'], 'ready')
self.assertEqual(sum(len(chart['categories']) for chart in result['visuals']), 100)
changed = copy.deepcopy(document); changed['result']['protocolEvidence']['scopeId'] = 'wrong-scope'
denied = protocol_visual_result([changed],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':task})
self.assertEqual(denied['visuals'], [])
empty=copy.deepcopy(task);empty['aggregateSelection']['weightFilter']['value']=1e12
no_chart=protocol_visual_result([document],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':empty})
self.assertEqual(no_chart['visualState'],'not_requested')
def test_initial_top_comparison_read_is_never_a_single_period_weight_rank(self):
import runpy
initial = runpy.run_path(str(ROOT/'runtime/dsh/application/quality_native_run.py'))['initial_read_arguments']
task = root();task.update(freshQuery=True, dimensions=['product'], fields=['rate','totalWeight'], stages=['bright'])
self.assertEqual(initial(task,'合格率下降最多Top100产品'), {'mode':'compare'})
task['aggregateSelection']=selection(100)
self.assertEqual(initial(task,'合格率下降最多Top100产品'), {'mode':'compare'})
def test_two_period_rate_change_uses_existing_changes_contract(self):
question='比较2026年9月和10月,列出亮片合格率下降最多的产品Top100'
value=task_value(fields=['rate'],stages=['bright'],dimensions=['product'],
aggregateSelection={'dimension':'product','metric':'rate','rateStage':'bright','direction':'asc','limit':100,'requestQuestion':question})
fixed=canonical_proposal(value,question=question)
self.assertNotIn('aggregateSelection',fixed)
self.assertEqual(fixed['period'],{'year':2026,'month':10})
self.assertEqual(fixed['comparisonPeriod'],{'year':2026,'month':9})
if __name__ == '__main__': unittest.main()
+347
View File
@@ -0,0 +1,347 @@
"""Exercise the staged DSH worker contract with only authored local fixtures."""
import importlib
import io
import json
from pathlib import Path
import sys
import tempfile
import threading
from types import SimpleNamespace
import unittest
import uuid
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
from build_dsh_application import build
class DshApplicationTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
temporary = tempfile.TemporaryDirectory(prefix='quality-dsh-application-')
cls.addClassCleanup(temporary.cleanup)
cls.application = Path(temporary.name) / 'worker'
build(cls.application)
candidate = str(cls.application / 'scripts')
sys.path.insert(0, candidate)
cls.addClassCleanup(sys.path.remove, candidate)
names = [path.stem for path in (cls.application / 'scripts').glob('*.py')]
original_modules = {name: sys.modules.pop(name) for name in names if name in sys.modules}
def restore_modules():
for name in names:
sys.modules.pop(name, None)
sys.modules.update(original_modules)
cls.addClassCleanup(restore_modules)
cls.worker = importlib.import_module('quality_analysis_runtime')
cls.context = importlib.import_module('quality_runtime_context')
cls.drafts = importlib.import_module('quality_draft')
cls.reports = importlib.import_module('quality_report_deliverable')
cls.original = importlib.import_module('quality_original_answer')
@staticmethod
def request(question, **changes):
request = {key: str(uuid.uuid4()) for key in ('turnId', 'sessionId', 'actorId', 'releaseId')}
request.update(question=question, runtimeEngine='DSH', accessToken='fixture-only-delegation',
context={'_protocol': {'version': 1, 'attemptId': request['sessionId'],
'autonomousEnabled': True}})
return {**request, **changes}
def test_independent_worker_preserves_delegation_and_publishes_original_report(self):
request = self.request('请输出黑片10月质量对比报告')
task = {'kind': 'analysis', 'executionMode': 'native_agent'}
calls = []
access_files = []
answer = '## 本期质量概况\n\n| 项目 | 本期 |\n| --- | --- |\n| 例示指标 | 42 |'
def delegate(instance, path, **kwargs):
calls.append(path)
self.assertEqual(instance.access_path.stat().st_mode & 0o777, 0o600)
access_files.append(instance.access_path)
if path == '/api/analysis/protocol/task':
self.assertEqual(kwargs['payload'], task)
return {'task': task, 'attemptId': request['sessionId']}
return {}
def execute(**options):
self.assertEqual(options['authority']['task'], task)
self.assertNotIn('accessToken', options['request'])
self.assertEqual(options['connection'][0].split(':')[1], '//127.0.0.1')
self.original.current().capture_native(answer)
options['emit']({'type': 'result', 'status': 'COMPLETED', 'answer': answer,
'agentTask': task, 'reportTitle': '10月黑片质量对比', 'protocolEvidence': [{'queryId': 'fixture'}],
'completionMethod': 'native_autonomous_analysis',
'answerQuality': {'narrativeFactBinding': 'verified_composition'},
'timeline': [{'kind': 'assistant', 'text': '临时草稿'},
{'kind': 'tool_result', 'toolCallId': 'dsh-tool-fixture'}]})
output = io.StringIO()
with (patch('setup_dsh_runtime.inspect_runtime', return_value={'ready': True}),
patch('quality_model_settings.model_configuration', return_value=('https://fixture.invalid/v1', 'fixture-key', 'fixture-model', {})),
patch.object(self.context.DelegatedApi, 'call', delegate),
patch.object(self.worker, '_plan', return_value=task),
patch('quality_native_run.run_native_exploration', side_effect=execute)):
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
self.assertEqual(calls, ['/api/analysis/model-authorization', '/api/analysis/model-authorization',
'/api/analysis/protocol/task'])
self.assertTrue(all(not path.exists() for path in access_files))
result = json.loads(output.getvalue().splitlines()[-1])
self.assertEqual(result['engine'], 'DSH')
self.assertEqual(result['reportMarkdown'], result['answer'])
self.assertEqual(result['reportTitle'], '本期质量概况')
self.assertEqual(result['timeline'], [{'kind': 'tool_result', 'toolCallId': 'dsh-tool-fixture'}])
self.assertNotIn('fixture-only-delegation', output.getvalue())
def test_engine_and_attempt_are_pinned_before_access(self):
requests = (self.request('检查指标', runtimeEngine='legacy'),
self.request('检查指标', context={'_protocol': {'version': 1, 'attemptId': str(uuid.uuid4())}}))
with patch.object(self.context.DelegatedApi, 'call') as delegated:
for request in requests:
output = io.StringIO()
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
self.assertEqual(json.loads(output.getvalue())['status'], 'FAILED')
delegated.assert_not_called()
def test_worker_retains_original_text_when_execution_raises(self):
request = self.request('输出质量报告')
task = {'kind': 'analysis', 'executionMode': 'native_agent'}
body = ' 模型原始内容,后续检查异常也要保留。\n'
def execute(**_options):
self.original.current().capture_native(body)
raise ValueError('fixture validation rejection')
output = io.StringIO()
with (patch('setup_dsh_runtime.inspect_runtime', return_value={'ready': True}),
patch('quality_model_settings.model_configuration', return_value=('https://fixture.invalid/v1', 'fixture-key', 'fixture-model', {})),
patch.object(self.context.DelegatedApi, 'call', return_value={'task': task, 'attemptId': request['sessionId']}),
patch.object(self.worker, '_plan', return_value=task),
patch('quality_native_run.run_native_exploration', side_effect=execute)):
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
result = json.loads(output.getvalue().splitlines()[-1])
self.assertEqual(result['status'], 'FAILED')
self.assertEqual(result['answer'], body)
self.assertEqual(result['modelAnswer'], body)
self.assertEqual(result['reportMarkdown'], body)
def test_native_draft_replaces_and_clears_without_emitting_private_text(self):
frames = []
now = [1.0]
publisher = self.drafts.DraftPublisher(frames.append, clock=lambda: now[0])
text = '10月黑片质量指标已进入核验,正在检查异常变化和对象范围。'
publisher.update('fixture-attempt-id', text)
publisher.update('fixture-attempt-id', text + '第二个片段。')
self.assertEqual(len(frames), 1)
self.assertNotEqual(frames[0]['dshAttemptId'], 'fixture-attempt-id')
now[0] += 0.26
publisher.update('fixture-attempt-id', text + '补充结论。')
self.assertEqual(frames[1]['dshDraft'], text + '补充结论。')
publisher.update('fixture-attempt-id', text + '内部请求归属标记')
self.assertEqual(frames[-1], {'type': 'progress', 'dshDraft': ''})
def test_terminal_status_preserves_native_body_after_failed_attempt(self):
import dsh_agent_runtime
for status in ('PARTIAL', 'FAILED', 'CANCELLED'):
frames = []
body = ' ## 模型的原始判断\n\n98.04%显著低于组合乘积。\n'
with self.original.OriginalAnswer(frames.append) as original:
projection = dsh_agent_runtime._Projection(self.context.ConversationAccumulator(), frames.append)
projection._text('failed-provider-attempt', body)
projection._discard('failed-provider-attempt')
result = {'type': 'result', 'status': status, 'answer': '程序替换文本',
'readingSummary': {'overview': '固定摘要'}, 'stopReason': 'validation_failed'}
original.settle(result)
self.reports.attach(result, '输出质量分析报告')
self.assertEqual(result['answer'], body)
self.assertEqual(result['modelAnswer'], body)
self.assertEqual(result['reportMarkdown'], body)
self.assertNotIn('readingSummary', result)
self.assertEqual(result['answerValidation']['stopReason'], 'validation_failed')
self.assertTrue(any(frame.get('modelAnswer') == body for frame in frames))
def test_bounded_business_blocks_preserve_order_whitespace_and_original_title(self):
frames = []
texts = [' ## 先看这项\n原始数字999.9%。 ', '| 项目 | 结果 |\n| --- | --- |\n| X | 5 |', '最后一段\n']
raw = json.dumps({'reportTitle': '模型原始标题', 'blocks': [
{'role': 'detail', 'text': texts[0]}, {'role': 'core', 'text': texts[1]},
{'role': 'key_evidence', 'text': texts[2]}]}, ensure_ascii=False)
with self.original.OriginalAnswer(frames.append) as original:
self.original.capture(raw)
self.original.capture(json.dumps({'blocks': [{'text': '第二次修复后的替代内容'}]}))
result = {'type': 'result', 'status': 'PARTIAL', 'answer': '正式统计固定回退',
'reportTitle': '程序改写标题', 'answerQuality': {'narrativeComplete': False}}
original.settle(result)
self.assertEqual(result['answer'], '\n\n'.join(texts))
self.assertEqual(result['reportTitle'], '模型原始标题')
self.assertEqual(result['answerOrigin'], 'model_business_text')
def test_business_response_survives_invalid_composition_and_interrupted_finish(self):
import quality_native_answer
import threading
raw_body = '{"blocks":[{"text":"模型未完成的原始回复,格式未闭合'
envelope = json.dumps({'choices': [{'message': {'content': raw_body}, 'finish_reason': 'length'}]})
with self.original.OriginalAnswer(lambda _value: None) as original:
with io.BytesIO(envelope.encode()) as response:
content, _, finish = quality_native_answer._request_model(
connection=('https://fixture.invalid/v1', 'fixture-only-key', 'fixture-model'),
messages=[{'role': 'user', 'content': '查询质量'}], cancel_event=threading.Event(),
opener=lambda *_args, **_kwargs: response, phase='native_synthesis')
result = {'type': 'result', 'status': 'FAILED', 'answer': '固定回退文本'}
original.settle(result)
self.assertEqual(content, raw_body)
self.assertEqual(finish, 'length')
self.assertEqual(result['answer'], raw_body)
def test_review_metadata_does_not_replace_native_answer_and_empty_model_stays_empty(self):
frames = []
with self.original.OriginalAnswer(frames.append) as original:
original.capture_native('原始模型正文')
self.original.capture('{"blocks":[{"text":"复核模型改写"}]}')
result = {'type': 'result', 'status': 'PARTIAL', 'answer': '固定统计摘要'}
original.settle(result)
self.assertEqual(result['answer'], '原始模型正文')
with self.original.OriginalAnswer(frames.append) as original:
self.original.capture('{"reviews":[]}', phase='native_draft_review')
result = {'type': 'result', 'status': 'FAILED', 'answer': '程序生成的失败正文', 'message': '模型连接失败'}
original.settle(result)
self.assertEqual(result['answer'], '')
self.assertEqual(result['modelAnswer'], '')
self.assertEqual(result['message'], '模型连接失败')
def test_report_request_reuses_original_body_for_any_validation_status(self):
self.assertFalse(self.reports.requested('只列产品排名,不生成报告'))
self.assertTrue(self.reports.requested('不分析原因,生成报告'))
self.assertTrue(self.reports.requested('生成一份不推断原因的报告'))
for status in ('COMPLETED', 'PARTIAL', 'FAILED', 'CANCELLED'):
for question in ('生成一份对比报告', '请给我一份排查简报,优先列风险'):
result = {'type': 'result', 'status': status, 'modelAnswer': '## 模型报告\n\n原始正文。',
'answerQuality': {'narrativeComplete': False}, 'protocolEvidence': []}
self.reports.attach(result, question)
self.assertEqual(result['reportMarkdown'], result['modelAnswer'])
result = {'type': 'result', 'status': 'COMPLETED', 'modelAnswer': '原文', 'reportMarkdown': '过期的报告'}
self.reports.attach(result, '不要写报告,先核对质量数据')
self.assertNotIn('reportMarkdown', result)
def test_direct_delivery_never_recomposes_or_requires_query_evidence(self):
runner = importlib.import_module('quality_native_run')
task = {'kind': 'analysis', 'executionMode': 'native_agent', 'filters': {},
'period': {'year': 2026}, 'fields': ['rate'], 'stages': []}
request = self.request('生成质量报告')
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
accumulator = SimpleNamespace(answer='## 模型原文\n\n未经额外核验的原始回答。',
finish_reasons=['stop'], tool_calls=[], timeline=[])
delegate = SimpleNamespace(call=lambda *args, **kwargs: {})
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'private').mkdir()
(root / 'evidence').mkdir()
with (patch('dsh_agent_runtime.run_dsh_agent', return_value=(accumulator, 'COMPLETED', 'stop', True)),
patch('quality_evidence_finalizer.synthesize_native', side_effect=AssertionError('extra model review')),
patch('analysis_visual_evidence.protocol_visual_result', side_effect=ValueError('chart unavailable'))):
result = runner.run_native_exploration(request=request, task=task, authority=authority,
actor_root=root, delegated=delegate, connection=('http://127.0.0.1:1/v1', 'fixture', 'fixture'),
cancel_event=threading.Event(), emit=lambda event: None, prior_evidence_files=frozenset())
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['answer'], accumulator.answer)
self.assertEqual(result['phaseTimingsMs']['synthesis'], 0)
self.assertNotIn('answerQuality', result)
def test_business_data_is_read_before_model_and_seeded_in_the_native_stream(self):
runner = importlib.import_module('quality_native_run')
task = {'kind': 'analysis', 'freshQuery': True, 'executionMode': 'native_agent', 'filters': {},
'period': {'year': 2026, 'month': 8}, 'fields': ['rate'], 'stages': []}
request = self.request('2026年8月综合合格率,一句话')
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
phases = []
def data(_args):
phases.append('read')
return {'kind': 'quality_metrics', 'summary': {'compositeRate': .9557}, 'evidenceRef': 'Q12345678'}
def model(**kwargs):
phases.append('model')
self.assertEqual(kwargs['context']['initialObservation']['summary']['compositeRate'], .9557)
seeded = kwargs['initial_accumulator']
self.assertEqual(seeded.tool_calls[0]['status'], 'completed')
seeded.feed({'type':'text-delta','id':'original','delta':'模型原文:95.57%。'})
return seeded, 'COMPLETED', 'stop', True
with tempfile.TemporaryDirectory() as directory:
root=Path(directory); (root/'private').mkdir(); (root/'evidence').mkdir()
with (patch('quality_native_exploration.NativeExplorationTools.quality_data', side_effect=data),
patch('dsh_agent_runtime.run_dsh_agent', side_effect=model)):
result = runner.run_native_exploration(request=request, task=task, authority=authority,
actor_root=root, delegated=SimpleNamespace(call=lambda *args, **kwargs: {}),
connection=('http://127.0.0.1:1/v1','fixture','fixture'),cancel_event=threading.Event(),
emit=lambda event:None,prior_evidence_files=frozenset())
self.assertEqual(phases,['read','model'])
self.assertEqual(result['answer'],'模型原文:95.57%。')
self.assertEqual(result['toolCount'],1)
def test_all_failed_business_calls_keep_raw_answer_but_do_not_claim_completion(self):
runner = importlib.import_module('quality_native_run')
task = {'kind': 'analysis', 'executionMode': 'native_agent', 'filters': {},
'period': {'year': 2026}, 'fields': ['rate'], 'stages': []}
request = self.request('比较9月和10月')
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
accumulator = SimpleNamespace(answer='原始模型说明:本轮查询没有成功。', finish_reasons=['stop'],
tool_calls=[{'input': {'mode': 'compare'}, 'status': 'failed'} for _ in range(3)], timeline=[])
with tempfile.TemporaryDirectory() as directory:
root = Path(directory); (root/'private').mkdir(); (root/'evidence').mkdir()
with patch('dsh_agent_runtime.run_dsh_agent', return_value=(accumulator, 'COMPLETED', 'stop', True)):
result = runner.run_native_exploration(request=request, task=task, authority=authority,
actor_root=root, delegated=SimpleNamespace(call=lambda *args, **kwargs: {}),
connection=('http://127.0.0.1:1/v1', 'fixture', 'fixture'),
cancel_event=threading.Event(), emit=lambda event: None, prior_evidence_files=frozenset())
self.assertEqual(result['status'], 'FAILED')
self.assertEqual(result['answer'], accumulator.answer)
self.assertEqual(result['stopReason'], 'tool_queries_failed')
def test_unchecked_planning_calls_model_once_without_semantic_review_or_repair(self):
planner = importlib.import_module('quality_agent_task')
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
'_businessCatalog': {'years': [2026], 'latestDataPeriod': '2026-10'}}
value = {'goal': '2026年综合合格率', 'kind': 'metrics', 'freshQuery': True,
'dimensions': [], 'stages': [], 'fields': ['rate'], 'verifySource': False,
'fullDetails': False, 'needsClarification': False, 'filterUpdates': {}, 'period': {'year': 2026}}
with patch.object(planner, '_model_json', return_value=value) as call:
result = planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
question='2026年综合合格率', context=context, cancel_event=threading.Event(),
checks_enabled=False, semantic_review=True)
self.assertEqual(call.call_count, 1)
self.assertEqual(result['executionMode'], 'native_agent')
with patch.object(planner, '_model_json', return_value={'kind': 'invalid'}) as call:
with self.assertRaises(planner.TaskUnavailable):
planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
question='q', context=context, cancel_event=threading.Event(), checks_enabled=False)
self.assertEqual(call.call_count, 1)
def test_direct_agent_planning_does_not_block_presentation_or_grouping_choices(self):
planner = importlib.import_module('quality_agent_task')
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
'_businessCatalog': {'years': [2026], 'latestDataPeriod': {'year': 2026, 'month': 10}}}
# The exact period/trend combination which stopped the real report.
proposal = {'goal': '生成最近月份质量报告', 'kind': 'analysis', 'freshQuery': True,
'dimensions': [], 'stages': ['black', 'bright'],
'fields': ['rate', 'totalWeight', 'scrapWeight', 'scrapRate'],
'verifySource': False, 'fullDetails': False, 'needsClarification': False,
'filterUpdates': {}, 'period': {'year': 2026, 'month': 10},
'trendRequired': True, 'comparisonRequired': False}
for dimensions in ([], ['group']):
with self.subTest(dimensions=dimensions), patch.object(
planner, '_model_json', return_value={**proposal, 'dimensions': dimensions}) as call:
result = planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
question='生成一份最近月份的质量分析报告', context=context,
cancel_event=threading.Event(), checks_enabled=False, semantic_review=True)
self.assertEqual(call.call_count, 1)
self.assertEqual(result['period'], proposal['period'])
self.assertTrue(result['trendRequired'])
self.assertEqual(result['dimensions'], dimensions)
self.assertEqual(result['executionMode'], 'native_agent')
self.assertNotIn('queryPlan', result)
if __name__ == '__main__':
unittest.main()
+80
View File
@@ -0,0 +1,80 @@
import sys
import hashlib
import tempfile
import unittest
import xml.etree.ElementTree as ET
from pathlib import Path
from zipfile import ZipFile
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'verification')]
import verify_dsh_formal as fixture
import isolated_runtime
class DshFormalFixtureTests(unittest.TestCase):
def test_seed_is_three_months_two_stages_two_org_groups_and_six_products(self):
rows = fixture.fixture_rows()
self.assertEqual(len(rows), 36)
self.assertEqual({row.month for row in rows}, {'2026-08', '2026-09', '2026-10'})
self.assertEqual({row.stage for row in rows}, {'black', 'bright'})
self.assertEqual({row.department for row in rows}, set(fixture.DEPARTMENTS))
self.assertEqual({row.group for row in rows}, set(fixture.GROUPS))
self.assertEqual({row.product for row in rows}, set(fixture.PRODUCTS))
for month in fixture.PERIODS:
for stage in ('black', 'bright'):
selected = [row for row in rows if row.month == month and row.stage == stage]
self.assertEqual({row.product for row in selected}, set(fixture.PRODUCTS))
def test_missing_detail_is_explicit_and_formal_rollup_stays_complete(self):
rows = fixture.fixture_rows()
missing = [row for row in rows if row.month == '2026-09' and row.stage == 'black'
and row.product == '合成产品06']
self.assertEqual(len(missing), 1)
self.assertIn('责废尺寸重量', missing[0].raw)
self.assertIsNone(missing[0].raw['责废尺寸重量'])
self.assertGreater(missing[0].components['black_responsibility_scrap'], 0)
self.assertEqual(fixture.fixture_totals(rows, 'black', '2026-10')['rows'], 6)
self.assertEqual(fixture.fixture_totals(rows, 'bright', '2026-10')['rows'], 6)
def test_generated_archive_is_a_hashed_synthetic_xlsx_with_source_fields(self):
rows = [row for row in fixture.fixture_rows()
if row.stage == 'black' and row.month == '2026-09']
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / 'synthetic.xlsx'
digest, size = fixture._write_workbook(path, rows)
self.assertEqual(digest, hashlib.sha256(path.read_bytes()).hexdigest())
self.assertEqual(size, path.stat().st_size)
with ZipFile(path) as archive:
self.assertTrue({'[Content_Types].xml', '_rels/.rels', 'xl/workbook.xml',
'xl/worksheets/sheet1.xml'}.issubset(set(archive.namelist())))
worksheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
cells = [element.text for element in worksheet.iter() if element.tag.endswith('}t')]
self.assertIn('责废尺寸重量', cells)
self.assertIn('责废缺角重量', cells)
def test_source_physical_rows_restart_for_each_source_workbook(self):
values = list(fixture.source_physical_rows(fixture.fixture_rows()))
first_file = [row_no for row, row_no in values if row.stage == 'black' and row.month == '2026-08']
next_file = [row_no for row, row_no in values if row.stage == 'bright' and row.month == '2026-08']
self.assertEqual(first_file, list(range(2, 8)))
self.assertEqual(next_file, list(range(2, 8)))
def test_private_sql_literals_are_hex_encoded(self):
self.assertEqual(fixture._literal("模型'; DROP DATABASE nope;"),
'CONVERT(0x' + "模型'; DROP DATABASE nope;".encode().hex() + ' USING utf8mb4)')
self.assertIn('CAST(CONVERT(0x', fixture._json_sql({'来源': '合成值'}))
def test_isolated_runtime_accepts_only_scoped_engine_and_protocol_overrides(self):
signature = isolated_runtime.isolated_api.__wrapped__.__code__.co_varnames
self.assertIn('env_overrides', signature)
self.assertIn('auth_credentials', signature)
source = (ROOT / 'verification/isolated_runtime.py').read_text(encoding='utf-8')
self.assertIn("'QUALITY_ARCHIVE_ROOT'", source)
self.assertIn("'SERVER_SERVLET_SESSION_COOKIE_NAME'", source)
self.assertTrue(callable(isolated_runtime.sql))
self.assertTrue(callable(isolated_runtime.clone_database))
if __name__ == '__main__':
unittest.main()
+145
View File
@@ -0,0 +1,145 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts')]
from dsh_model_input import compact_tool_result, planning_input, canonical_proposal, explicit_comparison_periods, recent_trend_period, DIRECT_PLANNING_PROMPT
from nao_quality_tools import tool_definition
from nao_weight_ranking import rank_response
class DshModelInputTests(unittest.TestCase):
def test_recent_monthly_trend_binds_published_months_instead_of_one_month(self):
context = {'_businessCatalog': {'periods': [f'2026-{month:02d}' for month in range(1, 11)],
'latestDataPeriod': {'year': 2026, 'month': 10}}}
value = {'kind': 'analysis', 'period': {'year': 2026, 'month': 10},
'trendRequired': True, 'comparisonRequired': True}
fixed = canonical_proposal(value, question='分析最近月份合格率的变化趋势', context=context)
self.assertEqual(fixed['period'], {'year': 2026, 'monthStart': 5, 'monthEnd': 10})
self.assertTrue(fixed['trendRequired'])
self.assertFalse(fixed['comparisonRequired'])
three = recent_trend_period('最近3个月合格率走势', context)
self.assertEqual(three['period'], {'year': 2026, 'monthStart': 8, 'monthEnd': 10})
self.assertEqual(recent_trend_period('最近三个月走势', context), three)
self.assertEqual(recent_trend_period('最近3个已发布月份走势', context), three)
self.assertFalse(recent_trend_period('最近1个月走势', context)['trendRequired'])
self.assertIsNone(recent_trend_period('查看2026年8月至10月的近期走势', context))
self.assertIsNone(recent_trend_period('最近12个月走势', context))
self.assertIsNone(recent_trend_period('最近三年合格率趋势', context))
self.assertEqual(value['period'], {'year': 2026, 'month': 10})
def test_stage_comparison_and_monthly_trend_do_not_require_a_second_period(self):
result = canonical_proposal({'kind': 'analysis', 'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 10}, 'dimensions': ['month'],
'comparisonRequired': True, 'trendRequired': True, 'newTask': True},
question='生成2026年1—10月报告,给出黑片和亮片月度对照及趋势')
self.assertFalse(result['comparisonRequired'])
self.assertEqual(result['dimensions'], [])
self.assertTrue(result['trendRequired'])
prior = {'comparisonPeriod': {'year': 2026, 'month': 9}}
followup = canonical_proposal({'kind': 'analysis', 'comparisonRequired': True},
question='只看黑片', context=prior)
self.assertEqual(followup['comparisonPeriod'], prior['comparisonPeriod'])
def test_explicit_two_month_comparison_cannot_be_misread_as_one_range(self):
source = {'kind': 'analysis', 'fields': ['rate'], 'comparisonPeriod': {'year': 2026, 'monthStart': 9, 'monthEnd': 10}}
binding = {'period': {'year': 2026, 'month': 10}, 'comparisonPeriod': {'year': 2026, 'month': 9}}
result = canonical_proposal(source, question='比较2026年9月和10月公司总体合格率')
self.assertEqual(result['period'], binding['period'])
self.assertEqual(result['comparisonPeriod'], binding['comparisonPeriod'])
self.assertEqual(explicit_comparison_periods('2026年10月比9月变化多少'), binding)
self.assertIsNone(explicit_comparison_periods('生成2026年1—10月总体质量报告'))
self.assertIsNone(explicit_comparison_periods('不要比较2026年9月和10月'))
self.assertEqual(explicit_comparison_periods('比较2026年1—3月和2025年1—3月')['period'],
{'year': 2026, 'monthStart': 1, 'monthEnd': 3})
def test_response_field_aliases_do_not_require_another_model_call(self):
value = {'fields': ['compositeRate', 'blackRate', 'brightRate', 'rateDeltaPP']}
self.assertEqual(canonical_proposal(value)['fields'], ['rate'])
self.assertEqual(value['fields'], ['compositeRate', 'blackRate', 'brightRate', 'rateDeltaPP'])
self.assertEqual(canonical_proposal({'fields': ['unknownMetric']})['fields'], ['unknownMetric'])
def test_one_rank_query_preserves_auxiliary_metrics_for_the_same_selected_objects(self):
source = {'summary': {'black': {'scrapWeight': 3, 'totalWeight': 100}},
'rows': [{'key': 'A', 'label': '甲', 'black': {'scrapWeight': 2, 'totalWeight': 80}},
{'key': 'B', 'label': '乙', 'black': {'scrapWeight': 1, 'totalWeight': 20}}],
'coverage': {'sourceMonths': [{'year': 2026, 'month': 10, 'blackRows': 2}]}}
before = copy.deepcopy(source)
result = rank_response(source, {'id': 'fixture'}, {'year': 2026, 'month': 10}, {},
'product', 'black', 'scrapWeight', 1, include_metrics=True)
self.assertEqual(result['rows'][0]['key'], 'A')
self.assertEqual(result['rows'][0]['metrics'], {'scrapWeight': 2.0, 'totalWeight': 80.0})
self.assertEqual(result['totalWeight'], 3.0)
self.assertEqual(source, before)
def test_native_tool_contract_exposes_only_executable_task_operations(self):
task = {'period': {'year': 2026, 'month': 10}, 'stages': ['black'], 'dimensions': ['product','defect']}
schema = tool_definition(native_protocol=True, task=task, release_id='release')['inputSchema']
self.assertEqual(set(schema['properties']['mode']['enum']), {'catalog', 'query', 'rank', 'defects'})
self.assertNotIn('top', tool_definition(native_protocol=True)['inputSchema']['properties']['mode']['enum'])
self.assertEqual(schema['properties']['period']['const'], task['period'])
self.assertEqual(schema['properties']['stage']['enum'], ['black'])
self.assertEqual(schema['properties']['stage']['type'], 'string')
self.assertNotIn('comparisonPeriod', schema['properties'])
self.assertNotIn('action', schema['properties'])
task.update(comparisonPeriod={'year': 2026, 'month': 9}, fullDetails=True)
schema = tool_definition(native_protocol=True, task=task)['inputSchema']
self.assertTrue({'compare', 'changes', 'details', 'investigate'} <= set(schema['properties']['mode']['enum']))
self.assertEqual(schema['properties']['action']['enum'], ['rows'])
overview = tool_definition(native_protocol=True, task={'kind':'analysis','dimensions':[],
'question':'生成总体报告,给出月度趋势和报废大类'})['inputSchema']
self.assertEqual(overview['properties']['mode']['enum'], ['catalog','query','rank'])
def test_compaction_keeps_numbers_nulls_coverage_and_source_immutable(self):
summary = {'compositeRate': 0, 'black': {'rate': None, 'scrapWeight': 12.3456,
'formalDisplay': {'scrapWeight': {'formatted': '12.35 kg'}}}}
source = {'summary': summary, 'rows': [copy.deepcopy(summary)], 'rowCount': 1,
'coverage': {'missingStages': ['black']}, 'calculationRules': {'internal': 'rule'},
'metricFormulaDescriptions': {'compositeRate': 'formal rule'},
'trend': [{'periodLabel': '2026-09', 'compositeRate': 0.9512},
{'periodLabel': '2026-10', 'compositeRate': 0.9546}]}
original = copy.deepcopy(source)
view = compact_tool_result(source)
self.assertEqual(source, original)
self.assertEqual(view['summary']['black'], {'rate': None, 'scrapWeight': 12.3456})
self.assertEqual(view['coverage'], original['coverage'])
self.assertEqual(view['metricFormulaDescriptions'], original['metricFormulaDescriptions'])
self.assertNotIn('rows', view)
self.assertEqual(view['trend'], source['trend'])
self.assertEqual(view['trendSummary']['compositeRate']['minimum']['period'], '2026-09')
self.assertEqual(view['trendSummary']['compositeRate']['firstToLastPP'], 0.34)
source['rows'].append({'compositeRate': 0.9})
remaining = compact_tool_result(source)['rows']
self.assertEqual(len(remaining), 2)
self.assertEqual(remaining[0]['black']['scrapWeight'], 12.3456)
self.assertEqual(remaining[1], {'compositeRate': 0.9})
def test_planning_keeps_scope_and_questions_without_duplicate_answer_bodies(self):
payload = {'question': '只看黑片', 'previousTask': {'filters': {'department': '甲'}},
'protocol': {'availableRankings': [{'name': '真实产品'}]},
'previousVerifiedTurns': [{'turnId': 'old', 'question': '前问', 'answer': '重复正文'*100}],
'previousUserQuestions': [{'question': '前问', 'status': 'COMPLETED'}],
'lastVisibleAnswer': '重复正文'*100}
before = copy.deepcopy(payload)
result = planning_input(payload)
self.assertEqual(payload, before)
self.assertEqual(result['previousTask'], payload['previousTask'])
self.assertEqual(result['protocol'], payload['protocol'])
self.assertEqual(result['previousVerifiedTurns'], [{'turnId': 'old', 'question': '前问'}])
self.assertNotIn('lastVisibleAnswer', result)
self.assertNotIn('queryPlan=', DIRECT_PLANNING_PROMPT)
def test_catalog_does_not_advertise_legacy_top_or_unavailable_raw_reads(self):
source = {'kind': 'quality_catalog', 'topDimensions': ['category', 'product', 'group'],
'sourceFields': [{'key': 'source-field'}], 'fineDefects': {'fields': ['real-defect']},
'investigation': {'actions': ['overview', 'fields', 'rows', 'source']}}
view = compact_tool_result(source, task={})
self.assertEqual(view['rankingDimensions'], ['product', 'group'])
self.assertNotIn('topDimensions', view)
self.assertNotIn('sourceFields', view)
self.assertNotIn('investigation', view)
self.assertEqual(view['fineDefects'], source['fineDefects'])
self.assertEqual(compact_tool_result(source, task={'fullDetails': True})['sourceFields'], source['sourceFields'])
if __name__ == '__main__':
unittest.main()
+58
View File
@@ -0,0 +1,58 @@
import copy,json,sys,unittest
from pathlib import Path
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).parent)]
from dsh_model_input import canonical_proposal,planning_input,adjacent_month_period,compact_tool_result
from nao_native_exploration import prepare_step
from nao_quality_tools import QualityToolError
from test_native_exploration import root
from test_nao_quality_tools import RELEASE
class SevenQuestionRegressions(unittest.TestCase):
def test_captured_json_string_periods_are_equivalent_to_object_periods(self):
task=root();task.update(period={'year':2026,'month':10},comparisonPeriod={'year':2026,'month':9})
args={'mode':'compare','period':json.dumps(task['period']),'comparisonPeriod':json.dumps(task['comparisonPeriod'])}
before=copy.deepcopy(args)
actual=prepare_step(args,task,{'releaseId':RELEASE,'scopes':[]})
expected=prepare_step({'mode':'compare'},task,{'releaseId':RELEASE,'scopes':[]})
self.assertEqual(actual,expected);self.assertEqual(args,before)
for value in ('{bad','[]','2026-10'):
with self.assertRaises(QualityToolError):prepare_step({'mode':'compare','period':value},task,{'releaseId':RELEASE,'scopes':[]})
def test_redundant_query_stage_uses_same_formal_query_without_widening_root(self):
task=root();task.update(stages=['bright'],comparisonPeriod=None,visualType='donut')
args,proposal=prepare_step({'mode':'query','stage':'bright'},task,{'releaseId':RELEASE,'scopes':[]})
self.assertNotIn('stage',args);self.assertEqual(proposal['node']['stages'],['bright'])
self.assertIsNone(proposal['node']['visualType'])
with self.assertRaises(QualityToolError):prepare_step({'mode':'query','stage':'black'},task,{'releaseId':RELEASE,'scopes':[]})
def test_company_department_ranking_cannot_invent_a_single_department(self):
context={'_businessCatalog':{'departments':['工业电机产业部','消费电子产业部','新能源汽车磁钢']}}
value={'kind':'analysis','filterUpdates':{'department':'工业电机产业部'},'dimensions':['group'],'stages':['bright']}
fixed=canonical_proposal(value,question='生成2026年1—10月产业部亮片报废重量Top5对比简报,说明报废重量主要集中在哪里。',context=context)
self.assertEqual(fixed['filterUpdates']['department'],None);self.assertEqual(fixed['dimensions'],['department'])
explicit=canonical_proposal(value,question='生成工业电机产业部亮片报废重量Top5对比简报',context=context)
self.assertEqual(explicit['filterUpdates']['department'],'工业电机产业部')
def test_neighboring_months_are_part_of_the_same_requested_read_window(self):
question='生成2026年5月质量异常诊断简报,说明当月表现、与前后月份的差异和可观察到的报废线索。数据不能证明的原因请直接说明。'
value={'kind':'analysis','period':{'year':2026,'month':5},'comparisonRequired':False}
fixed=canonical_proposal(value,question=question)
self.assertEqual(fixed['period'],{'year':2026,'monthStart':4,'monthEnd':6})
self.assertTrue(fixed['trendRequired']);self.assertFalse(fixed['comparisonRequired'])
payload=planning_input({'question':question})
self.assertEqual(payload['adjacentPeriodBinding']['focusPeriod'],{'year':2026,'month':5})
self.assertEqual(value['period'],{'year':2026,'month':5})
self.assertIsNone(adjacent_month_period('2026年5月总体表现'))
def test_focus_month_and_complete_ranking_metadata_do_not_change_source_values(self):
source={'kind':'quality_metrics','summary':{'black':{'scrapWeight':80}},'trend':[
{'periodLabel':'2026-04','black':{'scrapWeight':20}},
{'periodLabel':'2026-05','black':{'scrapWeight':35}},
{'periodLabel':'2026-06','black':{'scrapWeight':25}}]}
before=copy.deepcopy(source)
projected=compact_tool_result(source,task={'question':'2026年5月诊断,与前后月份比较'})
self.assertEqual(projected['focusMonth'],source['trend'][1]);self.assertEqual(source,before)
rank=compact_tool_result({'kind':'quality_weight_ranking','dimension':'department','returnedRows':3,'truncated':False})
self.assertTrue(rank['rankingDisplay']['complete']);self.assertEqual(rank['rankingDisplay']['returnedRows'],3)
def test_fine_defect_top_does_not_inherit_a_report_composition_hint(self):
task=root();task.update(stages=['bright'],visualType='pareto',comparisonPeriod=None)
args,proposal=prepare_step({'mode':'defects','stage':'bright','limit':5},task,{'releaseId':RELEASE,'scopes':[]})
self.assertEqual(proposal['node']['visualType'],'bar')
self.assertEqual(args['limit'],5)
if __name__=='__main__':unittest.main()
+115
View File
@@ -0,0 +1,115 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import build_visuals
from nao_agent_task import bind_task
from analysis_visual_evidence import protocol_visual_result
def fixture(dimension='group', fields=None, stages=None, count=3):
fields = fields or ['scrapWeight']
stages = ['black', 'bright'] if stages is None else stages
task = dict(version=1, kind='metrics', question='各对象指标排名', fields=fields,
stages=stages, dimensions=[dimension], dimensionStages={dimension: stages},
period={'year': 2026, 'month': 10}, filters={},
visualPreference='auto', visualType=None, trendRequired=False,
comparisonRequired=False)
rows = [dict(key=f'item-{i:03}', label=f'对象{i}',
black={'totalWeight': 100, 'scrapWeight': i + 1, 'rate': .99 - i / 100, 'scrapRate': .01 + i / 100},
bright={'totalWeight': 200, 'scrapWeight': 2 * (i + 1), 'rate': .99 - i / 100, 'scrapRate': .01 + i / 100},
compositeRate=.98 - i / 100, coverage={'blackRows': 1, 'brightRows': 1}) for i in range(count)]
total = count * (count + 1) / 2
result = dict(kind='quality_metrics', releaseId='release', release={'id': 'release'},
view=dimension, period=task['period'], filters={}, rows=rows,
rowCount=count, rowsReturned=count, rowsTruncated=False,
summary={'black': {'totalWeight': count * 100, 'scrapWeight': total},
'bright': {'totalWeight': count * 200, 'scrapWeight': 2 * total}},
coverage={'sourceMonths': [{'year': 2026, 'month': 10, 'blackRows': count, 'brightRows': count}]})
return task, {'tool': 'quality_data', 'mode': 'query', 'releaseId': 'release',
'evidenceRef': 'Q1234ABCD', 'result': result}
class GroupedQueryVisualTests(unittest.TestCase):
def render(self, task, document):
return build_visuals([document], task['question'], 'turn-grouped', context=bind_task({'releaseId': 'release'}, task))
def test_all_dimensions_and_stage_weights_use_the_ranking_converter(self):
for dimension in ('department', 'group', 'product'):
for field in ('totalWeight', 'scrapWeight'):
with self.subTest(dimension=dimension, field=field):
task, doc = fixture(dimension, [field])
before = copy.deepcopy(doc)
visuals = self.render(task, doc)['visuals']
self.assertEqual(len(visuals), 2)
self.assertEqual({v['stage'] for v in visuals}, {'black', 'bright'})
self.assertEqual(visuals[0]['categories'][0]['key'], 'item-000' if field == 'totalWeight' else 'item-002')
self.assertEqual(doc, before)
def test_single_stage_and_combined_scrap_keep_their_distinct_meanings(self):
for stages, expected in ((['bright'], {'bright'}), ([], {None})):
task, doc = fixture(stages=stages)
visuals = self.render(task, doc)['visuals']
self.assertEqual({v.get('stage') for v in visuals}, expected)
def test_full_top100_and_truncated_input(self):
task, doc = fixture(count=123)
charts = self.render(task, doc)['visuals']
self.assertEqual(len(charts[0]['categories']), 100)
self.assertEqual(charts[0]['coverage']['totalRows'], 123)
doc['result']['rowsTruncated'] = True
self.assertEqual(self.render(task, doc)['visuals'], [])
def test_query_top_count_controls_chart_rows_without_changing_query_data(self):
task, doc = fixture(count=30)
task['question'] = '各分组不合格重量Top20'
charts = self.render(task, doc)['visuals']
self.assertEqual([len(chart['categories']) for chart in charts], [20, 20])
self.assertEqual(len(doc['result']['rows']), 30)
def test_published_rates_are_not_recalculated_and_bad_metrics_are_isolated(self):
task, doc = fixture(fields=['rate', 'scrapWeight'], stages=['bright'])
doc['result']['summary']['bright']['scrapWeight'] = -1
visuals = self.render(task, doc)['visuals']
rate = next(v for v in visuals if v['metric'] == 'bright_rate')
self.assertEqual(rate['series'][0]['values'], [.99, .98, .97])
self.assertFalse(any(v['metric'] == 'bright_scrap_weight' for v in visuals))
def test_no_charts_for_empty_data_or_explicit_none(self):
task, doc = fixture(count=0)
self.assertEqual(self.render(task, doc)['visuals'], [])
task, doc = fixture()
task['visualPreference'] = 'none'
self.assertEqual(self.render(task, doc)['visuals'], [])
def test_each_dimension_uses_published_scrap_rates(self):
for dimension in ('department', 'group', 'product'):
task, doc = fixture(dimension, ['scrapRate'])
charts = self.render(task, doc)['visuals']
self.assertEqual({v['metric'] for v in charts}, {'black_scrap_rate', 'bright_scrap_rate'})
self.assertEqual(charts[0]['series'][0]['values'], [.01, .02, .03])
def test_duplicate_rows_and_wrong_release_cannot_enter_charts(self):
task, doc = fixture()
doc['result']['rows'][1]['key'] = doc['result']['rows'][0]['key']
self.assertEqual(self.render(task, doc)['visuals'], [])
task, doc = fixture()
doc['releaseId'] = doc['result']['releaseId'] = 'wrong-release'
self.assertEqual(self.render(task, doc)['visuals'], [])
def test_broken_scope_does_not_remove_a_healthy_query_chart(self):
task, doc = fixture()
doc['result']['protocolEvidence'] = {'queryId': 'q1', 'nodeId': 'good', 'scopeId': 'good-scope'}
scope = {'nodeId': 'good', 'scopeId': 'good-scope', 'purpose': 'metrics', 'task': task}
broken = copy.deepcopy(scope)
broken['task']['stages'] = 5
broken['nodeId'] = 'broken'
result = protocol_visual_result([doc], [broken, scope], task['question'], 'turn-grouped', release_id='release')
self.assertEqual(result['visualState'], 'ready')
self.assertEqual(len(result['visuals']), 2)
if __name__ == '__main__':
unittest.main()
+56
View File
@@ -1636,6 +1636,62 @@ class HistoryReportExecutionTest(unittest.TestCase):
wrong_collective = _bind_literal_measures(wrong_collective, ranked_tokens, {}, facts=ranked)
self.assertIn('1155.48340000、1106.25、1060.75', wrong_collective['blocks'][0]['text'])
def test_unknown_unit_rank_mass_requires_exact_local_metric_and_never_invents_unit(self):
from nao_native_answer import (NativeCompositionError, _bind_exact_identifiers,
_bind_exact_periods, _bind_exact_year_literals, _bind_literal_measures,
_binding_inventory, _validate_blocks)
def ranked_fact(fact_id='R1', *, rank_group='history-rank:verified', source_turn='turn-1', value='36.00'):
return {'id': fact_id, 'kind': 'product', 'label': '规格一',
'text': f'2026年8月:规格一综合报废重量 = {value}。',
'evidenceRef': 'history:' + fact_id,
'sourceTurnId': source_turn, 'sectionIndex': 2, 'rank': 1,
'rankGroupKey': rank_group,
'scope': {'period': {'year': 2026, 'month': 8}, 'filters': {},
'dimension': 'product', 'stage': 'overall'},
'validity': {'verified': True, 'basis': 'protocol_history_snapshot'},
'verifiedIdentifiers': [{'value': '规格一'}],
'verifiedDisplayValues': [{'metricCode': 'scrapWeight', 'rawValue': value,
'rawUnit': 'source_weight', 'unit': 'source_weight', 'displayValue': value}]}
fact = ranked_fact()
facts = [fact]
tokens = _binding_inventory(facts)
def bind(text, cited=None):
draft = {'blocks': [{'text': text, 'factIds': cited or ['R1'], 'role': 'core'}]}
for binder in (_bind_exact_identifiers, _bind_exact_periods, _bind_exact_year_literals):
draft = binder(draft, tokens, {})
return _bind_literal_measures(draft, tokens, {}, facts=facts)
accepted = bind('2026年8月规格一综合报废重量为36.000。')
self.assertRegex(accepted['blocks'][0]['text'], r'\{\{N[1-9][0-9]*\}\}')
rendered, _ = _validate_blocks(accepted, facts, tokens)
self.assertIn('36.00', rendered[0]['text'])
self.assertNotIn('kg', rendered[0]['text'])
self.assertNotIn('source_weight', rendered[0]['text'])
for invalid in (
'2026年8月规格一综合报废重量为36.000 kg。',
'2026年8月规格一综合合格率97.19%,29。',
):
with self.subTest(invalid=invalid):
draft = bind(invalid)
self.assertNotIn('{{N', draft['blocks'][0]['text'])
with self.assertRaises(NativeCompositionError):
_validate_blocks(draft, facts, tokens)
same_value_other_provenance = ranked_fact('R2', rank_group='history-rank:other',
source_turn='turn-2')
ambiguous_facts = [fact, same_value_other_provenance]
ambiguous_tokens = _binding_inventory(ambiguous_facts)
ambiguous = {'blocks': [{'text': '2026年8月规格一综合报废重量为36.00。',
'factIds': ['R1', 'R2'], 'role': 'core'}]}
for binder in (_bind_exact_identifiers, _bind_exact_periods, _bind_exact_year_literals):
ambiguous = binder(ambiguous, ambiguous_tokens, {})
ambiguous = _bind_literal_measures(ambiguous, ambiguous_tokens, {}, facts=ambiguous_facts)
self.assertNotIn('{{N', ambiguous['blocks'][0]['text'])
def test_history_parent_predicate_mass_inheritance_uses_family_stage_and_exact_fact(self):
from nao_native_answer import (NativeCompositionError, _bind_exact_identifiers,
_bind_exact_periods, _bind_exact_year_literals, _binding_inventory, _validate_blocks)
+80
View File
@@ -96,6 +96,64 @@ class ModelGateTests(unittest.TestCase):
self.assertEqual(self.request(model='different-model')[0], 400)
self.assertEqual(len(self.calls), 0)
def test_dsh_keeps_only_its_bound_quality_tool_at_the_provider_boundary(self):
self.gate.runtime_engine='DSH'
names=['mcp__quality__quality_data','bash','mcp_call','story','mcp__other__quality_data']
tools=[{'type':'function','function':{'name':name,'parameters':{'type':'object'}}} for name in names]
self.assertEqual(self.request(extra={'tools':tools})[0],200)
self.assertEqual([x['function']['name'] for x in self.calls[-1]['tools']],['mcp__quality__quality_data'])
def test_dsh_stream_separates_complete_calls_when_the_provider_reuses_an_index(self):
from unittest.mock import Mock
from nao_thinking_stream import ProviderStream
self.gate.runtime_engine='DSH'
name='mcp__quality__quality_data'
tool={'type':'function','function':{'name':name,'parameters':{'type':'object'}}}
frames=[{'choices':[{'index':0,'delta':{'tool_calls':[{'index':index,'id':identity,'type':'function',
'function':{'name':name,'arguments':json.dumps({'probe':identity})}}]},'finish_reason':None}]}
for index,identity in [(0,'a'),(1,'b'),(1,'c'),(2,'d')]]
frames.append({'choices':[{'index':0,'delta':{},'finish_reason':'tool_calls'}]})
chunks=[('data: '+json.dumps(frame)+'\n\n').encode() for frame in frames]+[b'data: [DONE]\n\n']
upstream=HeldOpenProviderResponse(chunks);upstream.release.set()
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
code,body=self.request(extra={'stream':True,'tools':[tool]})
self.assertEqual(code,200)
observer=ProviderStream(lambda _:None);observer.feed(body.encode());observer.finish()
calls=observer.completion()['choices'][0]['message']['tool_calls']
self.assertEqual([call['id'] for call in calls],['a','b','c','d'])
self.assertEqual([call['function']['name'] for call in calls],[name]*4)
self.assertEqual([json.loads(call['function']['arguments']) for call in calls],[{'probe':x} for x in 'abcd'])
def test_revocation_interrupts_an_inflight_provider_read(self):
from unittest.mock import Mock
class StalledResponse:
status=200
headers={'Content-Type':'text/event-stream'}
def __init__(self):
self.reading=threading.Event();self.released=threading.Event()
def __enter__(self):return self
def __exit__(self,*_):self.close()
def read1(self,_):
self.reading.set();self.released.wait(5);return b''
def close(self):self.released.set()
stalled=StalledResponse();self.gate._opener=Mock()
self.gate._opener.open.return_value=stalled
done=threading.Event();result=[]
def query():
try:result.append(self.request())
finally:done.set()
worker=threading.Thread(target=query,daemon=True);worker.start()
self.assertTrue(stalled.reading.wait(1))
self.gate.revoke()
self.assertTrue(done.wait(1),'revoked provider read was left waiting for its timeout')
worker.join(1)
# SSE headers may have been sent before revocation; HTTP 200 alone is
# not a completion receipt. The interrupted body has no terminal event.
self.assertIn(result[0][0],(200,502))
self.assertNotIn('[DONE]',result[0][1])
self.assertNotIn('"finish_reason": "stop"',result[0][1])
self.assertEqual(self.request()[0],403)
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
@@ -123,6 +181,14 @@ class ModelGateTests(unittest.TestCase):
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
code,body=self.request(extra={'tools':[data_tool]})
self.assertEqual(code,502);self.assertNotIn('clarification',body)
self.assertEqual(self.gate.rejected_tool_names,['clarification'])
from types import SimpleNamespace
from nao_model_gate import ToolPolicyError
observer=SimpleNamespace(tools={0:{0:{'function':{'name':'secret=never-log','arguments':'private argument'}}}},
choices={0:{'finish_reason':'tool_calls'}},done=True)
with self.assertRaises(ToolPolicyError):
self.gate.check_returned_tools(observer,{'mcp_call'},final=True)
self.assertEqual(self.gate.rejected_tool_names,['clarification','<invalid_name>'])
for newline in (True,False):
prefix=b'data: {"choices":[{"index":0,"delta":{"content":"Preserved text"}}]}\n\n'
bad=('data: '+json.dumps({'choices':[{'index':0,'delta':{'tool_calls':[tool]},'finish_reason':'tool_calls'}]})+ ('\n\n' if newline else '')).encode()
@@ -215,6 +281,20 @@ class ModelGateTests(unittest.TestCase):
self.assertNotIn('reasoning_effort',body)
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
def test_mimo_disables_thinking_only_for_native_tool_requests(self):
self.gate._model='mimo-v2.6-flash'
self.gate.runtime_engine='DSH'
self.gate._thinking={'enabled':True,'effort':'auto','protocol':'thinking'}
tool={'type':'function','function':{'name':'mcp__quality__quality_data','parameters':{'type':'object'}}}
for native,tools in ((False,[]),(True,[tool]),(True,[])):
with self.subTest(native=native,tools=bool(tools)):
self.gate.native_agent=native
self.assertEqual(self.request(model=self.gate._model,extra={'tools':tools})[0],200)
expected=not(native and tools)
self.assertEqual(self.calls[-1]['thinking'],{'type':'enabled' if expected else 'disabled'})
self.assertTrue(self.gate._thinking['enabled'])
self.assertTrue(self.gate.request_audit[-1]['requestedEnabled'])
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
+87
View File
@@ -295,6 +295,54 @@ class RawPlanExecutionTests(unittest.TestCase):
self.assertNotIn('recordList',result)
self.assertEqual({doc['mode'] for doc in documents},{'investigate'})
def test_autonomous_source_and_records_only_plan_uses_finalizer_and_keeps_raw_coverage(self):
task=self.root_task(purposes=('source','records'))
api=InvestigationApi(total_rows=1)
for purpose in ('source','records'):
api.add_scope(purpose,purpose+'-scope',purpose,task)
captured={}
def synthesize_stub(**kwargs):
captured['evidence']=copy.deepcopy(kwargs['evidence'])
captured['question']=kwargs['question']
return {'answer':'仅确认了原表核验,原问题仍缺少正式分析依据。',
'evidenceSufficient':False,'missingEvidence':['原问题所需正式分析依据未取得。'],
'modelDeclaredEvidenceSufficient':None,'usage':{},'finishReason':'stop',
'selectedFactIds':[]}
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize',side_effect=synthesize_stub) as synth:
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
'question':task['question'],'_autonomousAnswer':True},task=task,
authority={**identity(),'task':task},actor_root=root,delegated=api,
connection=('http://offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
self.assertTrue(synth.called)
source_docs=[doc for doc in captured['evidence'] if doc.get('mode')=='investigate'
and (doc.get('result') or {}).get('action')=='source']
self.assertEqual(len(source_docs),1)
self.assertFalse(any((doc.get('result') or {}).get('action')=='rows'
for doc in captured['evidence']))
self.assertEqual(source_docs[0]['result']['current']['sourceVerification']['verifiedRows'],1)
self.assertIn('已按稳定分页读全1条',result['answer'])
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('原问题所需正式分析依据未取得。',result['evidenceCompletion']['missingEvidence'])
def test_autonomous_records_only_plan_stays_deterministic(self):
task=self.root_task(purposes=('records',))
api=InvestigationApi(total_rows=1)
api.add_scope('records','records-scope','records',task)
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize') as synth:
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
'question':task['question'],'_autonomousAnswer':True},task=task,
authority={**identity(),'task':task},actor_root=root,delegated=api,
connection=('http://offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
self.assertFalse(synth.called)
self.assertEqual(result['status'],'COMPLETED')
self.assertIn('已按本轮服务器授权范围核对原始数据。',result['answer'])
def test_unselected_column_filters_accept_canonical_empty_context_echo(self):
api=InvestigationApi(total_rows=2)
result,_,_=self.run_query(api,with_column_filters=False)
@@ -468,6 +516,45 @@ class MixedRawPlanExecutionTests(unittest.TestCase):
self.assertIn('原表值与存储记录一致',result['answer'])
self.assertNotIn('PRIVATE_RAW_VALUE',result['answer'])
def test_autonomous_mixed_synthesis_receives_source_proof_without_record_rows(self):
plan=[
{'nodeId':'records','purpose':'records','fields':[],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
{'nodeId':'verify','purpose':'source','fields':[],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
{'nodeId':'metric','purpose':'metrics','fields':['scrapRate'],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
]
value=task_value(kind='details',dimensions=[],stages=['bright'],fields=['scrapRate'],
verifySource=True,fullDetails=True,period={'year':2026,'month':9},queryPlan=plan,
visualPreference='none',visualType=None,comparisonRequired=False,trendRequired=False)
task=normalize_task(value,'查询亮片报废率,并核验原表及读取明细',{'_protocol':{'version':1}})
api=MixedRawMetricsApi(task,task['queryPlan'])
captured={}
def synthesize_stub(**kwargs):
captured['evidence']=copy.deepcopy(kwargs['evidence'])
return {'answer':'已基于正式指标和原表核验完成说明。','evidenceSufficient':True,
'missingEvidence':[],'modelDeclaredEvidenceSufficient':True,'usage':{},
'finishReason':'stop','selectedFactIds':[]}
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize',side_effect=synthesize_stub):
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':task['question'],
'_autonomousAnswer':True},task=task,authority={**identity(),'task':task},
actor_root=root,delegated=api,connection=('http://offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
docs=captured['evidence']
self.assertTrue(any(doc.get('mode')=='query' for doc in docs))
source_docs=[doc for doc in docs if doc.get('mode')=='investigate'
and (doc.get('result') or {}).get('action')=='source']
self.assertTrue(source_docs)
self.assertFalse(any((doc.get('result') or {}).get('action')=='rows' for doc in docs))
self.assertNotIn('PRIVATE_RAW_VALUE',json.dumps(docs,ensure_ascii=False))
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
if __name__ == '__main__':
unittest.main()
+362
View File
@@ -0,0 +1,362 @@
"""Authored SSE exercises the real observer and installed SDK without network I/O."""
import json
from pathlib import Path
import subprocess
import sys
import unittest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
from nao_thinking_stream import ProviderStream
from nao_tool_stream import ToolStreamNormalizer, ToolStreamProtocolError
TOOL = 'mcp__quality__quality_data'
def frame(*calls, choice=0, finish=None, **delta):
if calls:
delta['tool_calls'] = list(calls)
return {'choices': [{'index': choice, 'delta': delta, 'finish_reason': finish}]}
def call(index, identity=None, arguments=None, name=None):
value = {'index': index}
if identity is not None:
value.update(id=identity, type='function')
function = {}
if name is not None:
function['name'] = name
if arguments is not None:
function['arguments'] = arguments
if function:
value['function'] = function
return value
def wire(rows, *, done=True, newline=b'\n'):
result = b''.join(b'data: ' + json.dumps(row, ensure_ascii=False).encode() + newline * 2 for row in rows)
return result + (b'data: [DONE]' + newline * 2 if done else b'')
def collision_rows():
return [frame(call(0, 'a', '{"mode":"catalog"}', TOOL)),
frame(call(1, 'b', '{"mode":"compare"}', TOOL)),
frame(call(1, 'c', '{"mode":', TOOL)),
frame(call(1, arguments='"changes"}')),
frame(call(2, 'd', '{"mode":"metrics"}', TOOL)),
frame(finish='tool_calls')]
class ToolStreamTests(unittest.TestCase):
def normalize(self, data, pieces=None):
normalizer = ToolStreamNormalizer({TOOL})
output = b''
for piece in pieces if pieces is not None else [data]:
output += normalizer.feed(piece)
return output + normalizer.finish()
def completion(self, data):
observer = ProviderStream(lambda _text: None)
observer.feed(data)
observer.finish()
return observer.completion()['choices']
def test_raw_collision_is_a_red_capable_real_observer_fixture(self):
tools = self.completion(wire(collision_rows()))[0]['message']['tool_calls']
self.assertEqual(len(tools), 3)
self.assertEqual(tools[1]['function']['name'], TOOL + TOOL)
with self.assertRaises(ValueError):
json.loads(tools[1]['function']['arguments'])
def test_collision_preserves_three_calls_and_following_wire_index_cannot_collide(self):
result = self.completion(self.normalize(wire(collision_rows())))[0]['message']['tool_calls']
self.assertEqual([value['id'] for value in result], ['a', 'b', 'c', 'd'])
self.assertEqual([value['function']['name'] for value in result], [TOOL] * 4)
self.assertEqual([json.loads(value['function']['arguments'])['mode'] for value in result],
['catalog', 'compare', 'changes', 'metrics'])
rows = [json.loads(line[5:]) for line in self.normalize(wire(collision_rows())).splitlines()
if line.startswith(b'data:') and b'[DONE]' not in line]
self.assertEqual([item['index'] for row in rows for choice in row['choices']
for item in choice['delta'].get('tool_calls', [])], [0, 1, 2, 2, 3])
def test_legal_parallel_interleaved_fragments_preserve_exact_wire(self):
rows = [frame(call(0, 'a', '{"value":', TOOL), call(1, 'b', '{"value":', TOOL)),
frame(call(1, arguments='"亮片🙂"}')),
frame(call(0, arguments='"黑片"}')), frame(finish='tool_calls')]
data = wire(rows, newline=b'\r\n')
self.assertEqual(self.normalize(data), data)
result = self.completion(data)[0]['message']['tool_calls']
self.assertEqual([json.loads(item['function']['arguments'])['value'] for item in result], ['黑片', '亮片🙂'])
def test_every_byte_boundary_and_headers_heartbeats_are_preserved(self):
data = b': heartbeat\n\nevent: message\n' + wire(collision_rows())
expected = self.normalize(data)
self.assertEqual(self.normalize(data, [bytes([byte]) for byte in data]), expected)
self.assertTrue(expected.startswith(b': heartbeat\n\nevent: message\n'))
def test_reindexed_frame_preserves_name_id_argument_string_and_other_fields(self):
arguments = ' {"quoted":"a\\\"b","path":"x\\\\y","unicode":"亮片🙂"} '
original = frame(call(0, 'identity/亮片', arguments, TOOL), content='原文🙂')
original.update(id='completion-original', extra={'unchanged': True})
data = wire([frame(call(0, 'first', '{}', TOOL)), original, frame(finish='tool_calls')])
normalized = self.normalize(data)
rows = [json.loads(line[5:]) for line in normalized.splitlines()
if line.startswith(b'data:') and b'[DONE]' not in line]
expected = json.loads(json.dumps(original))
expected['choices'][0]['delta']['tool_calls'][0]['index'] = 1
self.assertEqual(rows[1], expected)
self.assertEqual(rows[1]['choices'][0]['delta']['tool_calls'][0]['function']['arguments'], arguments)
def test_feed_returns_valid_events_before_the_rest_of_the_stream_arrives(self):
normalizer = ToolStreamNormalizer({TOOL})
first = wire([frame(call(0, 'a', '{"mode":', TOOL))], done=False)
self.assertEqual(normalizer.feed(first), first)
final = wire([frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')])
self.assertEqual(normalizer.feed(final) + normalizer.finish(), final)
def test_fragmented_name_and_delayed_initial_id_remain_one_call(self):
rows = [frame(call(0, arguments='{', name='mcp__quality__')),
frame(call(0, 'a', arguments='"mode":', name='quality_data')),
frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')]
data = wire(rows)
self.assertEqual(self.normalize(data), data)
self.assertEqual(self.completion(data)[0]['message']['tool_calls'][0]['id'], 'a')
def test_message_snapshot_and_non_sse_json_are_not_rewritten(self):
snapshot = {'choices': [{'index': 0, 'message': {'tool_calls': [call(1, 'a', '{}', TOOL),
call(1, 'b', '{}', TOOL)]},
'finish_reason': 'tool_calls'}]}
data = wire([snapshot])
self.assertEqual(self.normalize(data), data)
ordinary = json.dumps(snapshot).encode()
self.assertEqual(self.normalize(ordinary), ordinary)
def test_choices_have_independent_index_and_id_namespaces(self):
rows = [frame(call(0, 'a', '{}', TOOL), choice=0), frame(call(0, 'a', '{}', TOOL), choice=1),
frame(call(0, 'b', '{}', TOOL), choice=0), frame(call(1, 'b', '{}', TOOL), choice=1),
frame(choice=0, finish='tool_calls'), frame(choice=1, finish='tool_calls')]
choices = self.completion(self.normalize(wire(rows)))
self.assertEqual([[item['id'] for item in choice['message']['tool_calls']] for choice in choices],
[['a', 'b'], ['a', 'b']])
def test_repeated_active_id_continues_but_repeated_name_is_not_silently_deduplicated(self):
data = wire([frame(call(0, 'a', '{"mode":', TOOL)),
frame(call(0, 'a', '"catalog"}')), frame(finish='tool_calls')])
self.assertEqual(self.normalize(data), data)
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'a', '', TOOL))])
def reject(self, rows, *, code=None, done=False):
with self.assertRaises(ToolStreamProtocolError) as caught:
self.normalize(wire(rows, done=done))
if code:
self.assertEqual(caught.exception.code, code)
self.assertNotIn('sensitive-fixture', str(caught.exception))
def test_argument_errors_are_preserved_for_sdk_without_aborting_later_calls(self):
for old in ('', '{"mode":', '[]', 'null', '42', '{}{}', '{"x":NaN}', '{"x":Infinity}',
'{"x":1,"x":2}'):
with self.subTest(old=old):
result = self.completion(self.normalize(wire([
frame(call(0, 'a', old, TOOL)), frame(call(0, 'b', '{}', TOOL)),
frame(finish='tool_calls')])))[0]['message']['tool_calls']
self.assertEqual([item['id'] for item in result], ['a', 'b'])
self.assertEqual([item['function']['arguments'] for item in result], [old, '{}'])
def test_reuse_inside_one_event_keeps_each_original_call(self):
data = wire([frame(call(0, 'a', '{}', TOOL), call(0, 'b', '{}', TOOL), call(0, 'c', '{}', TOOL)),
frame(finish='tool_calls')])
result = self.completion(self.normalize(data))[0]['message']['tool_calls']
self.assertEqual([item['id'] for item in result], ['a', 'b', 'c'])
def test_retired_call_is_not_recovered_from_its_complete_arguments_or_name(self):
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"x":', TOOL)),
frame(call(0, 'a', '1}'))], code='reused_call_id')
def test_malformed_frame_and_invalid_tool_payload_do_not_leak_input(self):
for data in (b'data: {sensitive-fixture}\n\n', b'data: []\n\n',
b'data: {"choices":[],"choices":[]}\n\n'):
with self.assertRaises(ToolStreamProtocolError) as caught:
self.normalize(data)
self.assertNotIn('sensitive-fixture', str(caught.exception))
for changed in (call(0, 'a', '{}', TOOL) | {'type': 'custom'},
call(0, 'a', '{}', TOOL) | {'function': {'name': TOOL, 'arguments': {}}}):
self.reject([frame(changed)])
def test_partial_name_never_authorizes_a_rotation(self):
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(call(0, 'b', '{}', TOOL))],
code='incomplete_previous_call')
def test_unknown_tool_prefix_and_final_incomplete_name_fail_closed(self):
self.reject([frame(call(0, 'a', '{}', 'sensitive-fixture'))], code='unadvertised_tool')
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(finish='tool_calls')],
code='unadvertised_tool')
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', 'shell'))],
code='unadvertised_tool')
def test_retired_id_cannot_return_or_move_to_another_wire_index(self):
first = [frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', TOOL))]
for last in (call(0, 'a', '', None), call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)):
with self.subTest(last=last):
self.reject(first + [frame(last)], code='reused_call_id')
def test_empty_or_invalid_id_and_missing_or_invalid_index_fail_closed(self):
base = call(0, 'a', '{}', TOOL)
for changed in ({**base, 'id': ''}, {**base, 'id': 3}, {**base, 'index': True},
{**base, 'index': -1}, {**base, 'index': 0.5},
{key: value for key, value in base.items() if key != 'index'}):
with self.subTest(changed=changed):
self.reject([frame(changed)])
def test_mimo_null_identity_and_name_are_continuations_of_an_identified_call(self):
rows = [frame(call(0, 'call_mimo', '', TOOL)),
frame({'index': 0, 'id': None, 'type': 'function',
'function': {'name': None, 'arguments': '{"mode": '}}),
frame({'index': 0, 'id': None, 'type': 'function',
'function': {'name': None, 'arguments': '"catalog"}'}}),
frame(finish='tool_calls')]
result = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['id'], 'call_mimo')
self.assertEqual(result[0]['function']['name'], TOOL)
self.assertEqual(json.loads(result[0]['function']['arguments']), {'mode': 'catalog'})
def test_mimo_observed_namespace_alias_keeps_call_identity_and_arguments(self):
# Captured with the real task input: MiMo Flash omitted only "mcp__".
arguments = '{"mode": "query"}'
for fragments in (['quality__quality_data'], ['quality__', 'quality_data'],
['q', 'uality__quality_data']):
with self.subTest(fragments=fragments):
rows = [frame(call(0, 'mimo_alias', '', fragments[0]))]
rows += [frame(call(0, name=part)) for part in fragments[1:]]
rows += [frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': arguments}}),
frame(finish='tool_calls')]
tools = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
self.assertEqual(len(tools), 1)
self.assertEqual(tools[0]['id'], 'mimo_alias')
self.assertEqual(tools[0]['function'], {'name': TOOL, 'arguments': arguments})
def test_namespace_alias_only_resolves_to_an_offered_tool(self):
for name in ('quality_data', 'quality__shell', 'quality__quality_data_extra',
'functions.mcp__quality__quality_data'):
self.reject([frame(call(0, 'a', '{}', name))], code='unadvertised_tool')
normalizer = ToolStreamNormalizer({'other_tool'})
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(call(0, 'a', '{}', 'quality__quality_data'))]))
def test_null_identity_cannot_start_a_call_or_change_its_tool(self):
self.reject([frame({'index': 0, 'id': None, 'function': {'name': TOOL, 'arguments': '{}'}})],
code='invalid_call_id')
self.reject([frame(call(0, 'a', '', TOOL)),
frame({'index': 0, 'id': None, 'function': {'name': 'shell', 'arguments': '{}'}})],
code='unadvertised_tool')
def test_calls_after_choice_finish_or_done_fail_closed(self):
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
frame(call(1, 'b', '{}', TOOL))], code='call_after_finish')
normalizer = ToolStreamNormalizer({TOOL})
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls')]))
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(call(1, 'b', '{}', TOOL))]))
def test_missing_finish_and_failed_normalizer_cannot_be_resumed(self):
normalizer = ToolStreamNormalizer({TOOL})
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL))], done=False))
with self.assertRaises(ToolStreamProtocolError):
normalizer.finish()
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(wire([frame(finish='tool_calls')]))
def test_final_line_without_newline_and_usage_tail_are_preserved(self):
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
{'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2}}]).rstrip(b'\n')
self.assertEqual(self.normalize(data), data)
def test_sparse_wire_indices_remain_unchanged_until_they_would_collide(self):
data = wire([frame(call(4, 'a', '{}', TOOL)), frame(call(9, 'b', '{}', TOOL)), frame(finish='tool_calls')])
self.assertEqual(self.normalize(data), data)
def test_one_failed_feed_cannot_return_an_earlier_valid_line_or_be_resumed(self):
normalizer = ToolStreamNormalizer({TOOL})
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"sensitive-fixture":', 'shell'))], done=False)
with self.assertRaises(ToolStreamProtocolError):
normalizer.feed(data)
with self.assertRaises(ToolStreamProtocolError):
normalizer.finish()
def test_oversized_unterminated_frame_fails_closed(self):
normalizer = ToolStreamNormalizer({TOOL})
normalizer.MAX_FRAME_BYTES = 32
with self.assertRaises(ToolStreamProtocolError) as caught:
normalizer.feed(b'data: ' + b'x' * 33)
self.assertEqual(caught.exception.code, 'frame_too_large')
@unittest.skipUnless((ROOT / 'runtime/dsh/bin/node').is_file()
and (ROOT / 'runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js').is_file(),
'requires the installed pinned DSH SDK')
class InstalledSdkTests(unittest.TestCase):
def sdk(self, data):
script = r'''
import fs from 'node:fs';
import { stream } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js';
import { normalizeContext } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/utils/transcript.js';
const wire = fs.readFileSync(0, 'utf8');
const model = {id:'m', name:'m', api:'openai-completions', provider:'test', baseUrl:'http://offline.invalid/v1',
reasoning:false, input:['text'], cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:1024};
const context = normalizeContext({messages:[{role:'user',content:'authored offline fixture',timestamp:0}]});
let fetches = 0;
const events = stream(model,context,{apiKey:'authored-no-network',fetch:async()=>{
fetches++; return new Response(wire,{status:200,headers:{'Content-Type':'text/event-stream'}});
}});
const calls = []; let terminal;
for await (const event of events) {
if (event.type === 'toolcall_end') calls.push({id:event.toolCall.id,name:event.toolCall.name,
arguments:event.toolCall.arguments,invalid:event.toolCall.argumentsError !== undefined});
if (event.type === 'done' || event.type === 'error') terminal = event.type;
}
console.log(JSON.stringify({calls,terminal,fetches}));
'''
result = subprocess.run([str(ROOT / 'runtime/dsh/bin/node'), '--input-type=module', '-e', script],
cwd=ROOT, input=data, capture_output=True, timeout=15, check=True)
return json.loads(result.stdout)
def test_real_sdk_raw_collision_fails_and_normalized_calls_keep_ids_and_arguments(self):
data = wire(collision_rows())
bad = self.sdk(data)
self.assertEqual(len(bad['calls']), 3)
self.assertTrue(bad['calls'][1]['invalid'])
good = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(good['terminal'], 'done')
self.assertEqual(good['fetches'], 1)
self.assertEqual([value['id'] for value in good['calls']], ['a', 'b', 'c', 'd'])
self.assertTrue(all(value['name'] == TOOL and not value['invalid'] for value in good['calls']))
self.assertEqual([value['arguments']['mode'] for value in good['calls']],
['catalog', 'compare', 'changes', 'metrics'])
def test_real_sdk_still_rejects_a_final_incomplete_argument_object(self):
data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertTrue(result['calls'][0]['invalid'])
def test_real_sdk_keeps_argument_error_and_subsequent_independent_call(self):
data = wire([frame(call(0, 'bad', '{"mode":', TOOL)),
frame(call(0, 'next', '{"mode":"query"}', TOOL)), frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(result['terminal'], 'done')
self.assertEqual([item['id'] for item in result['calls']], ['bad', 'next'])
self.assertTrue(result['calls'][0]['invalid'])
self.assertFalse(result['calls'][1]['invalid'])
self.assertEqual(result['calls'][1]['arguments'], {'mode': 'query'})
def test_real_sdk_receives_the_advertised_name_for_observed_mimo_alias(self):
data = wire([frame(call(0, 'mimo_alias', '', 'quality__quality_data')),
frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': '{"mode":"query"}'}}),
frame(finish='tool_calls')])
result = self.sdk(ToolStreamTests().normalize(data))
self.assertEqual(result['terminal'], 'done')
self.assertEqual(result['calls'], [{'id': 'mimo_alias', 'name': TOOL,
'arguments': {'mode': 'query'}, 'invalid': False}])
if __name__ == '__main__':
unittest.main()
+22
View File
@@ -1,3 +1,4 @@
import json
import copy
import sys
import unittest
@@ -28,6 +29,27 @@ def bundle(value,queries):
class VisualRecoveryTests(unittest.TestCase):
def test_native_product_ranking_recovers_from_exact_frozen_batch_without_queryplan(self):
value=task();value.update(executionMode='native_agent',kind='analysis',trendRequired=False,
dimensions=['product'],stages=['black'],fields=['rate','totalWeight'])
value['aggregateSelection']={'dimension':'product','metric':'rate','rateStage':None,'direction':'asc',
'limit':100,'requestQuestion':'筛选合格率最低Top100产品并生成报告',
'weightFilter':{'metric':'totalWeight','stage':'black','operator':'gt','value':1.0}}
raw={**value,'_view':'product'};saved=query(raw,BoundRankingApi())
for row in saved['response']['rows']:row['compositeRate']=.9
scope={'nodeId':'product-batch','scopeId':'scope-products','purpose':'metrics','limit':500,'task':{**value,'kind':'metrics'}}
for key in ('nodeId','scopeId'):saved[key]=saved['proof'][key]=scope[key]
request=bundle(value,[saved]);request['scopes']=[scope]
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
result=rebuild(request)
self.assertEqual(result['visualState'],'ready')
self.assertTrue(result['visuals'])
self.assertEqual(result['visuals'][0]['dimension'],'product')
self.assertEqual(result['queryRefs'][0]['queryIds'],[saved['queryId']])
from decimal import Decimal
decimal_wire = json.loads(json.dumps(request), parse_float=Decimal)
self.assertEqual(rebuild(decimal_wire)['visualState'], 'ready')
def test_trend_is_rebuilt_from_only_the_frozen_response_without_api_or_model(self):
value=task({'product':'18X15X4+T'});saved=query(value);original=copy.deepcopy(saved)
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
+554 -1
View File
@@ -2,6 +2,7 @@
import copy
import io
import json
import re
import sys
import tempfile
import threading
@@ -12,7 +13,7 @@ from urllib.error import HTTPError
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_evidence_finalizer import synthesize_native
from nao_native_answer import build_native_report, numeric_inventory, _validate_blocks, _validate_summary_selection, _within_filters, _bind_exact_year_literals, _select_fact_pool, _binding_inventory, _bind_exact_identifiers, _bind_exact_periods, _bind_literal_measures, _metric_semantics, NativeCompositionError
from nao_native_answer import build_native_report, numeric_inventory, _validate_blocks, _validate_summary_selection, _within_filters, _bind_exact_year_literals, _select_fact_pool, _binding_inventory, _bind_exact_identifiers, _bind_exact_periods, _bind_literal_measures, _metric_semantics, NativeCompositionError, synthesize_native_answer
from nao_native_answer import _affirmative_production_cause, _source_relation_gap, _review_supported_remainder
from test_agent_answer import evidence
@@ -65,6 +66,514 @@ def token(payload, value, fact_id='F1'):
class NativeAnswerTests(unittest.TestCase):
def test_stage_scrap_total_cannot_bind_to_a_named_formal_category(self):
fact = {'id':'F1','kind':'stage','label':'黑片正式指标',
'text':'2026年9月黑片报废重量14.00 kg。','evidenceRef':'Q00000001',
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'14','displayValue':'14.00',
'unit':'kg','metricField':'scrapWeight','metricRole':'scrapWeight','stageHint':'黑片'}]}
tokens = _binding_inventory([fact])
for category in ('责废','料废'):
text = f'正式{category}14.00 kg。'
parsed = _bind_literal_measures({'blocks':[{'text':text,'factIds':['F1']}]},tokens,{},facts=[fact])
with self.subTest(category=category):
self.assertEqual(parsed['blocks'][0]['text'],text)
with self.assertRaises(NativeCompositionError):
_validate_blocks(parsed,[fact],tokens)
self.assertIsNotNone(_source_relation_gap({'text':text,'factIds':['F1']},[fact]))
def test_source_proof_projects_category_total_and_missing_selected_field_separately(self):
period = {'year':2026,'month':9}
result = {'kind':'quality_investigation','action':'source','releaseId':'release-1',
'period':period,'filters':{},'stage':'black',
'current':{'context':{'releaseId':'release-1','stage':'black','year':2026,'month':9},
'totalRows':1,'aggregateComplete':True,'selectedFields':['责废尺寸重量'],
'categories':[{'category':'责废','formalWeight':'7',
'formalCoverage':{'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
'invalidNumericRows':0,'negativeRows':0},
'fields':[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':0,
'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
'negativeRows':0,'numericSum':'3'}}]}],
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,
'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0,
'files':[{'fields':[{'field':'责废尺寸重量','sourceCoverage':{'rows':1,
'numericRows':0,'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
'negativeRows':0,'numericSum':'3'}}]}]}},
'comparisonPeriod':None}
document = {'releaseId':'release-1','evidenceRef':'Q00000001','mode':'investigate','result':result}
context = {'releaseId':'release-1','period':period,'filters':{},
'_agentTask':{'kind':'analysis','period':period,'comparisonPeriod':None,
'filters':{},'stages':['black'],'executionMode':'native_agent'}}
report = build_native_report([document],context,'核对责废原件和正式分量')
category = next(fact for fact in report['facts'] if fact.get('kind') == 'category')
missing = next(fact for fact in report['facts'] if fact.get('kind') == 'source_coverage')
self.assertIn('责废',category['text'])
self.assertIn('7.00 kg',category['text'])
self.assertIn('空值',missing['text'])
self.assertIn('不能按零处理',missing['text'])
self.assertNotIn('3',missing['text'])
typed = category['verifiedNativeNumericValues'][0]
self.assertEqual(typed['metricField'],'categoryWeight')
self.assertEqual(typed['categoryHint'],'责废')
source = next(fact for fact in report['facts'] if fact.get('kind') == 'source')
reconciliation = next(fact for fact in report['facts'] if fact.get('kind') == 'source_reconciliation')
self.assertIn('不证明正式类别与来源细项已对账', source['text'])
self.assertIn('未完整对账', reconciliation['text'])
self.assertFalse(reconciliation['_nativeSourceCategoryReconciliation']['reconciled'])
source_metrics = {value['metricField'] for value in source['verifiedNativeNumericValues']}
self.assertEqual(source_metrics, {'sourceVerifiedRows','sourceExpectedRows','sourceMissingRows','sourceMismatchRows'})
tokens = _binding_inventory(report['facts'])
parsed = _bind_literal_measures({'blocks':[{'text':'正式责废7.00 kg。','factIds':[category['id']]}]},
tokens,{},facts=report['facts'])
blocks, _ = _validate_blocks(parsed,report['facts'],tokens)
self.assertIn('7.00 kg',blocks[0]['text'])
self.assertIsNone(_source_relation_gap(blocks[0],report['facts']))
proof_text = '黑片原表已核验行数与应核验行数均为1,未核验0条,不一致0条。'
proof = _bind_literal_measures({'blocks':[{'text':proof_text,'factIds':[source['id']]}]},tokens,{})
self.assertEqual(proof['blocks'][0]['text'].count('{{N'),3)
proof_blocks, _ = _validate_blocks(proof,report['facts'],tokens)
self.assertIn('已核验行数与应核验行数均为1',proof_blocks[0]['text'])
self.assertIn('未核验0条,不一致0条',proof_blocks[0]['text'])
exact_proof = _bind_literal_measures({'blocks':[{'text':'已核验1行/应核验1行;未核验0条;不一致0条。',
'factIds':[source['id']]}]},tokens,{})
self.assertEqual(exact_proof['blocks'][0]['text'].count('{{N'),4)
_validate_blocks(exact_proof,report['facts'],tokens)
mismatch = _bind_literal_measures({'blocks':[{'text':'未核验1条。','factIds':[source['id']]}]},tokens,{})
with self.assertRaises(NativeCompositionError):
_validate_blocks(mismatch,report['facts'],tokens)
uneven_source = {'id':'F90','text':'已核验1行/应核验2行;未核验1条;不一致0条。',
'evidenceRef':'Q00000090','verifiedNativeNumericValues':[
{'sourceOrdinal':0,'rawValue':'1','displayValue':'1','unit':'metadata_count',
'metricField':'sourceVerifiedRows','sourceVerificationCount':True},
{'sourceOrdinal':1,'rawValue':'2','displayValue':'2','unit':'metadata_count',
'metricField':'sourceExpectedRows','sourceVerificationCount':True},
{'sourceOrdinal':2,'rawValue':'1','displayValue':'1','unit':'metadata_count',
'metricField':'sourceMissingRows','sourceVerificationCount':True},
{'sourceOrdinal':3,'rawValue':'0','displayValue':'0','unit':'metadata_count',
'metricField':'sourceMismatchRows','sourceVerificationCount':True}]}
uneven_tokens = _binding_inventory([uneven_source])
exact_uneven = _bind_literal_measures({'blocks':[{'text':'已核验1行/应核验2行;未核验1条;不一致0条。',
'factIds':['F90']}]},uneven_tokens,{})
self.assertEqual(exact_uneven['blocks'][0]['text'].count('{{N'),4)
_validate_blocks(exact_uneven,[uneven_source],uneven_tokens)
false_equal = _bind_literal_measures({'blocks':[{'text':'已核验行数与应核验行数均为1。',
'factIds':['F90']}]},uneven_tokens,{})
self.assertNotIn('{{N',false_equal['blocks'][0]['text'])
with self.assertRaises(NativeCompositionError):
_validate_blocks(false_equal,[uneven_source],uneven_tokens)
scoped_source = {'id':'F91','kind':'source','label':'黑片原表核验',
'text':'2026年9月黑片规格A已核验1行/应核验2行;未核验1条;不一致0条。',
'evidenceRef':'Q00000091','scope':{'period':{'year':2026,'month':9},
'stage':'black','filters':{'product':'规格A'}},
'verifiedNativeNumericValues':[
{'sourceOrdinal':2,'rawValue':'1','displayValue':'1','unit':'metadata_count',
'metricField':'sourceVerifiedRows','sourceVerificationCount':True,'stageHint':'黑片',
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
{'sourceOrdinal':3,'rawValue':'2','displayValue':'2','unit':'metadata_count',
'metricField':'sourceExpectedRows','sourceVerificationCount':True,'stageHint':'黑片',
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
{'sourceOrdinal':4,'rawValue':'1','displayValue':'1','unit':'metadata_count',
'metricField':'sourceMissingRows','sourceVerificationCount':True,'stageHint':'黑片',
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
{'sourceOrdinal':5,'rawValue':'0','displayValue':'0','unit':'metadata_count',
'metricField':'sourceMismatchRows','sourceVerificationCount':True,'stageHint':'黑片',
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}}]}
scoped_tokens = _binding_inventory([scoped_source])
def bind_scoped(text):
parsed = {'blocks':[{'text':text,'factIds':['F91']}]}
parsed = _bind_exact_periods(parsed,scoped_tokens,{})
parsed = _bind_exact_year_literals(parsed,scoped_tokens,{})
return _bind_literal_measures(parsed,scoped_tokens,{},facts=[scoped_source])
matched = bind_scoped('2026年9月黑片规格A已核验1行。')
_validate_blocks(matched,[scoped_source],scoped_tokens)
for wrong_scope in ('2026年9月亮片规格A已核验1行。',
'2026年8月黑片规格A已核验1行。',
'2026年9月黑片规格B已核验1行。'):
with self.subTest(wrong_scope=wrong_scope):
mismatch = bind_scoped(wrong_scope)
self.assertFalse(any('{{' + key + '}}' in mismatch['blocks'][0]['text']
for key, token_value in scoped_tokens.items() if token_value.get('sourceVerificationCount')))
with self.assertRaises(NativeCompositionError):
_validate_blocks(mismatch,[scoped_source],scoped_tokens)
def test_formal_category_source_relation_needs_same_scope_server_reconciliation(self):
period = {'year':2026,'month':9}
base = {'kind':'quality_investigation','action':'source','releaseId':'release-1',
'period':period,'filters':{'product':'规格06'},'stage':'black',
'current':{'context':{'releaseId':'release-1','stage':'black','year':2026,'month':9,
'product':'规格06'},
'totalRows':1,'aggregateComplete':True,'selectedFields':['责废尺寸重量'],
'categories':[{'category':'责废','formalWeight':'7','rawTotalWeight':None,
'rawTotalCoverage':{'rows':1,'numericRows':0,'keyAbsentRows':0,'nullOrBlankRows':1,
'invalidNumericRows':0,'negativeRows':0},
'formalCoverage':{'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
'invalidNumericRows':0,'negativeRows':0},
'fields':[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':0,
'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
'negativeRows':0,'numericSum':None}}],
'subitemWeightSum':'0','differenceFormalMinusSubitems':'7',
'comparableRows':1,'missingOrInvalidFormalRows':0,'rawFormalMismatchRows':1,
'noFineFields':False,'requiresFurtherInvestigation':True}],
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,
'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0}},
'comparisonPeriod':None}
document = {'releaseId':'release-1','evidenceRef':'Q00000001','mode':'investigate','result':base}
context = {'releaseId':'release-1','period':period,'filters':{'product':'规格06'},
'_agentTask':{'kind':'analysis','period':period,'comparisonPeriod':None,
'filters':{'product':'规格06'},'stages':['black'],'executionMode':'native_agent'}}
report = build_native_report([document],context,'核对规格06责废原件与正式分量')
category = next(fact for fact in report['facts'] if fact.get('kind') == 'category')
source = next(fact for fact in report['facts'] if fact.get('kind') == 'source')
fact_ids = [category['id'],source['id']]
raw_only = '责废7.00 kg;原件所选字段与存储记录匹配,但无法确认正式责废分量与来源细项相等。'
self.assertIsNone(_source_relation_gap({'text':raw_only,'factIds':fact_ids},report['facts']))
raw_label_only = '原件中的责废原始值与原表一致。'
self.assertIsNone(_source_relation_gap({'text':raw_label_only,'factIds':fact_ids},report['facts']))
raw_amount_audit = '原件中的责废原始值7.00 kg已通过原件核验。'
self.assertIsNone(_source_relation_gap({'text':raw_amount_audit,'factIds':fact_ids},report['facts']))
for negative_claim in ('正式责废重量7.00 kg未经原件核验。',
'正式责废重量7.00 kg尚未通过原件核验。'):
self.assertIsNone(_source_relation_gap({'text':negative_claim,'factIds':fact_ids},report['facts']))
explicit_difference = '正式责废重量7.00 kg,与原件细项不一致。'
self.assertIsNone(_source_relation_gap({'text':explicit_difference,'factIds':fact_ids},report['facts']))
invalid_claims = (
'正式责废重量7.00 kg,与原始Excel核验结果一致。',
'正式**责废**重量7.00 kg,与**原件**核验结果相符。',
'正式责废重量7.00 kg已通过原件核验。',
'原件核验确认正式责废重量7.00 kg。',
'责废7.00 kg已通过原件核验。',
'| 正式责废分量 | 原件对账状态 |\n|---|---|\n| 7.00 kg | 与Excel一致 |',
)
for text in invalid_claims:
with self.subTest(text=text):
self.assertIsNotNone(_source_relation_gap({'text':text,'factIds':fact_ids},report['facts']))
reconciled = copy.deepcopy(base)
category_data = reconciled['current']['categories'][0]
category_data.update(rawTotalWeight='7',subitemWeightSum='7',differenceFormalMinusSubitems='0',
rawFormalMismatchRows=0,requiresFurtherInvestigation=False,
rawTotalCoverage={'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
'invalidNumericRows':0,'negativeRows':0},
fields=[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':1,
'nullOrBlankRows':0,'keyAbsentRows':0,'invalidNumericRows':0,'negativeRows':0,
'numericSum':'7'}}])
reconciled_report = build_native_report([{**document,'result':reconciled}],context,
'核对规格06责废原件与正式分量')
reconciled_category = next(fact for fact in reconciled_report['facts'] if fact.get('kind') == 'category')
reconciled_source = next(fact for fact in reconciled_report['facts'] if fact.get('kind') == 'source')
self.assertTrue(reconciled_category['_nativeSourceCategoryReconciliation']['reconciled'])
valid_claim = '正式责废重量7.00 kg,与原表细项对账一致。'
self.assertIsNone(_source_relation_gap({'text':valid_claim,
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
stage_qualified_claim = '黑片正式责废分量与原表细项对账一致。'
self.assertIsNone(_source_relation_gap({'text':stage_qualified_claim,
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
typed_amount_claim = '责废7.00 kg已通过原件核验。'
self.assertIsNone(_source_relation_gap({'text':typed_amount_claim,
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
independent_following_clause = '黑片正式责废分量与原表细项对账一致,综合指标仍使用正式口径。'
self.assertIsNone(_source_relation_gap({'text':independent_following_clause,
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
for wrong_scope_claim in (
'亮片正式责废分量与原表细项对账一致。',
'黑片2026年8月正式责废分量与原表细项对账一致。'):
with self.subTest(wrong_scope_claim=wrong_scope_claim):
self.assertIsNotNone(_source_relation_gap({'text':wrong_scope_claim,
'factIds':[reconciled_category['id'],reconciled_source['id']]},
reconciled_report['facts']))
wrong_scope_source = copy.deepcopy(reconciled_source)
wrong_scope_source['_nativeSourceCategoryReconciliations'][0]['stage'] = 'bright'
wrong_scope_category = copy.deepcopy(reconciled_category)
wrong_scope_category.pop('_nativeSourceCategoryReconciliation',None)
self.assertIsNotNone(_source_relation_gap({'text':valid_claim,
'factIds':[wrong_scope_category['id'],wrong_scope_source['id']]},
[wrong_scope_category,wrong_scope_source]))
def approving_review(request, timeout):
return response({'reportTitle':'原件与正式责废核验','reviews':[{'sourceId':'D1',
'role':'core','factIds':fact_ids,'supported':True,'statisticalClaimsSupported':True}],
'missingGoalDescriptions':[],'visualIds':[]})
unsafe_drafts = (
'正式责废重量7.00 kg,与原始Excel核验结果一致。',
'正式责废重量7.00 kg已通过原件核验。',
'原件核验确认正式责废重量7.00 kg。',
'责废7.00 kg已通过原件核验。',
)
for draft in unsafe_drafts:
with self.subTest(reviewed_draft=draft), patch('nao_native_answer.build_native_report',
return_value=copy.deepcopy(report)):
reviewed = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
model='fixture',question='核对规格06责废正式分量与原件',evidence=[],
cancel_event=threading.Event(),context={'_nativeExecutionId':'exec',
'_dshDraftReview':True,'_autonomousAnswer':True,
'_agentTask':{'kind':'analysis','period':period,'filters':{'product':'规格06'},
'stages':['black']}},opener=approving_review,
native_answer='## 重点\n\n'+draft)
self.assertNotIn(draft,reviewed['answer'])
self.assertTrue(reviewed['compositionDiagnostics'].get('sourceRelationGaps'))
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
negative_reviewed=synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
model='fixture',question='核对规格06责废正式分量与原件',evidence=[],
cancel_event=threading.Event(),context={'_nativeExecutionId':'exec',
'_dshDraftReview':True,'_autonomousAnswer':True,
'_agentTask':{'kind':'analysis','period':period,'filters':{'product':'规格06'},
'stages':['black']}},opener=approving_review,
native_answer='## 重点\n\n正式责废重量7.00 kg未经原件核验。')
self.assertFalse(negative_reviewed['compositionDiagnostics'].get('sourceRelationGaps'))
def test_native_review_repair_requires_remainder_verdicts_and_nonoverlapping_clause_ids(self):
fact = {'id':'F1','kind':'stage','label':'黑片正式指标',
'text':'黑片正式报废重量14.00 kg。','evidenceRef':'Q00000001',
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'14','displayValue':'14.00',
'unit':'kg','metricField':'scrapWeight','stageHint':'黑片',
'period':{'year':2026,'month':9}}]}
report = {'facts':[fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
'scopes':[],'sections':[],'_presentationSources':[],'rejectedEvidenceRefs':[],
'notices':[],'calculationRules':[],'metricFormulaDescriptions':[],
'unverifiedEvidenceWarnings':[]}
draft = '## 重点\n\n黑片正式报废重量14.00 kg。无法支持的观察句。'
calls = []
unsupported = '无法支持的观察句。'
def provider(request, timeout):
messages = json.loads(request.data)['messages']
payload = json.loads(messages[1]['content'])
calls.append(messages)
block = payload['draftBlocks'][0]
bad_id = next(item['clauseId'] for item in block['clauseCatalog']
if item.get('text') == unsupported)
if len(calls) == 1:
parent_id = next(item['clauseId'] for item in block['clauseCatalog']
if item.get('kind') == 'block')
# Parent and child overlap, and the required remainder verdicts
# are deliberately missing to exercise the bounded repair.
row = {'sourceId':block['sourceId'],'role':'core','factIds':['F1'],
'supported':False,'statisticalClaimsSupported':False,
'unsupportedClauseIds':[parent_id,bad_id]}
return response({'reportTitle':'黑片报废依据','reviews':[row],
'missingGoalDescriptions':[],'visualIds':[]})
row = {'sourceId':block['sourceId'],'role':'core','factIds':['F1'],
'supported':False,'statisticalClaimsSupported':False,'unsupportedClauseIds':[bad_id],
'remainingClaimsSupported':True,'remainingStatisticalClaimsSupported':True}
return response({'reportTitle':'黑片报废依据','reviews':[row],
'missingGoalDescriptions':[],'visualIds':[]})
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
final = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
model='fixture',question='解释黑片正式报废重量',evidence=[],cancel_event=threading.Event(),
context={'_nativeExecutionId':'exec','_dshDraftReview':True,
'_agentTask':{'kind':'analysis','period':{'year':2026,'month':9},
'filters':{},'stages':['black']}},opener=provider,native_answer=draft)
self.assertEqual(len(calls),2)
self.assertTrue(final['compositionDiagnostics'].get('reviewClauseRemovals'))
self.assertNotIn(unsupported,final['answer'])
self.assertIn('14.00 kg',final['answer'])
for field in ('remainingClaimsSupported','remainingStatisticalClaimsSupported'):
self.assertIn(field,calls[0][0]['content'])
self.assertIn(field,calls[1][-1]['content'])
self.assertIn('互不重叠',calls[1][-1]['content'])
self.assertIn('父节点与其子句/表格行不可同时选择',calls[1][-1]['content'])
def test_pp_table_headers_bind_same_stage_roles_when_rounded_values_collide(self):
from nao_native_answer import (_native_atomic_inventory,_native_unique_span,
_native_resolve_span,_prepare_native_draft,_native_draft_review_messages)
period={'year':2026,'month':9}
facts=[
{'id':'F1','kind':'stage','label':'黑片自身率变化',
'text':'黑片自身率上升0.15个百分点。','evidenceRef':'Q00000011',
'scope':{'period':period,'filters':{},'stage':'black'},
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'0.14687895375728394',
'displayValue':'0.15','unit':'pp','metricField':'rateChange',
'metricRole':'OwnStageRateChangePp','stageHint':'黑片','period':period,'ownerScope':{}}]},
{'id':'F2','kind':'stage','label':'黑片对综合率贡献',
'text':'黑片对综合率变化贡献为0.15个百分点。','evidenceRef':'Q00000012',
'scope':{'period':period,'filters':{},'stage':'black'},
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'0.14521243189527',
'displayValue':'0.15','unit':'pp','metricField':'stageContribution',
'metricRole':'ContributionPp','stageHint':'黑片','period':period,'ownerScope':{}}]},
]
context={'_agentTask':{'kind':'analysis','period':period,'filters':{},'stages':['black']}}
draft=('| 阶段 | 自身率变化 (pp) | 对综合率贡献 (pp) |\n'
'|---|---:|---:|\n| 黑片 | 0.15 | 0.15 |')
prepared,metadata=_prepare_native_draft(draft,facts,context)
self.assertEqual(metadata['protectedMetricSpans'],2)
by_start={item['start']:item for item in prepared[0]['_protected']}
own_start=draft.index('0.15')
contribution_start=draft.rindex('0.15')
self.assertEqual(by_start[own_start]['factId'],'F1')
self.assertEqual(by_start[contribution_start]['factId'],'F2')
tokens=_native_atomic_inventory(facts)
fact_map={fact['id']:fact for fact in facts}
own_token=next(token for token in tokens.values() if token.get('metricRole')=='OwnStageRateChangePp')
contribution_token=next(token for token in tokens.values() if token.get('metricRole')=='ContributionPp')
for start,correct,wrong in ((own_start,own_token,contribution_token),
(contribution_start,contribution_token,own_token)):
match=re.match(r'0\.15',draft[start:])
self.assertIsNotNone(_native_resolve_span(draft,start,start+len(match[0]),correct,
fact_map,tokens,context=context,require_positive=True))
self.assertIsNone(_native_resolve_span(draft,start,start+len(match[0]),wrong,
fact_map,tokens,context=context,require_positive=True))
for wrong_stage in ('亮片','综合'):
wrong=draft.replace('| 黑片 |','| '+wrong_stage+' |')
self.assertIsNone(_native_unique_span(wrong,re.search(r'0\.15',wrong),fact_map,tokens,context,True))
messages=_native_draft_review_messages(prepared,facts,'说明黑片自身率变化与综合贡献',context,{}, {})
payload=json.loads(messages[1]['content'])
catalog=payload['bindingCatalog']
for token in (own_token,contribution_token):
item=catalog[next(key for key,value in catalog.items() if value['factId']==token['factId'])]
self.assertEqual(item['metricRole'],token['metricRole'])
self.assertEqual(item['metricField'],token['metricField'])
self.assertEqual(item['stage'],'黑片')
self.assertEqual(item['ownerScope'],{})
self.assertEqual(item['period'],period)
self.assertNotEqual(own_token['rawValue'],contribution_token['rawValue'])
def test_dsh_draft_review_flag_preserves_draft_and_unflagged_execution_keeps_presentation(self):
fact = {'id':'F1','kind':'stage','label':'黑片正式指标','text':'黑片报废重量14.00 kg。',
'evidenceRef':'Q00000001','verifiedNativeNumericValues':[{'sourceOrdinal':0,
'rawValue':'14','displayValue':'14.00','unit':'kg','metricField':'scrapWeight',
'metricRole':'scrapWeight','stageHint':'黑片','period':{'year':2026,'month':9}}]}
report = {'facts':[fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
'scopes':[],'sections':[],'_presentationSources':[{'markdown':'### 黑片\n\n报废重量14.00 kg。',
'evidenceRefs':['Q00000001'],'scope':{}}]}
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)), \
patch('nao_native_answer._synthesize_verified_report',return_value={'answer':'checked'}) as finalize:
synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
question='解释黑片报废重量',evidence=[],cancel_event=threading.Event(),
context={'_nativeExecutionId':'exec','_dshDraftReview':True,
'_agentTask':{'kind':'analysis'}},native_answer='## 重点\n\n黑片报废重量14.00 kg。')
self.assertEqual(finalize.call_args.kwargs['native_answer'],'## 重点\n\n黑片报废重量14.00 kg。')
self.assertNotIn('presentationSections',finalize.call_args.kwargs['report'])
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)), \
patch('nao_native_answer.native_sections',return_value=['### 黑片\n\n报废重量14.00 kg。']), \
patch('nao_native_answer._synthesize_verified_report',return_value={'answer':'checked'}) as finalize:
synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
question='解释黑片报废重量',evidence=[],cancel_event=threading.Event(),
context={'_nativeExecutionId':'exec','_agentTask':{'kind':'analysis'}},
native_answer='## 重点\n\n黑片报废重量14.00 kg。')
self.assertIsNone(finalize.call_args.kwargs['native_answer'])
self.assertTrue(finalize.call_args.kwargs['report']['presentationSections'])
def test_stage_contribution_shorthand_binds_only_the_named_stage_and_contribution_role(self):
from nao_native_answer import _native_atomic_inventory, _native_resolve_span
period={'year':2026,'month':10}
facts=[]
for fact_id,field,role,raw,label in [('F1','stageContribution','ContributionPp','-0.14687895375728394','对综合率变化贡献'),
('F2','rateChange','OwnStageRateChangePp','-0.14841364717368','自身率变化')]:
facts.append({'id':fact_id,'kind':'stage','label':'黑片'+label,'text':'黑片'+label+'-0.15个百分点。',
'evidenceRef':'Q00000001','scope':{'period':period,'filters':{},'stage':'black'},
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':raw,'displayValue':'-0.15',
'unit':'pp','metricField':field,'metricRole':role,'stageHint':'黑片','period':period,'ownerScope':{}}]})
tokens=_native_atomic_inventory(facts);known={f['id']:f for f in facts}
contribution=next(t for t in tokens.values() if t.get('metricRole')=='ContributionPp')
own=next(t for t in tokens.values() if t.get('metricRole')=='OwnStageRateChangePp')
context={'_agentTask':{'kind':'analysis','period':period,'filters':{},'stages':['black'],'fields':['rate']}}
text='黑片阶段贡献约 -0.15 pp。';match=re.search('-0.15',text)
self.assertIsNotNone(_native_resolve_span(text,*match.span(),contribution,known,tokens,context=context,require_positive=True))
self.assertIsNone(_native_resolve_span(text,*match.span(),own,known,tokens,context=context,require_positive=True))
for wrong in ('亮片阶段贡献约 -0.15 pp。','黑片对亮片合格率变化贡献约 -0.15 pp。'):
match=re.search('-0.15',wrong)
self.assertIsNone(_native_resolve_span(wrong,*match.span(),contribution,known,tokens,context=context,require_positive=True))
def test_native_formal_presentation_gets_one_bounded_repair_before_independent_review(self):
period={'year':2026,'month':10}
fact={'id':'F1','kind':'stage','label':'黑片合格率','text':'黑片合格率98.84%。','evidenceRef':'Q00000001',
'scope':{'period':period,'filters':{},'stage':'black'}}
report={'facts':[fact],'missingEvidence':[],'requiredFactIds':['F1'],'priorityFactIds':['F1'],'scopes':[],
'sections':[],'notices':[],'rejectedEvidenceRefs':[],'_presentationSources':[{'markdown':'### 黑片合格率\n\n| 指标 | 正式值 |\n|---|---|\n| 黑片合格率 | 98.84% |',
'evidenceRefs':['Q00000001'],'scope':{'period':period,'stage':'black'}}]}
phases=[];generations=[]
def provider(request,timeout):
body=json.loads(request.data)
if body['messages'][0]['content'].startswith('逐句核验证据'):
phases.append('evidence_review')
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
phases.append('native_synthesis');generations.append(body)
if len(generations)==2:
self.assertIn('校验错误',body['messages'][-1]['content'])
self.assertNotIn('{{N',body['messages'][-1]['content'])
value='99.99' if len(generations)==1 else '98.84'
return response({'reportTitle':'2026年10月黑片合格率核对','blocks':[{'role':'core',
'text':'黑片合格率'+value+'%。','factIds':['F1']}],'visualIds':[]})
with patch('nao_native_answer.build_native_report',return_value=report):
result=synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
question='生成黑片合格率报告',evidence=[],cancel_event=threading.Event(),opener=provider,
context={'_nativeExecutionId':'exec','_agentTask':{'kind':'analysis','period':period,
'filters':{},'fields':['rate'],'stages':['black']}},native_answer='未核验的调查草稿')
self.assertTrue(result['evidenceSufficient'],result['compositionDiagnostics'])
self.assertEqual(phases,['native_synthesis','native_synthesis','evidence_review'])
self.assertNotIn('99.99',result['answer'])
self.assertIn('98.84%',result['answer'])
def test_dsh_draft_cannot_keep_category_value_from_stage_total_after_review_true(self):
fact = {'id':'F1','kind':'stage','label':'黑片正式指标','text':'黑片报废重量14.00 kg。',
'evidenceRef':'Q00000001','verifiedNativeNumericValues':[{'sourceOrdinal':0,
'rawValue':'14','displayValue':'14.00','unit':'kg','metricField':'scrapWeight',
'metricRole':'scrapWeight','stageHint':'黑片','period':{'year':2026,'month':9}}]}
category_fact = {'id':'F2','kind':'category','label':'责废',
'text':'黑片正式大类分量:责废为7.00 kg。','evidenceRef':'Q00000002',
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'7','displayValue':'7.00',
'unit':'kg','metricField':'categoryWeight','metricRole':'categoryWeight',
'stageHint':'黑片','categoryHint':'责废','period':{'year':2026,'month':9}}]}
report = {'facts':[fact,category_fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
'scopes':[],'sections':[],'_presentationSources':[],'rejectedEvidenceRefs':[],
'notices':[],'calculationRules':[],'metricFormulaDescriptions':[],
'unverifiedEvidenceWarnings':[]}
context = {'_nativeExecutionId':'exec','_dshDraftReview':True,
'_autonomousAnswer':True,'_agentTask':{'kind':'analysis','period':{'year':2026,'month':9},
'filters':{},'stages':['black']}}
invalid_drafts = ('黑片正式责废重量为**14.00 kg**。',
'黑片正式**责废**重量为14.00 kg。', '黑片正式责废(14.00 kg)。')
for draft in invalid_drafts:
def provider(request, timeout):
return response({'reportTitle':'黑片报废依据','reviews':[{'sourceId':'D1','role':'core',
'factIds':['F1'],'supported':True,'statisticalClaimsSupported':True}],
'missingGoalDescriptions':[],'visualIds':[]})
with self.subTest(draft=draft), patch('nao_native_answer.build_native_report',
return_value=copy.deepcopy(report)):
final = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
model='fixture',question='核对黑片责废重量',evidence=[],cancel_event=threading.Event(),
context=context,opener=provider,native_answer=draft)
self.assertEqual(final['compositionDiagnostics']['compositionMode'],'native_draft_review')
normalized_answer = re.sub(r'[*_`()()\[\]【】]', '', final['answer'])
self.assertNotRegex(normalized_answer, r'责废[^。;\n]{0,20}14\.00\s*kg')
valid_draft = '黑片正式责废重量为7.00 kg。'
def valid_provider(request, timeout):
return response({'reportTitle':'责废正式分量','reviews':[{'sourceId':'D1','role':'core',
'factIds':['F2'],'supported':True,'statisticalClaimsSupported':True}],
'missingGoalDescriptions':[],'visualIds':[]})
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
accepted = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
model='fixture',question='核对黑片责废重量',evidence=[],cancel_event=threading.Event(),
context=context,opener=valid_provider,native_answer=valid_draft)
self.assertTrue(accepted['evidenceSufficient'],accepted['compositionDiagnostics'])
self.assertIn('7.00 kg',accepted['answer'])
tokens = _binding_inventory([fact,category_fact])
table_cases = (
('| 对象 | 责废重量(kg) |\n|---|---:|\n| 黑片 | 14.00 |', ['F1'], False),
('| 大类 | 黑片报废重量(kg) |\n|---|---:|\n| 责废 | 7.00 |', ['F2'], True),
)
for table, fact_ids, supported in table_cases:
with self.subTest(table=table):
parsed = _bind_literal_measures({'blocks':[{'text':table,'factIds':fact_ids}]},tokens,{},facts=[fact,category_fact])
if supported:
validated, _ = _validate_blocks(parsed,[fact,category_fact],tokens)
self.assertIn('7.00',validated[0]['text'])
self.assertIsNone(_source_relation_gap({'text':validated[0]['text'],'factIds':fact_ids},[fact,category_fact]))
else:
self.assertNotIn('{{N',parsed['blocks'][0]['text'])
with self.assertRaises(NativeCompositionError):
_validate_blocks(parsed,[fact,category_fact],tokens)
def test_short_markdown_tables_and_verified_chart_selection_survive_finalization(self):
report = {'facts': [{'id': 'F1', 'text': '合格率95.00%。', 'evidenceRef': 'Q00000001'}],
'requiredFactIds': ['F1'], 'priorityFactIds': [], 'scopes': [],
@@ -87,6 +596,50 @@ class NativeAnswerTests(unittest.TestCase):
self.assertEqual(result['readingSummary']['visualIds'], ['verified-chart'])
self.assertFalse(result['compositionRepairAttempted'])
def test_data_presentation_repair_request_carries_natural_names_and_structure_contract(self):
from nao_native_answer import _synthesize_verified_report
fact = {'id':'F1','kind':'product','label':'产品04',
'text':'2026年9月产品04综合投入重量120.00 kg。','evidenceRef':'Q00000001',
'scope':{'period':{'year':2026,'month':9},'filters':{'product':'产品04'},
'dimension':'product','stage':'overall'}}
report = {'facts':[fact],'requiredFactIds':['F1'],'priorityFactIds':[],'scopes':[],
'missingEvidence':[],'rejectedEvidenceRefs':[],'notices':[],
'presentationSections':[{'markdown':'### 第一章:重点规格\n\n| 规格 | 综合投入重量 |\n|---|---:|\n| 产品04 | 120.00 kg |',
'factIds':['F1'],'evidenceRefs':['Q00000001'],'scope':fact['scope']}]}
calls = []
def provider(request, timeout):
messages = json.loads(request.data)['messages']
calls.append(messages)
if len(calls) == 1:
return response({'reportTitle':'质量分析简报','summarySelection':{'leadIndex':0,'evidenceIndexes':[]},
'blocks':[{'role':'core','text':'产品04综合投入重量{{N30}} kg。','factIds':['F1']}],
'visualIds':[]})
return response({'reportTitle':'质量分析简报','summarySelection':{'leadIndex':0,'evidenceIndexes':[]},
'blocks':[{'role':'core','text':'{{F1}}','factIds':['F1']}],'visualIds':[]})
result = _synthesize_verified_report(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question='按用户指定三章结构分析产品04。',report=report,cancel_event=threading.Event(),
context={'_agentTask':{'kind':'analysis'}},opener=provider)
self.assertEqual(len(calls),2)
self.assertTrue(result['compositionRepairAttempted'])
generation_rules = calls[0][0]['content']
repair_rules = calls[1][-1]['content']
for instruction in ('平台会按已核验来源自动插入完整数据表','不重复列出全部投入规模、分母',
'产品、组织和客户必须写完整原名','保留用户明确指定的章节数量、顺序和标题',
'简短概括放在第一章内','每章Markdown标题与该章正文必须放在同一个block',
'不能把标题单独做成factIds为空的block','N/I/P开头的内部诊断编号'):
self.assertIn(instruction,generation_rules)
for instruction in ('平台会插入完整已核验排名表与投入/分母数据','避免在text中逐行重述排名或重复完整规模/分母',
'产品、组织和客户写完整原名','保留用户指定的章节数量、顺序、标题',
'简短概括并入第一章','每章Markdown标题与该章正文必须留在同一个block',
'不要保留空factIds标题','N/I/P编号只是定位问题的诊断线索'):
self.assertIn(instruction,repair_rules)
self.assertIn('校验错误中的N/I/P编号只是定位问题的诊断线索',repair_rules)
self.assertIn('不可用编号替代产品名、组织名或指标名',repair_rules)
self.assertIn('保留用户指定的章节数量、顺序、标题',repair_rules)
self.assertNotIn('N30',repair_rules)
self.assertNotIn('改用已有证据的token',repair_rules)
self.assertTrue(result['evidenceSufficient'],result['compositionDiagnostics'])
def test_strong_defect_comparison_needs_same_scope_counterpart_and_deletes_only_clause(self):
scope = {'filters': {'department': '新能源汽车磁钢'}, 'period': {'year': 2026, 'month': 9},
'comparisonPeriod': {'year': 2026, 'month': 8}, 'dimension': 'defect', 'stage': 'black'}
+315 -1
View File
@@ -1,5 +1,6 @@
"""Native tool loop boundaries, using actual tool adapters and captured-like APIs."""
import copy
import json
import sys
import tempfile
import unittest
@@ -10,7 +11,7 @@ from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_native_exploration import prepare_step, NativeExplorationTools, _proofs, native_model_view
from nao_agent_task import normalize_task, plan_task
from nao_quality_tools import QualityToolError
from nao_quality_tools import QualityToolError, tool_definition
from test_nao_protocol_execution import BoundApi, identity
from test_nao_quality_tools import RELEASE
from test_nao_agent_task import task_value
@@ -59,6 +60,80 @@ class Delegated(BoundApi):
class NativeExplorationTests(unittest.TestCase):
def state(self): return {'releaseId': RELEASE, 'scopes': []}
def test_report_rank_is_available_without_a_preplanned_rank_dimension(self):
task=root();task.update(dimensions=['defect'],comparisonPeriod=None,
question='生成2026年1—10月总体质量分析报告,重点说明总体表现、月度变化和主要报废结构。')
schema=tool_definition(native_protocol=True,task=task,release_id=RELEASE)['inputSchema']
self.assertIn('rank',schema['properties']['mode']['enum'])
self.assertNotIn('investigate',schema['properties']['mode']['enum'])
def test_captured_rank_calls_explain_missing_metric_before_proposing_any_scope(self):
for dimension in ('department','product'):
with self.subTest(dimension=dimension), self.assertRaisesRegex(QualityToolError,'缺少metric'):
prepare_step({'mode':'rank','stage':'black','dimension':dimension,'limit':10},root(),self.state())
def test_public_rank_trace_retains_the_actual_metric_without_credentials(self):
from nao_quality_runtime import public_tool_call
call=public_tool_call('mcp_call',{'tool':'quality_data','arguments':{
'mode':'rank','metric':'totalWeight','stage':'black','dimension':'product','token':'private'}})
self.assertEqual(call['input']['metric'],'totalWeight')
self.assertEqual(call['label'],'查询重量排名')
self.assertNotIn('token',call['input'])
def test_batch_overview_uses_one_bound_scope_and_overall_alias_cannot_widen_parent(self):
task=root();task['filters']={'department':'A'}
args,proposal=prepare_step({'mode':'query','view':'group','filters':{'department':'__ALL__'}},task,self.state())
self.assertEqual(args['filters'],{'department':'A'})
self.assertEqual(args['period'],task['period'])
self.assertEqual(args['limit'],100)
self.assertEqual(proposal['node']['purpose'],'metrics')
self.assertEqual(proposal['node']['dimension'],'group')
self.assertEqual(proposal['node']['selector'],{'kind':'base'})
def test_inactive_rows_are_omitted_from_model_projection_without_recalculating_summary(self):
tools=NativeExplorationTools(delegated=None,evidence_dir=Path('/unused'),task=root(),identity={},
state_path=Path('/unused/state'),directory={'departments':['有效产业部']})
raw={'view':'department','summary':{'compositeRate':.95},'rows':[{'key':'有效产业部'},
{'key':'欧洲磁钢'},{'key':'美洲产业部'}],'rowCount':3,'rowsTruncated':False}
view=tools.active_result(raw)
self.assertEqual(view['rows'],[{'key':'有效产业部'}])
self.assertEqual(view['summary'],raw['summary'])
self.assertEqual(len(raw['rows']),3)
def test_no_published_rows_are_cached_as_no_data_with_original_proof(self):
from nao_quality_tools import SourceCoverageUnavailable
with tempfile.TemporaryDirectory() as folder:
p=Path(folder);api=Delegated()
tools=NativeExplorationTools(delegated=api,evidence_dir=p/'evidence',task=root(),
identity={**identity(),'releaseId':RELEASE},state_path=p/'state.json')
raw={'coverage':{'sourceMonths':[]},'protocolEvidence':{**identity(),'queryId':'empty-query'}}
with patch('nao_quality_tools.QualityTools.quality_data',side_effect=SourceCoverageUnavailable(root()['period'],raw)) as query:
first=tools.quality_data({'mode':'query'})
second=tools.quality_data({'mode':'query'})
self.assertEqual(query.call_count,1)
self.assertEqual(first['status'],'no_data')
self.assertIsNone(first['summary'])
self.assertEqual(first['protocolEvidence'],raw['protocolEvidence'])
self.assertTrue(second['reusedEvidence'])
@staticmethod
def source_task():
value = root()
value.update(kind='analysis', fields=[], dimensions=[], dimensionStages={}, stages=['black'],
period={'year': 2026, 'month': 9}, comparisonPeriod=None,
comparisonRequired=False, trendRequired=False, verifySource=True,
fullDetails=False)
return value
@staticmethod
def source_catalog():
return {'releaseId': RELEASE, 'sourceFields': [
{'key': 'src_responsibility_detail', 'label': '责废细项重量', 'stages': ['black'],
'columns': {'black': '原表责废细项重量', 'bright': None}},
{'key': 'src_black_total', 'label': '黑片报废合计', 'stages': ['black'],
'columns': {'black': '原表黑片报废合计', 'bright': None}},
]}
def test_model_view_removes_only_transport_and_identical_signal_duplicates(self):
row={'key':'甲','label':'甲','rank':1,'current':{'rate':0,'scrapWeight':None},
'comparison':{'rate':None,'scrapWeight':0}}
@@ -85,6 +160,15 @@ class NativeExplorationTests(unittest.TestCase):
changed=native_model_view(source)
self.assertIn('rateDrop',changed['prioritySignals']['scale'][0])
self.assertIsNone(changed['prioritySignals']['scale'][0]['rateDrop'])
def test_official_category_share_names_its_actual_denominator_without_mutating_signed_values(self):
source={'kind':'quality_metrics','summary':{'bright':{'rate':.98,'scrapWeight':100,
'defects':[{'name':'加工废','weight':30,'rate':.6},{'name':'镀废','weight':20,'rate':.4}]}}}
original=copy.deepcopy(source);view=native_model_view(source)['summary']['bright']
self.assertEqual(view['rate'],.98)
self.assertEqual(view['defects'][0]['shareOfListedCategoryWeight'],.6)
self.assertNotIn('rate',view['defects'][0])
self.assertEqual(view['categoryShareBasis'],'sum_of_listed_category_weights')
self.assertEqual(source,original)
def test_explicit_cause_cannot_silently_become_a_static_recipe(self):
from nao_native_planning import explicit_investigation
self.assertTrue(explicit_investigation('为什么下降?请自行深挖'))
@@ -292,6 +376,178 @@ class NativeExplorationTests(unittest.TestCase):
self.assertEqual(args['limit'], 100)
self.assertEqual(proposal['node']['purpose'], 'source')
def test_native_source_fields_map_only_exact_current_catalog_keys_labels_and_columns(self):
task = self.source_task()
catalog = self.source_catalog()
args, proposal = prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': ['责废细项重量']}, task, self.state(), source_catalog=catalog)
self.assertEqual(args['fields'], ['src_responsibility_detail'])
self.assertEqual(proposal['node']['sourceFields'], {'black': ['src_responsibility_detail']})
args, proposal = prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': ['src_black_total']}, task, self.state(), source_catalog=catalog)
self.assertEqual(args['fields'], ['src_black_total'])
self.assertEqual(proposal['node']['sourceFields'], {'black': ['src_black_total']})
duplicate_label = copy.deepcopy(catalog)
duplicate_label['sourceFields'].append({'key': 'src_duplicate', 'label': '责废细项重量',
'stages': ['black'], 'columns': {'black': '另一原列', 'bright': None}})
for bad_catalog, bad_field, error in (
(catalog, '未经授权的字段', '不在当前releaseId'),
(catalog, 'src_bright_only', '不在当前releaseId'),
(duplicate_label, '责废细项重量', '不唯一')):
with self.subTest(field=bad_field), self.assertRaisesRegex(QualityToolError, error):
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': [bad_field]}, task, self.state(), source_catalog=bad_catalog)
stale = {**catalog, 'releaseId': 'older-release'}
with self.assertRaisesRegex(QualityToolError, '不属于当前releaseId'):
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': ['src_responsibility_detail']}, task, self.state(), source_catalog=stale)
def test_source_scope_rejects_unsupported_actions_and_category_without_registering(self):
task = self.source_task()
for action in ('overview', 'fields'):
with self.subTest(action=action), self.assertRaisesRegex(QualityToolError, '只支持action=source或rows'):
prepare_step({'mode': 'investigate', 'action': action, 'stage': 'black'}, task, self.state())
with self.assertRaisesRegex(QualityToolError, '不支持category筛选.*categoryReconciliation'):
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'category': '责废'}, task, self.state())
with self.assertRaisesRegex(QualityToolError, '未签发sourceRevisionId'):
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'sourceRevisionId': 'revision-2'}, task, self.state())
# Category filtering remains a supported defect query and is not silently stripped.
_, proposal = prepare_step({'mode': 'defects', 'stage': 'black', 'dimension': 'defect',
'category': '责废'}, root(), self.state())
self.assertEqual(proposal['node']['category'], '责废')
with tempfile.TemporaryDirectory() as folder:
path = Path(folder); api = Delegated()
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=task,
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
with self.assertRaisesRegex(QualityToolError, 'categoryReconciliation'):
tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'category': '责废'})
self.assertEqual(api.steps, [])
def test_dsh_schema_uses_only_protocol_supported_investigation_actions(self):
legacy = tool_definition()
native = tool_definition(native_protocol=True)
self.assertEqual(legacy['inputSchema']['properties']['action']['enum'],
['overview', 'rows', 'fields', 'source'])
self.assertEqual(native['inputSchema']['properties']['action']['enum'], ['source', 'rows'])
self.assertIn('正式指标概览使用query', native['description'])
self.assertIn('字段目录使用catalog', native['description'])
self.assertIn('categoryReconciliation', native['inputSchema']['properties']['category']['description'])
query_rule = next(rule for rule in native['inputSchema']['allOf']
if rule.get('if', {}).get('properties', {}).get('mode', {}).get('const') == 'query')
query_forbidden = [clause['not']['required'][0] for clause in query_rule['then']['allOf']]
self.assertIn('stage', query_forbidden)
self.assertIn('category', query_forbidden)
native_raw_scope = next(rule for rule in native['inputSchema']['allOf']
if rule.get('if', {}).get('properties', {}).get('action', {}).get('enum') == ['source', 'rows'])
self.assertEqual([clause['not']['required'][0] for clause in native_raw_scope['then']['allOf']],
['category', 'sourceRevisionId'])
def test_source_query_signs_catalog_stable_key_and_sends_only_signed_original_column(self):
source_fields = [{'key': 'src_responsibility_detail', 'label': '责废细项重量',
'stages': ['black'], 'columns': {'black': '原表责废细项重量', 'bright': None}}]
fine_catalog = {'kind': 'fine_defect_catalog', 'grain': 'source_defect_weight',
'fields': [{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
'category': '责废', 'available': True}],
'categories': [{'stage': 'black', 'category': '责废', 'available': True}]}
class SourceApi(Delegated):
def __init__(self):
super().__init__()
self.registered = []
self.source_payloads = []
self.registered_scopes = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
if path == '/api/analysis/model-authorization':
self.authorization_calls += 1
return {}
if path == '/api/releases':
return {'currentReleaseId': RELEASE, 'items': [
{'id': RELEASE, 'name': 'Published', 'status': 'PUBLISHED'}]}
if path == '/api/options':
return {'years': [2026], 'departments': [], 'groups': [], 'products': [], 'projectGroups': []}
if path == '/api/analysis/tools/catalog':
return {'kind': 'analysis_catalog', 'visibleWeightFields': [], 'visibleDefectFields': [],
'dimensions': [], 'topDimensions': [], 'grouping': ['overall'], 'limitations': [],
'sourceFields': copy.deepcopy(source_fields), 'fineDefects': copy.deepcopy(fine_catalog)}
if path == '/api/analysis/protocol/exploration/steps':
self.registered.append(copy.deepcopy(payload))
node = payload['node']
task = NativeExplorationTests.source_task()
task['stages'] = copy.deepcopy(node['stages'])
selected = node.get('sourceFields')
if selected:
task['sourceFields'] = copy.deepcopy(selected)
task['investigationFields'] = {
stage: [next(row['columns'][stage] for row in source_fields if row['key'] == key)
for key in keys]
for stage, keys in selected.items()}
registered = {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
'task': task, 'scope': {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
'purpose': node['purpose'], 'task': task}}
self.registered_scopes.append(registered)
return copy.deepcopy(registered)
if path == '/api/analysis/investigate':
self.source_payloads.append(copy.deepcopy(payload))
period = {key: payload[key] for key in ('year', 'month', 'quarter', 'monthStart', 'monthEnd')
if key in payload}
context = {**period, 'releaseId': RELEASE, 'stage': payload['stage'], 'category': None,
'sourceRevisionId': None, 'includeUnassigned': False,
'department': payload.get('department'), 'group': payload.get('group'),
'product': payload.get('product'), 'projectGroup': payload.get('projectGroup'),
'extraFilters': payload.get('extraFilters', {}), 'columnFilters': payload.get('columnFilters', []),
'exclusions': payload.get('exclusions', {})}
proof = {**identity(), 'queryId': 'source-query', 'responseHash': 'a' * 64,
'nodeId': node_id, 'scopeId': scope_id}
return {'kind': 'quality_investigation', 'action': 'source', 'releaseId': RELEASE,
'context': context, 'totalRows': 1, 'aggregateComplete': True,
'categories': [{'category': '责废', 'categoryReconciliation': {'status': 'INCOMPLETE'}}],
'lineage': [], 'selectedFields': copy.deepcopy(payload.get('fields')),
'sourceVerification': {'expectedRows': 1, 'complete': True, 'verifiedRows': 1,
'missingRows': 0, 'mismatchRows': 0, 'allSelectedValuesMatchStored': True},
'protocolEvidence': proof}
if path == '/api/analysis/protocol/queries':
return {'scopeCoverage': [{'nodeId': row['nodeId'], 'scopeId': row['scopeId'],
'complete': True, 'missingEvidence': []} for row in self.registered_scopes], 'rootGoalGaps': []}
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder)
api = SourceApi()
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=self.source_task(),
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
tools.quality_data({'mode': 'catalog'})
result = tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': ['责废细项重量']})
self.assertEqual(len(api.registered), 1)
signed_node = api.registered[0]['node']
self.assertEqual(signed_node['sourceFields'], {'black': ['src_responsibility_detail']})
self.assertEqual(api.source_payloads[0]['fields'], ['原表责废细项重量'])
self.assertNotIn('category', api.source_payloads[0])
self.assertEqual(result['action'], 'source')
self.assertEqual(result['category'], None)
self.assertEqual(_proofs(result)[0]['nodeId'], signed_node['nodeId'])
def test_native_source_fields_require_catalog_from_current_release_before_registration(self):
class NoCatalog(Delegated):
pass
with tempfile.TemporaryDirectory() as folder:
path = Path(folder); api = NoCatalog()
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=self.source_task(),
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
with self.assertRaisesRegex(QualityToolError, '当前releaseId的catalog'):
tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
'fields': ['src_responsibility_detail']})
self.assertEqual(api.steps, [])
def test_multilevel_child_inherits_verified_parent(self):
state = self.state(); state['scopes'] = [{'filters': {'group': '甲'}, 'queryIds': ['q1']}]
_, step = prepare_step({'mode': 'defects', 'stage': 'black', 'filters': {'group': '甲', 'product': '乙'}}, root(), state)
@@ -314,6 +570,64 @@ class NativeExplorationTests(unittest.TestCase):
tools.quality_data({'mode': 'compare', 'dimension': 'product'})
self.assertEqual(api.steps, [])
def test_catalog_reads_only_authorized_metadata_and_formal_reads_still_register_scope(self):
fine_catalog = {
'kind': 'fine_defect_catalog', 'grain': 'source_defect_weight',
'fields': [
{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
'category': '责废', 'sourceField': '责废尺寸重量',
'grain': 'source_defect_weight', 'available': True},
{'stage': 'bright', 'key': '镀废重量', 'label': '镀废重量',
'category': '镀废', 'sourceField': '镀废重量',
'grain': 'source_defect_weight', 'available': True},
],
'categories': [
{'stage': 'black', 'category': '责废', 'available': True},
{'stage': 'bright', 'category': '镀废', 'available': True},
],
}
class CatalogApi(Delegated):
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
result = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
if path == '/api/analysis/tools/catalog':
result = {**result, 'fineDefects': copy.deepcopy(fine_catalog)}
return result
with tempfile.TemporaryDirectory() as folder:
p = Path(folder); api = CatalogApi()
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
catalog = tools.quality_data({'mode': 'catalog'})
self.assertEqual(catalog['periods']['availableMonthsInLatestYear'], [])
self.assertEqual(catalog['periods']['availableMonthsSource'], 'not_scanned')
self.assertEqual(catalog['fineDefects']['endpoint'], 'GET /api/analysis/tools/catalog')
self.assertEqual(catalog['fineDefects']['fields'], fine_catalog['fields'])
self.assertEqual(catalog['fineDefects']['scanned'], False)
self.assertNotIn('protocolEvidence', catalog)
narrowed = tools.quality_data({'mode': 'catalog', 'stage': 'black'})
self.assertEqual([item['stage'] for item in narrowed['fineDefects']['fields']], ['black'])
self.assertEqual([item['stage'] for item in narrowed['fineDefects']['categories']], ['black'])
paths = [call[1].split('?', 1)[0] for call in api.calls]
self.assertTrue(set(paths) <= {
'/api/analysis/model-authorization', '/api/releases', '/api/analysis/tools/catalog', '/api/options'
}, paths)
self.assertNotIn('/api/metrics', paths)
self.assertNotIn('/api/analysis/defects', paths)
for _ in range(3):
duplicate = tools.quality_data({'mode': 'catalog'})
self.assertTrue(duplicate['reusedEvidence'])
state = json.loads((p/'state.json').read_text())
self.assertEqual(state['noProgress'], 0)
self.assertFalse(state.get('halted', False))
tools.quality_data({'mode': 'compare'})
self.assertEqual(len(api.steps), 1)
self.assertTrue(any(path.startswith('/api/metrics') for _, path, _ in api.calls))
def test_actual_compare_preserves_both_proofs_and_duplicate_reads_are_cached(self):
with tempfile.TemporaryDirectory() as folder:
api = Delegated(); p = Path(folder)
@@ -21,5 +21,13 @@ class NativeLayoutCapacityTests(unittest.TestCase):
with self.assertRaises(NativeCompositionError):_native_draft_blocks(draft)
def test_layout_only_cannot_become_a_business_answer(self):
with self.assertRaises(NativeCompositionError):_native_draft_blocks('## 标题\n\n---\n\n* * *')
def test_full_report_packs_short_paragraphs_without_removing_or_reordering_text(self):
draft='## 期间汇总\n\n'+'\n\n'.join('第'+str(i)+'段已核实内容。' for i in range(30))
blocks=_native_draft_blocks(draft,pack_report=True)
self.assertLessEqual(len(blocks),24)
self.assertEqual('\n\n'.join(b['text'] for b in blocks),draft)
self.assertTrue(all(len(b['text'])<=2200 for b in blocks))
with self.assertRaises(NativeCompositionError):
_native_draft_blocks('| 完整表格 |\n'+'字'*2201,pack_report=True)
if __name__=='__main__':unittest.main()
@@ -0,0 +1,23 @@
"""Core safety requirements that must remain in the DSH planning contract."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_native_planning import NATIVE_ROOT_PROMPT, STATIC_NODE_CONTRACT
class NativePlanningContractTests(unittest.TestCase):
def test_ambiguous_catalog_prefix_requires_a_user_choice(self):
self.assertIn('边界前缀对应多个候选时必须needsClarification=true', NATIVE_ROOT_PROMPT)
self.assertIn('清除该产品filter', NATIVE_ROOT_PROMPT)
self.assertIn('不能把前缀直接传为产品筛选值', NATIVE_ROOT_PROMPT)
def test_source_receipts_do_not_substitute_for_formal_metric_reconciliation(self):
self.assertIn('两者都不提供正式指标或正式缺陷分类', STATIC_NODE_CONTRACT)
self.assertIn('还必须规划能返回对应正式值的metrics节点和/或正式defects节点', STATIC_NODE_CONTRACT)
self.assertIn('不得用“原件核验通过”或“记录已读全”的回执代替对账结果', STATIC_NODE_CONTRACT)
if __name__ == '__main__':
unittest.main()
+30
View File
@@ -71,6 +71,36 @@ class NativeStructureTests(unittest.TestCase):
self.assertIn('规格'+name,location['inheritedText'])
self.assertEqual(sum('规格'+other in location['inheritedText'] for other in '甲乙丙'),1)
def test_closed_annotation_siblings_do_not_leak_subjects_even_when_values_match(self):
tokens={'N1':{'ownerScope':{'group':'合成组织甲'}},
'N2':{'ownerScope':{'group':'合成组织乙'}}}
context={'_agentTask':{'period':{'year':2026,'month':9},'filters':{}},
'_nativeScopeCatalog':[{'group':'合成组织甲'},{'group':'合成组织乙'}]}
for second_rate in ('-0.28','-0.29'):
text=('合成组织甲(率降-0.28pp,报废增量+12kg)和'
f'合成组织乙(率降{second_rate}pp,报废增量+13kg)。')
for span in (second_rate,'+13'):
start=text.rindex(span)
location=native_span_structure(text).locate(start,start+len(span))
with self.subTest(second_rate=second_rate,span=span):
self.assertIn('合成组织乙',location['inheritedText'])
self.assertNotIn('合成组织甲',location['inheritedText'])
slots=_native_span_slots(text,start,start+len(span),context,tokens,
'pp' if span.startswith('-') else 'kg',location)
self.assertFalse(slots['scopeConflict'],slots)
self.assertEqual(slots['scope'].get('group'),'合成组织乙')
nested='范围:父部门(合成组织乙(率降-0.28pp))'
start=nested.index('-0.28')
location=native_span_structure(nested).locate(start,start+5)
nested_context={'_agentTask':{'period':{'year':2026,'month':9},
'filters':{'department':'父部门'}},
'_nativeScopeCatalog':[{'department':'父部门','group':'合成组织乙'}]}
nested_tokens={'N1':{'ownerScope':{'department':'父部门','group':'合成组织乙'}}}
slots=_native_span_slots(nested,start,start+5,nested_context,nested_tokens,'pp',location)
self.assertFalse(slots['scopeConflict'],slots)
self.assertEqual(slots['scope'],{'department':'父部门','group':'合成组织乙'})
def test_versions_and_explicit_ancestors_are_separate(self):
blocks=[{'sourceId':'D1','text':'## 产业部:部门甲\n\n分组比较:'},
{'sourceId':'D2','text':'| 分组 | 本期 |\n|---|---|\n| 分组乙 | 10 |'}]
@@ -4,6 +4,7 @@ import json
import copy
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
@@ -11,7 +12,7 @@ from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
from nao_product_identification import identify_products
from nao_product_identification import identify_products, resolve_clarification_choice
ACTOR = '11111111-1111-4111-8111-111111111111'
SESSION = '22222222-2222-4222-8222-222222222222'
@@ -20,6 +21,89 @@ TURN = '55555555-5555-4555-8555-555555555555'
class ProductClarificationRuntimeTests(unittest.TestCase):
def test_catalog_prefix_with_multiple_candidates_forces_explicit_choice(self):
question = ('查询2026年10月合成产品0的黑片合格率;如果名称不能唯一确定,'
'请让我明确选择,不要自行猜产品。')
authorized = [f'合成产品0{i}' for i in range(1, 7)]
options = {'products': authorized, 'organizationPaths': []}
match = identify_products(question, options)['productMatch']
self.assertEqual(match['status'], 'ambiguous')
self.assertEqual(match['query'], '合成产品0')
self.assertEqual(match['candidateCount'], 6)
self.assertFalse(match['truncated'])
self.assertEqual([item['name'] for item in match['candidates']], authorized)
# Even if the planner mistakenly forwards the partial name as a filter
# and omits needsClarification, the deterministic authorized catalog
# overrides it while preserving the user's metric and scope.
from nao_agent_task import plan_task
planned = {
'goal': question, 'kind': 'metrics', 'freshQuery': True,
'dimensions': [], 'dimensionStages': {}, 'stages': ['black'],
'fields': ['rate'], 'period': {'year': 2026, 'month': 10},
'comparisonRequired': False, 'trendRequired': False,
'verifySource': False, 'fullDetails': False,
'visualPreference': 'none', 'responseStyle': 'standard',
'needsClarification': False, 'clarification': '',
'filterUpdates': {'product': '合成产品0'},
}
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
'_businessCatalog': {'productMatch': match}}
with patch('nao_agent_task._model_json', return_value=planned):
pending = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question=question, context=context, cancel_event=threading.Event())
self.assertTrue(pending['needsClarification'])
self.assertNotIn('product', pending['filters'])
self.assertEqual(pending['period'], {'year': 2026, 'month': 10})
self.assertEqual(pending['fields'], ['rate'])
self.assertEqual(pending['stages'], ['black'])
self.assertEqual([item['value'] for item in pending['clarificationRequest']['options']], authorized)
selected = resolve_clarification_choice('选择合成产品03', options, pending)
self.assertEqual(selected['productMatch']['status'], 'exact')
self.assertTrue(selected['productMatch']['selectedFromClarification'])
self.assertEqual(selected['productMatch']['candidates'][0]['name'], '合成产品03')
def test_prefix_matching_stays_with_current_authorized_catalog_and_candidate_cap(self):
question = '查询合成产品0的黑片合格率'
names = [f'合成产品0{i:02d}' for i in range(25)]
match = identify_products(question, {'products': names})['productMatch']
self.assertEqual(match['status'], 'ambiguous')
self.assertEqual(match['candidateCount'], 25)
self.assertEqual(len(match['candidates']), 20)
self.assertTrue(match['truncated'])
self.assertEqual(set(item['name'] for item in match['candidates']), set(names[:20]))
not_authorized = identify_products('查询合成产品99的黑片合格率', {'products': names})['productMatch']
self.assertEqual(not_authorized['status'], 'none')
self.assertEqual(not_authorized['candidates'], [])
unique = identify_products('查询合成产品0的黑片合格率', {'products': ['合成产品01']})['productMatch']
self.assertEqual(unique['status'], 'unique')
self.assertEqual([item['name'] for item in unique['candidates']], ['合成产品01'])
# A unique partial prefix is bound to the full authorized directory
# name, never sent to the API as a partial product filter.
from nao_agent_task import plan_task
question = '查询合成产品0的黑片合格率'
planned = {
'goal': question, 'kind': 'metrics', 'freshQuery': True,
'dimensions': [], 'dimensionStages': {}, 'stages': ['black'],
'fields': ['rate'], 'period': {'year': 2026, 'month': 10},
'comparisonRequired': False, 'trendRequired': False,
'verifySource': False, 'fullDetails': False,
'visualPreference': 'none', 'responseStyle': 'standard',
'needsClarification': False, 'clarification': '',
'filterUpdates': {'product': '合成产品0'},
}
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
'_businessCatalog': {'productMatch': unique}}
with patch('nao_agent_task._model_json', return_value=planned):
task = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question=question, context=context, cancel_event=threading.Event())
self.assertFalse(task['needsClarification'])
self.assertEqual(task['filters']['product'], '合成产品01')
def exercise(self, *, restart=False, revoke=False, other_release=False):
opts = {'products': ['18X15X4+T', '18X15X4+Z'], 'organizationPaths': []}
pending_match = identify_products('今年18x15x4合格率变化趋势', opts)['productMatch']