统一:DSH 智能分析、Top100 与图表静默恢复
接入 DSH 专用运行时和对话界面,保留模型原始回答,支持会话分页、连续追问与显式报告。 补齐组合筛选、各维度 Top100、分组查询到排名图的转换,图表故障静默省略并基于保存数据自动恢复。 优化报告风格、图表顺序和表格换行,更新接入说明及构建输入。 验证:独立目录前后端构建通过;118 项 Java、102 项前端、102 项图表、14 项 DSH 应用和 13 项组合筛选检查通过。 仅提交相关源码、锁定依赖与测试;不包含凭据、会话、业务数据或本地产物。
This commit is contained in:
@@ -0,0 +1,274 @@
|
||||
"""Synthetic scale benchmark in a disposable database and a separate loopback API.
|
||||
|
||||
The live release and data are never modified. Replicas retain the sample's 15
|
||||
source periods and full JSON column widths; they are not real annual business data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import http.cookiejar
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, build_opener, HTTPCookieProcessor
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import backup
|
||||
import project
|
||||
import managed_test_resources as test_resources
|
||||
|
||||
OUT = ROOT / 'output/completion'
|
||||
|
||||
|
||||
def ident(value):
|
||||
if not value.replace('_', '').isalnum():
|
||||
raise ValueError('Invalid isolated database identifier')
|
||||
return '`' + value + '`'
|
||||
|
||||
|
||||
def sql(statement, database=None, empty=False):
|
||||
create = re.fullmatch(r'CREATE DATABASE `?([a-z0-9_]+)`? CHARACTER SET [a-z0-9_]+(?: COLLATE [a-z0-9_]+)?;?', statement.strip(), flags=re.I)
|
||||
if create:
|
||||
return test_resources.create(ROOT, create[1], backup.client, statement, empty=empty)
|
||||
drop = re.fullmatch(r'DROP DATABASE(?: IF EXISTS)? `?([a-z0-9_]+)`?;?', statement.strip(), flags=re.I)
|
||||
if drop:
|
||||
return test_resources.drop(ROOT, drop[1], backup.client)
|
||||
result = backup.client(statement, database)
|
||||
user = re.fullmatch(r"DROP USER(?: IF EXISTS)? '([a-z0-9_]+)'@'127\.0\.0\.1';?", statement.strip(), flags=re.I)
|
||||
if user:
|
||||
test_resources.user_dropped(ROOT, user[1])
|
||||
return result
|
||||
|
||||
|
||||
def clone_database(source, target):
|
||||
tables = sql('SHOW TABLES;', source).splitlines()
|
||||
for table in tables:
|
||||
sql(f'CREATE TABLE {ident(target)}.{ident(table)} LIKE {ident(source)}.{ident(table)};')
|
||||
if table not in ('fact_black', 'fact_bright'):
|
||||
sql(f'INSERT INTO {ident(target)}.{ident(table)} SELECT * FROM {ident(source)}.{ident(table)};')
|
||||
# A copied database must never resume the live installation's pending jobs.
|
||||
for table in ('task', 'import_run'):
|
||||
if table in tables:
|
||||
sql(f"UPDATE {ident(table)} SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING','RETRY_WAIT','CANCEL_REQUESTED');", target)
|
||||
for table in ('backup_plan', 'quality_inspection_strategy'):
|
||||
if table in tables:
|
||||
sql(f'UPDATE {ident(table)} SET active=FALSE;', target)
|
||||
if 'analysis_turn' in tables and 'status' in [row.split('\t')[0] for row in sql('SHOW COLUMNS FROM analysis_turn;', target).splitlines()]:
|
||||
sql("UPDATE analysis_turn SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING');", target)
|
||||
return tables
|
||||
|
||||
|
||||
def seed_facts(source, target, stage, count):
|
||||
table = 'fact_' + stage
|
||||
columns = [line.split('\t')[0] for line in sql(f'SHOW COLUMNS FROM {ident(table)};', source).splitlines()]
|
||||
columns.remove('id')
|
||||
# Replicate only the unique builds in the selected current release. Including
|
||||
# older transform builds would inflate table size without exercising the
|
||||
# promised number of query-visible records.
|
||||
selected = "EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id)"
|
||||
seed_count = int(sql(f'SELECT COUNT(*) FROM {ident(table)} f WHERE {selected};', source).strip())
|
||||
if not seed_count:
|
||||
raise RuntimeError('Source stage has no seed rows')
|
||||
offset = int(sql(f'SELECT MAX(physical_row_no)+1 FROM {ident(table)};', source).strip())
|
||||
repeats = math.ceil(count / seed_count)
|
||||
if offset * (repeats + 1) > 2_147_483_647:
|
||||
raise ValueError('Synthetic row identity exceeds the supported integer range')
|
||||
sql('CREATE TEMPORARY TABLE scale_numbers (n INT PRIMARY KEY);' +
|
||||
'INSERT INTO scale_numbers VALUES ' + ','.join(f'({i})' for i in range(repeats)) + ';' +
|
||||
f'INSERT INTO {ident(table)} (' + ','.join(ident(c) for c in columns) + ') SELECT ' +
|
||||
','.join(f'f.{ident(c)} + n.n*{offset}' if c == 'physical_row_no' else f'f.{ident(c)}' for c in columns) +
|
||||
f' FROM {ident(source)}.{ident(table)} f CROSS JOIN scale_numbers n WHERE ' +
|
||||
selected.replace('FROM release_member', 'FROM ' + ident(source) + '.release_member').replace('JOIN current_release', 'JOIN ' + ident(source) + '.current_release') +
|
||||
f' ORDER BY n.n,f.id LIMIT {int(count)};', target)
|
||||
return int(sql(f'SELECT COUNT(*) FROM {ident(table)};', target).strip())
|
||||
|
||||
|
||||
class BenchApi:
|
||||
def __init__(self, port, config):
|
||||
self.base = f'http://127.0.0.1:{port}'
|
||||
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
||||
token = self.get('/api/auth/csrf')['token']
|
||||
request = Request(self.base + '/api/auth/login',
|
||||
data=json.dumps({'username': config['adminUsername'], 'password': config['adminPassword']}).encode(),
|
||||
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
|
||||
with self.opener.open(request, timeout=30) as response:
|
||||
response.read()
|
||||
|
||||
def get(self, path, params=None):
|
||||
request = Request(self.base + path + ('?' + urlencode(params) if params else ''),
|
||||
headers={'Accept': 'application/json'})
|
||||
with self.opener.open(request, timeout=180) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None
|
||||
|
||||
|
||||
def run(args):
|
||||
if not args.run:
|
||||
print('Plan only. Use --run to create an isolated synthetic database, benchmark, and remove it.')
|
||||
return 0
|
||||
if args.black < 1 or args.bright < 1 or args.black + args.bright > 2_000_000:
|
||||
raise ValueError('Synthetic scale must be between two rows and two million rows')
|
||||
if args.port == 5281 or not 1 <= args.concurrency <= 20:
|
||||
raise ValueError('Use a separate loopback port and at most twenty workers')
|
||||
with socket.socket() as probe:
|
||||
if probe.connect_ex(('127.0.0.1', args.port)) == 0:
|
||||
raise RuntimeError('The requested benchmark port is already in use')
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
config = project.config()
|
||||
tag = datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
||||
database = 'quality_scale_' + tag
|
||||
account = 'qscale_' + uuid.uuid4().hex[:12]
|
||||
password = secrets.token_urlsafe(32)
|
||||
report = {'status': 'RUNNING', 'kind': 'SYNTHETIC_SAMPLE_REPLICATION',
|
||||
'sourcePeriods': '15 provided black/bright periods retained; not a full real year',
|
||||
'requestedRows': {'black': args.black, 'bright': args.bright},
|
||||
'concurrency': args.concurrency, 'database': database,
|
||||
'liveDatabaseModified': False, 'cleanup': {'database': False, 'account': False, 'process': False}}
|
||||
result_path = OUT / 'scale-benchmark.json'
|
||||
process = None
|
||||
created_db = created_user = False
|
||||
started = time.monotonic()
|
||||
try:
|
||||
report['livePointerBefore'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
||||
sql(f'CREATE DATABASE {ident(database)} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||
created_db = True
|
||||
clone_database(config['mysqlDatabase'], database)
|
||||
report['rows'] = {stage: seed_facts(config['mysqlDatabase'], database, stage, count)
|
||||
for stage, count in (('black', args.black), ('bright', args.bright))}
|
||||
report['currentReleaseRows'] = {stage: int(sql("SELECT COUNT(*) FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id);", database).strip()) for stage in ('black', 'bright')}
|
||||
if report['currentReleaseRows'] != report['rows']:
|
||||
raise RuntimeError('Not all synthetic rows are visible in the benchmark release')
|
||||
existing_tables = set(sql('SHOW TABLES;', database).splitlines())
|
||||
# A copied 750-row materialization must never stand in for the 800k facts.
|
||||
for materialization in ('metric_rollup', 'metric_rollup_build'):
|
||||
if materialization in existing_tables:
|
||||
sql('DELETE FROM ' + ident(materialization) + ';', database)
|
||||
sql(f"CREATE USER '{account}'@'127.0.0.1' IDENTIFIED BY '{password}';")
|
||||
created_user = True
|
||||
test_resources.attach_user(ROOT, database, account)
|
||||
sql(f"GRANT ALL PRIVILEGES ON {ident(database)}.* TO '{account}'@'127.0.0.1';")
|
||||
jars = [p for p in (ROOT / 'backend/target').glob('*.jar') if not p.name.endswith('.original')]
|
||||
if not jars:
|
||||
raise RuntimeError('Build the backend before benchmarking')
|
||||
jar = max(jars, key=lambda p: p.stat().st_mtime)
|
||||
env = project.environment()
|
||||
env.update({'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{config["mysqlPort"]}/{database}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
|
||||
'MYSQL_USER': account, 'MYSQL_PASSWORD': password, 'PORT': str(args.port), 'SERVER_PORT': str(args.port),
|
||||
'QUALITY_GOVERNANCE_BACKGROUND_ENABLED': 'false', 'QUALITY_BACKUP_DATABASE': database})
|
||||
with (OUT / 'scale-backend.log').open('wb') as log:
|
||||
process = subprocess.Popen([str(project.RUNTIME / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx768m', '-jar', str(jar)],
|
||||
cwd=ROOT, env=env, stdout=log, stderr=log, start_new_session=True)
|
||||
test_resources.attach_child(ROOT, database, process.pid)
|
||||
preparing = time.monotonic()
|
||||
project.wait_http(f'http://127.0.0.1:{args.port}/actuator/health', seconds=90)
|
||||
if 'metric_rollup_build' in set(sql('SHOW TABLES;', database).splitlines()):
|
||||
deadline = time.monotonic() + 300
|
||||
while True:
|
||||
status = sql("SELECT COUNT(DISTINCT rm.transform_build_id),COUNT(DISTINCT rb.build_id),COALESCE(SUM(rb.source_row_count),0) FROM current_release cr JOIN release_member rm ON rm.release_id=cr.release_id LEFT JOIN metric_rollup_build rb ON rb.build_id=rm.transform_build_id WHERE cr.dataset_id='quality';", database).strip().split('\t')
|
||||
visible, rolled, rolled_rows = map(int, status)
|
||||
if visible == rolled and rolled_rows == args.black + args.bright:
|
||||
report['rollupPreparation'] = {'visibleBuilds':visible,'rolledBuilds':rolled,'sourceRows':rolled_rows,'seconds':round(time.monotonic()-preparing,4)}
|
||||
break
|
||||
if process.poll() is not None or time.monotonic() >= deadline:
|
||||
raise RuntimeError('Synthetic rollup preparation did not cover all query-visible facts')
|
||||
time.sleep(.25)
|
||||
clients = [BenchApi(args.port, config) for _ in range(args.concurrency)]
|
||||
common = {'year': 2026, 'releaseId': report['livePointerBefore'], 'includeUnassigned': 'true'}
|
||||
cases = [('/api/metrics', {**common, 'view': view}) for view in ('overall', 'department', 'group', 'product')]
|
||||
cases.extend(('/api/analytics/top', {**common, 'stage': stage, 'dimension': dimension, 'limit': 20})
|
||||
for stage in ('black', 'bright') for dimension in ('category', 'product'))
|
||||
def verify_visible_counts(result):
|
||||
if 'summary' in result:
|
||||
coverage = result.get('summary', {}).get('coverage', {})
|
||||
valid = coverage.get('blackRows') == args.black and coverage.get('brightRows') == args.bright
|
||||
else:
|
||||
months = result.get('coverage', {}).get('sourceMonths', [])
|
||||
valid = (sum(row.get('blackRows', 0) for row in months) == args.black
|
||||
and sum(row.get('brightRows', 0) for row in months) == args.bright
|
||||
and result.get('cumulativeThresholdApplied') is False and bool(result.get('resultId')))
|
||||
if not valid: raise RuntimeError('The benchmark did not aggregate all requested synthetic records')
|
||||
cold = []
|
||||
for path, params in cases:
|
||||
tick = time.monotonic()
|
||||
result = clients[0].get(path, params)
|
||||
if 'summary' not in result and 'resultId' not in result:
|
||||
raise RuntimeError('Benchmark endpoint did not return metric data')
|
||||
verify_visible_counts(result)
|
||||
cold.append({'view': params.get('view', 'TOP_' + params.get('stage','') + '_' + params.get('dimension','')),
|
||||
'seconds': round(time.monotonic() - tick, 4), 'rows': len(result.get('rows', []))})
|
||||
report['firstQueryMeasurements'] = cold
|
||||
# MySQL process/buffer state is shared with the local installation, so this
|
||||
# is a first API query, not a claim that the physical cache was cold.
|
||||
def worker(index):
|
||||
timings = []
|
||||
for n in range(args.rounds):
|
||||
path, params = cases[(index + n) % len(cases)]
|
||||
tick = time.monotonic()
|
||||
result = clients[index].get(path, params)
|
||||
if 'summary' not in result and 'resultId' not in result:
|
||||
raise RuntimeError('Concurrent query returned no summary')
|
||||
verify_visible_counts(result)
|
||||
timings.append(time.monotonic() - tick)
|
||||
return timings
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
timings = [item for group in pool.map(worker, range(args.concurrency)) for item in group]
|
||||
report['concurrentSummary'] = {'requests': len(timings), 'p50Seconds': round(percentile(timings, .5), 4),
|
||||
'p95Seconds': round(percentile(timings, .95), 4), 'maxSeconds': round(max(timings), 4),
|
||||
'initialTwoSecondTargetMet': percentile(timings, .95) <= 2}
|
||||
report['firstQueryTwoSecondTargetMet'] = all(item['seconds'] <= 2 for item in cold)
|
||||
report['status'] = 'PASS' if report['concurrentSummary']['initialTwoSecondTargetMet'] and report['firstQueryTwoSecondTargetMet'] else 'PERFORMANCE_TARGET_NOT_MET'
|
||||
report['limits'] = ['Synthetic replicas do not prove real annual data quality', 'No mixed AI/import concurrency in this run',
|
||||
'The host MySQL buffer pool was not flushed', 'Production environment and business acceptance remain separate']
|
||||
except Exception as failure:
|
||||
report['status'] = 'ERROR'
|
||||
report['errorType'] = type(failure).__name__
|
||||
# Database/HTTP clients can include credentials in exception details.
|
||||
report['error'] = 'Isolated benchmark failed; inspect the private local log without publishing credentials.'
|
||||
finally:
|
||||
if process is not None:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try: process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
report['cleanup']['process'] = True
|
||||
if created_db:
|
||||
sql(f'DROP DATABASE {ident(database)};')
|
||||
report['cleanup']['database'] = True
|
||||
if created_user:
|
||||
sql(f"DROP USER '{account}'@'127.0.0.1';")
|
||||
report['cleanup']['account'] = True
|
||||
report['livePointerAfter'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
||||
report['livePointerUnchanged'] = report.get('livePointerBefore') == report['livePointerAfter']
|
||||
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
|
||||
result_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||||
print(json.dumps({key: report.get(key) for key in ('status', 'rows', 'concurrentSummary', 'cleanup', 'livePointerUnchanged', 'elapsedSeconds')}, ensure_ascii=False))
|
||||
return 0 if report['status'] == 'PASS' and report['livePointerUnchanged'] else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--run', action='store_true')
|
||||
parser.add_argument('--black', type=int, default=500_000)
|
||||
parser.add_argument('--bright', type=int, default=300_000)
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--rounds', type=int, default=3)
|
||||
parser.add_argument('--port', type=int, default=5283)
|
||||
sys.exit(run(parser.parse_args()))
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Private, disposable local API runtime for integration acceptance tests."""
|
||||
from contextlib import contextmanager
|
||||
import http.cookiejar
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, build_opener, HTTPCookieProcessor
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import project
|
||||
import backup
|
||||
import managed_test_resources as test_resources
|
||||
sys.path.insert(0, str(ROOT / 'verification'))
|
||||
from benchmark_scale import clone_database, ident, sql
|
||||
|
||||
|
||||
class LocalApi:
|
||||
def __init__(self, port, username=None, password=None):
|
||||
if port not in (5281, 5283, 5284, 5285, 5287):
|
||||
raise ValueError('Only fixed project loopback ports are allowed')
|
||||
self.base = f'http://127.0.0.1:{port}'
|
||||
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
||||
self.csrf = None
|
||||
if username is not None:
|
||||
status, response, _ = self.call('/api/auth/csrf')
|
||||
if status != 200: raise RuntimeError('CSRF initialization failed')
|
||||
self.csrf = response['token']
|
||||
status, _, _ = self.call('/api/auth/login', {'username': username, 'password': password})
|
||||
if status != 200: raise RuntimeError('Local test authentication failed')
|
||||
_, response, _ = self.call('/api/auth/csrf')
|
||||
self.csrf = response['token']
|
||||
|
||||
def call(self, path, payload=None, method=None, binary=False):
|
||||
if not path.startswith('/api/') or '://' in path:
|
||||
raise ValueError('Only local API paths are allowed')
|
||||
headers = {'Accept': 'application/json'}
|
||||
if self.csrf: headers['X-XSRF-TOKEN'] = self.csrf
|
||||
if payload is not None: headers['Content-Type'] = 'application/json'
|
||||
request = Request(self.base + path, method=method or ('POST' if payload is not None else 'GET'), headers=headers,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None)
|
||||
try:
|
||||
with self.opener.open(request, timeout=180) as response:
|
||||
data = response.read()
|
||||
return response.status, data if binary else json.loads(data) if data else {}, dict(response.headers)
|
||||
except HTTPError as failure:
|
||||
raw = failure.read()
|
||||
try: body = json.loads(raw)
|
||||
except (ValueError, UnicodeDecodeError): body = {'detail': 'Non-JSON error response'}
|
||||
return failure.code, body, dict(failure.headers)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def isolated_api(port=5284, fresh=False, evidence_dir=None, current_release_only=False, product_scope=None,
|
||||
env_overrides=None, auth_credentials=None):
|
||||
with socket.socket() as probe:
|
||||
if probe.connect_ex(('127.0.0.1', port)) == 0:
|
||||
raise RuntimeError('The integration port is already in use')
|
||||
if product_scope is not None and (not isinstance(product_scope,str) or not product_scope.strip() or len(product_scope)>256):
|
||||
raise ValueError('Invalid isolated product scope')
|
||||
allowed_overrides = {'QUALITY_NAO_MODEL_CONFIG_FILE', 'QUALITY_MODEL_CONFIG', 'QUALITY_ARCHIVE_ROOT',
|
||||
'QUALITY_ANALYSIS_MODEL_CONFIG_FILE', 'QUALITY_ANALYSIS_RUNTIME_SCRIPT',
|
||||
'QUALITY_ANALYSIS_STATE_DIR',
|
||||
'SERVER_SERVLET_SESSION_COOKIE_NAME',
|
||||
'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED', 'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'}
|
||||
if env_overrides is not None and (not isinstance(env_overrides, dict) or not set(env_overrides).issubset(allowed_overrides)
|
||||
or any(not isinstance(key, str) or not isinstance(value, str) for key, value in env_overrides.items())):
|
||||
raise ValueError('Invalid isolated runtime environment override')
|
||||
if auth_credentials is not None and (not isinstance(auth_credentials, tuple) or len(auth_credentials) != 2
|
||||
or any(not isinstance(value, str) or not value for value in auth_credentials)):
|
||||
raise ValueError('Invalid isolated test login credentials')
|
||||
config = project.config()
|
||||
database = 'quality_verify_' + uuid.uuid4().hex[:16]
|
||||
account = 'qverify_' + uuid.uuid4().hex[:12]
|
||||
password = secrets.token_urlsafe(32)
|
||||
process = None
|
||||
db_created = user_created = False
|
||||
evidence = {'database': database, 'port': port, 'freshInstallation': fresh,
|
||||
'factScope': ('currentReleaseProduct' if current_release_only else 'product') if product_scope is not None else 'currentRelease' if current_release_only else 'all', 'cleanup': {}}
|
||||
out = Path(evidence_dir) if evidence_dir is not None else ROOT / 'output/completion'
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
if evidence_dir is not None:
|
||||
out.chmod(0o700)
|
||||
try:
|
||||
evidence['livePointerBefore'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
||||
sql(f'CREATE DATABASE {ident(database)} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', empty=fresh)
|
||||
db_created = True
|
||||
if not fresh:
|
||||
clone_database(config['mysqlDatabase'], database)
|
||||
for table in ('fact_black', 'fact_bright'):
|
||||
predicate = (f" WHERE EXISTS (SELECT 1 FROM {ident(config['mysqlDatabase'])}.release_member rm "
|
||||
f"JOIN {ident(config['mysqlDatabase'])}.current_release cr ON cr.release_id=rm.release_id "
|
||||
"AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id)"
|
||||
if current_release_only else '')
|
||||
if product_scope is not None:
|
||||
# Hex encodes a text literal without interpolating caller-controlled SQL.
|
||||
predicate += (' AND ' if predicate else ' WHERE ') + f"f.product COLLATE utf8mb4_0900_as_cs=CONVERT(0x{product_scope.encode('utf-8').hex()} USING utf8mb4) COLLATE utf8mb4_0900_as_cs"
|
||||
sql(f'INSERT INTO {ident(database)}.{ident(table)} SELECT f.* FROM {ident(config["mysqlDatabase"])}.{ident(table)} f{predicate};')
|
||||
sql(f"CREATE USER '{account}'@'127.0.0.1' IDENTIFIED BY '{password}';")
|
||||
user_created = True
|
||||
test_resources.attach_user(ROOT, database, account)
|
||||
sql(f"GRANT ALL PRIVILEGES ON {ident(database)}.* TO '{account}'@'127.0.0.1';")
|
||||
jar = max((ROOT / 'backend/target').glob('*.jar'), key=lambda p: p.stat().st_mtime)
|
||||
jar = project.backend_snapshot(jar)
|
||||
evidence['jarSha256'] = jar.stem
|
||||
env = project.environment()
|
||||
env.update({'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{config["mysqlPort"]}/{database}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
|
||||
'MYSQL_USER': account, 'MYSQL_PASSWORD': password, 'PORT': str(port), 'SERVER_PORT': str(port),
|
||||
'QUALITY_BACKUP_DATABASE': database,
|
||||
'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(port)})
|
||||
env.update(env_overrides or {})
|
||||
login_user, login_password = auth_credentials or (config['adminUsername'], config['adminPassword'])
|
||||
if auth_credentials is not None:
|
||||
env.update({'QUALITY_BOOTSTRAP_ADMIN_USERNAME': login_user,
|
||||
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': login_password})
|
||||
with (out / ('fresh-install-backend.log' if fresh else 'acceptance-backend.log')).open('wb') as log:
|
||||
os.chmod(out / ('fresh-install-backend.log' if fresh else 'acceptance-backend.log'), 0o600)
|
||||
process = subprocess.Popen([str(project.RUNTIME / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx768m', '-jar', str(jar)],
|
||||
cwd=ROOT, env=env, stdout=log, stderr=log, start_new_session=True)
|
||||
test_resources.attach_child(ROOT, database, process.pid)
|
||||
project.wait_http(f'http://127.0.0.1:{port}/actuator/health', seconds=90)
|
||||
if fresh: test_resources.seal(ROOT, database, backup.client)
|
||||
api = LocalApi(port, login_user, login_password)
|
||||
evidence['flywayVersions'] = sql('SELECT version FROM flyway_schema_history WHERE success=1 ORDER BY installed_rank;', database).splitlines()
|
||||
yield api, database, evidence
|
||||
finally:
|
||||
if process is not None:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try: process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
evidence['cleanup']['process'] = True
|
||||
if db_created:
|
||||
sql(f'DROP DATABASE {ident(database)};')
|
||||
evidence['cleanup']['database'] = True
|
||||
if user_created:
|
||||
sql(f"DROP USER '{account}'@'127.0.0.1';")
|
||||
evidence['cleanup']['account'] = True
|
||||
evidence['livePointerAfter'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
||||
evidence['livePointerUnchanged'] = evidence.get('livePointerBefore') == evidence['livePointerAfter']
|
||||
evidence_path = out / ('fresh-install-isolation.json' if fresh else 'acceptance-isolation.json')
|
||||
evidence_path.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
|
||||
evidence_path.chmod(0o600)
|
||||
@@ -209,7 +209,7 @@ class MixedHistoryReportTest(unittest.TestCase):
|
||||
]
|
||||
report = build_agent_report([comparison, *investigations], context, question)
|
||||
rendered = '\n'.join(report['sections'])
|
||||
self.assertIn('原表值全部匹配且核验完整', rendered)
|
||||
self.assertIn('所选原表值与已存储原始值全部匹配且核验完整', rendered)
|
||||
self.assertIn('已核验205行/应核验205行', rendered)
|
||||
self.assertIn('共12条', rendered)
|
||||
self.assertIn('差异记录', rendered)
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Engine pinning, provider tool boundaries and shared formal completion."""
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT/'scripts'), str(ROOT/'verification')]
|
||||
from quality_analysis_runtime import engine_for_request, run_turn
|
||||
from nao_agent_contract import apply_platform_tools
|
||||
import test_native_run as native_fixture
|
||||
|
||||
|
||||
class EngineSelectionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.settings = Path(self.directory.name)/'model.json'
|
||||
|
||||
def test_old_config_defaults_to_nao_and_new_config_can_select_dsh(self):
|
||||
self.assertEqual(engine_for_request({}, settings_file=self.settings), 'NAO')
|
||||
self.settings.write_text(json.dumps({'runtimeEngine':'DSH'}))
|
||||
self.assertEqual(engine_for_request({}, settings_file=self.settings), 'DSH')
|
||||
|
||||
def test_queued_engine_survives_later_config_change(self):
|
||||
self.settings.write_text(json.dumps({'runtimeEngine':'NAO'}))
|
||||
self.assertEqual(engine_for_request({'runtimeEngine':'DSH'}, settings_file=self.settings), 'DSH')
|
||||
self.assertEqual(engine_for_request({'context':{'_runtimeEngine':'DSH'}}, settings_file=self.settings), 'DSH')
|
||||
|
||||
def test_conflicting_or_arbitrary_engine_is_not_an_executable(self):
|
||||
for request in ({'runtimeEngine':'DSH','context':{'_runtimeEngine':'NAO'}},
|
||||
{'runtimeEngine':'dsh'}, {'runtimeEngine':'/bin/sh'},
|
||||
{'runtimeEngine':True}):
|
||||
with self.subTest(request=request), self.assertRaises(ValueError):
|
||||
engine_for_request(request, settings_file=self.settings)
|
||||
|
||||
def test_symlinked_settings_are_rejected(self):
|
||||
target = self.settings.with_name('target.json'); target.write_text('{}')
|
||||
self.settings.symlink_to(target)
|
||||
with self.assertRaises(ValueError):engine_for_request({}, settings_file=self.settings)
|
||||
|
||||
def test_route_keeps_platform_protocol_and_never_silently_falls_back(self):
|
||||
request = {'runtimeEngine':'DSH','question':'fixture'}
|
||||
with patch('nao_quality_runtime.run_turn', side_effect=RuntimeError('DSH unavailable')) as platform:
|
||||
with self.assertRaises(RuntimeError):run_turn(request, stdout=io.StringIO(), stdin=io.StringIO())
|
||||
self.assertEqual(platform.call_count,1)
|
||||
self.assertEqual(platform.call_args.kwargs['runtime_engine'],'DSH')
|
||||
|
||||
def test_dsh_provider_cannot_see_shell_fs_story_or_nao_tools(self):
|
||||
names=['mcp__quality__quality_data','bash','read','mcp_call','story',
|
||||
'mcp__other__quality_data','mcp__quality__write','working_directory']
|
||||
payload={'tools':[{'type':'function','function':{'name':name}} for name in names],
|
||||
'tool_choice':{'type':'function','function':{'name':'bash'}}}
|
||||
value=apply_platform_tools(payload, native_agent=True, runtime_engine='DSH')
|
||||
self.assertEqual([x['function']['name'] for x in value['tools']],['mcp__quality__quality_data'])
|
||||
self.assertNotIn('tool_choice',value)
|
||||
|
||||
|
||||
class DshFormalCompletionTests(unittest.TestCase):
|
||||
# Reuse the established signed-evidence fixture, not the NAO implementation.
|
||||
setUp = native_fixture.NativeRunTests.setUp
|
||||
invoke = native_fixture.NativeRunTests.invoke
|
||||
native = native_fixture.NativeRunTests.native
|
||||
|
||||
def configure_dsh(self):
|
||||
self.request['runtimeEngine']='DSH'
|
||||
self.dsh=Mock(side_effect=self.native)
|
||||
module=types.ModuleType('dsh_agent_runtime'); module.run_dsh_agent=self.dsh
|
||||
self.enterContext(patch.dict(sys.modules,{'dsh_agent_runtime':module}))
|
||||
self.synthesis.side_effect=None
|
||||
self.synthesis.return_value={'answer':'正式证据支持的答复。','evidenceSufficient':True,
|
||||
'finishReason':'stop','missingEvidence':[],'titleRequired':False}
|
||||
self.exploration=self.enterContext(patch('nao_native_exploration.NativeExplorationTools'))
|
||||
|
||||
def test_dsh_never_starts_nao_and_keeps_java_proof_and_visual_completion(self):
|
||||
self.configure_dsh(); result=self.invoke()
|
||||
self.manager.ensure_instance.assert_not_called(); self.auth.assert_not_called()
|
||||
self.sse.assert_not_called(); self.dsh.assert_called_once()
|
||||
self.assertEqual(result['engine'],'DSH')
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'verified_composition')
|
||||
self.assertEqual(result['conclusionProtocolQueryIds'],['query-current','query-compare','query-group'])
|
||||
self.assertEqual(result['answer'],'正式证据支持的答复。')
|
||||
self.assertEqual(self.exploration.call_args.kwargs['identity']['releaseId'],self.request['releaseId'])
|
||||
|
||||
def test_loop_stop_does_not_publish_a_rejected_narrative_as_completed(self):
|
||||
self.configure_dsh()
|
||||
self.synthesis.return_value={'answer':'保留已核实数据,部分结论待核查。',
|
||||
'evidenceSufficient':False,'finishReason':'stop','missingEvidence':['正文与明细不一致']}
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
||||
self.assertIn('正文与明细不一致',result['answerQuality']['missingEvidence'])
|
||||
|
||||
def test_cancel_during_loop_has_no_composition_or_late_public_evidence(self):
|
||||
self.configure_dsh(); self.cancel_during_native=True
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'CANCELLED')
|
||||
self.synthesis.assert_not_called()
|
||||
self.assertEqual(result['answer'],''); self.assertEqual(result['queryEvidenceRefs'],[])
|
||||
|
||||
def test_synthesis_failure_preserves_partial_state_without_publishing_draft(self):
|
||||
from nao_evidence_finalizer import FinalizationUnavailable
|
||||
self.configure_dsh(); self.synthesis.side_effect=FinalizationUnavailable('fixture')
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'rejected')
|
||||
self.assertNotEqual(result['answer'],self.accumulator.answer)
|
||||
|
||||
def test_no_signed_evidence_cannot_publish_an_unchecked_dsh_draft(self):
|
||||
self.configure_dsh(); self.documents=[]
|
||||
self.accumulator.answer='TOP5全部属于缺角,合格率99%。'
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertEqual(result['answerQuality']['narrativeFactBinding'],'rejected')
|
||||
self.assertNotIn('99%',result['answer'])
|
||||
self.assertNotIn('全部属于缺角',result['answer'])
|
||||
self.synthesis.assert_not_called()
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -400,8 +400,8 @@ class AnalysisVisualsTest(unittest.TestCase):
|
||||
chart = build_visuals([evidence(result)], "哪些产品报废最多", "turn-top") ["visuals"][0]
|
||||
self.assertEqual(chart["metric"], "current_scrap_weight")
|
||||
self.assertEqual(chart["requiredFields"], ["black_scrap_weight"])
|
||||
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(10)])
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
|
||||
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(12)])
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 12, "truncated": False})
|
||||
self.assertNotIn("share", json.dumps(chart, ensure_ascii=False))
|
||||
|
||||
def test_quality_top_categories_require_stage_canonical_source_fields(self):
|
||||
@@ -642,7 +642,7 @@ class AnalysisVisualsTest(unittest.TestCase):
|
||||
paired_totals["pairedRankingComplete"] = True
|
||||
self.assertEqual(build_visuals([evidence(paired_totals)], "画图比较亮片来源具体缺陷增量", "turn-mixed")["visualState"], "unavailable")
|
||||
|
||||
def test_fine_defect_chart_limits_to_ten_and_authorizes_only_displayed_fields(self):
|
||||
def test_fine_defect_chart_keeps_requested_rows_and_authorizes_only_displayed_fields(self):
|
||||
keys = ["责废尺寸重量", "责废缺角重量", "责废垂直度重量", "责废刀丝重量", "责废烂毛重量",
|
||||
"责废开裂重量", "责废其他重量", "料废尺寸重量", "料废缺角重量", "料废开裂重量",
|
||||
"料废烂毛重量", "料废其他重量"]
|
||||
@@ -656,8 +656,8 @@ class AnalysisVisualsTest(unittest.TestCase):
|
||||
"totalRows": len(keys), "truncated": False, "rankingComplete": True,
|
||||
}
|
||||
chart = build_visuals([evidence(result)], "画图看黑片来源具体缺陷排名", "turn-ten")["visuals"][0]
|
||||
self.assertEqual(len(chart["categories"]), 10)
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
|
||||
self.assertEqual(len(chart["categories"]), 12)
|
||||
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 12, "truncated": False})
|
||||
self.assertEqual(chart["requiredFields"], [item["key"] for item in chart["categories"]])
|
||||
|
||||
def test_multi_query_trend_requires_same_release_and_filters_and_keeps_gaps_null(self):
|
||||
@@ -910,9 +910,9 @@ class AnalysisVisualsTest(unittest.TestCase):
|
||||
for index in range(15)
|
||||
], totalRows=15, returnedRows=15, truncated=False)
|
||||
chart = build_visuals([evidence(result)], "画产品报废增量排名", "turn-13")["visuals"][0]
|
||||
self.assertEqual(len(chart["categories"]), 10)
|
||||
self.assertTrue(chart["coverage"]["truncated"])
|
||||
self.assertEqual(chart["series"][0]["values"], list(map(float, range(10))))
|
||||
self.assertEqual(len(chart["categories"]), 15)
|
||||
self.assertFalse(chart["coverage"]["truncated"])
|
||||
self.assertEqual(chart["series"][0]["values"], list(map(float, range(15))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Pinned-runtime tests use authored local SSE, never a model or business credential."""
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import dsh_agent_runtime as runtime
|
||||
from nao_quality_runtime import ConversationAccumulator
|
||||
from setup_dsh_runtime import inspect_runtime, runtime_node
|
||||
|
||||
|
||||
class AuthoredModel:
|
||||
"""A deterministic provider wire fixture which exercises the actual DSH loop."""
|
||||
def __init__(self, *, stall=False, length=False, transient=False, malformed=False):
|
||||
self.calls, self.stall, self.length = [], stall, length
|
||||
self.started = threading.Event()
|
||||
self.release = threading.Event()
|
||||
owner = self
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
owner.calls.append(body)
|
||||
owner.started.set()
|
||||
if transient and len(owner.calls) == 1:
|
||||
encoded = b'{"error":{"message":"authored temporary failure"}}'
|
||||
self.send_response(503)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'text/event-stream')
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
|
||||
def send(delta, finish=None):
|
||||
value = {'id': 'authored-local-only', 'object': 'chat.completion.chunk', 'created': 0,
|
||||
'model': 'authored-check', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': finish}]}
|
||||
self.wfile.write(('data: ' + json.dumps(value, ensure_ascii=False) + '\n\n').encode())
|
||||
self.wfile.flush()
|
||||
|
||||
try:
|
||||
send({'role': 'assistant'})
|
||||
if owner.stall:
|
||||
owner.release.wait(20)
|
||||
return
|
||||
if not any(message.get('role') == 'tool' for message in body['messages']):
|
||||
send({'tool_calls': [{'index': 0, 'id': 'authored-quality-query', 'type': 'function',
|
||||
'function': {'name': runtime.PUBLIC_TOOL, 'arguments':
|
||||
'{"mode":' if malformed and len(owner.calls) == 1 else '{"mode":"catalog"}'}}]})
|
||||
send({}, 'tool_calls')
|
||||
else:
|
||||
send({'content': '已核实'})
|
||||
time.sleep(0.4) # SDK notification polling must survive ordinary idle intervals.
|
||||
send({'content': '质量指标。'})
|
||||
send({}, 'length' if owner.length else 'stop')
|
||||
usage = {'id': 'usage', 'choices': [], 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'total_tokens': 110}}
|
||||
self.wfile.write(('data: ' + json.dumps(usage) + '\n\ndata: [DONE]\n\n').encode())
|
||||
self.wfile.flush()
|
||||
except (OSError, BrokenPipeError):
|
||||
pass
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
self.server = Server(('127.0.0.1', 0), Handler)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
self.connection = (f'http://127.0.0.1:{self.server.server_port}/v1', 'authored-test-only', 'authored-check')
|
||||
|
||||
def close(self):
|
||||
self.release.set()
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=1)
|
||||
|
||||
|
||||
class ProjectionTests(unittest.TestCase):
|
||||
def test_failed_attempt_is_removed_before_retried_commit(self):
|
||||
events = []
|
||||
acc = ConversationAccumulator()
|
||||
projection = runtime._Projection(acc, events.append)
|
||||
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 1, 'turn': 1, 'step': 1})
|
||||
projection.frame({'type': 'chunk', 'attemptId': 'a', 'revision': 2, 'index': 0,
|
||||
'chunk': {'type': 'text-delta', 'index': 0, 'text': '失败草稿'}})
|
||||
self.assertEqual(acc.answer, '失败草稿')
|
||||
projection.event({'seq': 0, 'type': 'assistant/attempt', 'data': {'turn': 1, 'step': 1}})
|
||||
projection.frame({'type': 'end', 'attemptId': 'a', 'revision': 3, 'index': 1,
|
||||
'outcome': {'kind': 'committed', 'eventType': 'assistant/attempt', 'seq': 0}})
|
||||
self.assertEqual(acc.answer, '')
|
||||
self.assertFalse(any(entry.get('text') == '失败草稿' for entry in acc.timeline))
|
||||
projection.frame({'type': 'start', 'attemptId': 'b', 'revision': 4, 'turn': 1, 'step': 1})
|
||||
projection.frame({'type': 'chunk', 'attemptId': 'b', 'revision': 5, 'index': 0,
|
||||
'chunk': {'type': 'text-delta', 'index': 0, 'text': '新答复'}})
|
||||
projection.event({'seq': 1, 'type': 'assistant/message', 'data': {'turn': 1, 'step': 1,
|
||||
'message': {'content': [{'type': 'text', 'text': '已提交答复'}]}}})
|
||||
self.assertEqual(acc.answer, '已提交答复')
|
||||
self.assertNotIn('失败草稿', json.dumps(acc.timeline, ensure_ascii=False))
|
||||
self.assertNotIn('失败草稿', json.dumps(acc.timeline_segments, ensure_ascii=False))
|
||||
|
||||
def test_unknown_tool_and_stream_gap_fail_closed(self):
|
||||
projection = runtime._Projection(ConversationAccumulator(), lambda _value: None)
|
||||
with self.assertRaises(ValueError):
|
||||
projection.frame({'type': 'start', 'attemptId': 'a', 'revision': 2, 'turn': 1, 'step': 1})
|
||||
with self.assertRaises(ValueError):
|
||||
projection.event({'seq': 0, 'type': 'tool/call', 'data': {'name': 'shell'}})
|
||||
|
||||
def test_identity_and_versions_are_pinned(self):
|
||||
lock = json.loads((ROOT / 'runtime/dsh/vendor/python-sdk/source-lock.json').read_text())
|
||||
self.assertEqual(lock['commit'], 'fcc0ad462172522aaf5c93a7b2e01f6b9f0b0159')
|
||||
manifest = json.loads((ROOT / 'runtime/dsh/package.json').read_text())
|
||||
self.assertEqual(manifest['dependencies']['@deepseek-ai/dsh'], '0.2.2-rc.1')
|
||||
|
||||
|
||||
@unittest.skipUnless(inspect_runtime()['ready'], 'requires installed pinned runtime and macOS sandbox')
|
||||
class RealDshTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.folder = tempfile.TemporaryDirectory(prefix='dsh-real-verification-')
|
||||
self.addCleanup(self.folder.cleanup)
|
||||
self.root = Path(self.folder.name).resolve()
|
||||
self.events = []
|
||||
self.tool_calls = []
|
||||
self.authorizations = 0
|
||||
self.cancel = threading.Event()
|
||||
|
||||
def run_agent(self, model, *, tool=None, deadline=None, authorization=None, initial=None):
|
||||
self.addCleanup(model.close)
|
||||
from nao_quality_tools import tool_definition
|
||||
def authorize():
|
||||
self.authorizations += 1
|
||||
if authorization is not None:
|
||||
authorization()
|
||||
def query(arguments):
|
||||
self.tool_calls.append(arguments)
|
||||
return {'kind': 'quality_catalog', 'evidenceRef': 'QA1B2C3D4', 'message': '正式证据已核实'}
|
||||
result = runtime.run_dsh_agent(actor_root=self.root, connection=model.connection,
|
||||
question='/ordinary-user-input', context={'_agentTask': {}}, task={}, identity={'executionId': 'test-current'},
|
||||
turn_id=str(uuid.uuid4()), cancel_event=self.cancel, emit=self.events.append,
|
||||
tool_handler=tool or query, authorize=authorize, tool_definition=tool_definition(),
|
||||
deadline=deadline or time.monotonic() + 40, initial_accumulator=initial)
|
||||
self.summary = json.loads(next((self.root / 'private/dsh').glob('*/summary.json')).read_text())
|
||||
return result
|
||||
|
||||
def test_real_loop_mcp_stream_commit_and_isolation_lifecycle(self):
|
||||
model = AuthoredModel()
|
||||
acc, status, reason, stopped = self.run_agent(model)
|
||||
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
||||
self.assertEqual(acc.answer, '已核实质量指标。')
|
||||
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
|
||||
self.assertGreaterEqual(self.authorizations, 5)
|
||||
self.assertTrue(any(e.get('type') == 'answer' and e.get('answer') == '已核实' for e in self.events))
|
||||
self.assertEqual(acc.tool_calls[0]['tool'], 'mcp_call')
|
||||
self.assertEqual(acc.tool_calls[0]['sourceTool'], runtime.PUBLIC_TOOL)
|
||||
self.assertTrue(acc.final_answer_ready)
|
||||
self.assertEqual(len(model.calls), 2)
|
||||
for request in model.calls:
|
||||
self.assertEqual({t['function']['name'] for t in request.get('tools', [])}, {runtime.PUBLIC_TOOL})
|
||||
self.assertNotIn('dsh_session_log', request)
|
||||
self.assertEqual(self.summary['stopSucceeded'], True)
|
||||
|
||||
def test_initial_read_remains_in_the_real_native_event_stream(self):
|
||||
seed=ConversationAccumulator()
|
||||
seed.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'initial-fixture',
|
||||
'input':{'tool':'quality_data','arguments':{'mode':'query'}}})
|
||||
seed.feed({'type':'tool-output-available','toolName':'mcp_call','toolCallId':'initial-fixture',
|
||||
'output':{'kind':'quality_metrics','summary':{'compositeRate':.9557}}})
|
||||
acc,status,reason,stopped=self.run_agent(AuthoredModel(),initial=seed)
|
||||
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True),self.summary)
|
||||
self.assertIs(acc,seed)
|
||||
self.assertEqual(acc.answer,'已核实质量指标。')
|
||||
self.assertEqual([call['input']['mode'] for call in acc.tool_calls],['query','catalog'])
|
||||
self.assertTrue(all(call['status']=='completed' for call in acc.tool_calls))
|
||||
self.assertEqual(len({event['sequence'] for event in acc.timeline}),len(acc.timeline))
|
||||
|
||||
def test_real_cancel_revokes_capability_and_reaps_process_tree(self):
|
||||
model = AuthoredModel(stall=True)
|
||||
def cancel_after_start():
|
||||
model.started.wait(15)
|
||||
self.cancel.set()
|
||||
thread = threading.Thread(target=cancel_after_start, daemon=True)
|
||||
thread.start()
|
||||
began = time.monotonic()
|
||||
acc, status, reason, stopped = self.run_agent(model)
|
||||
self.assertEqual((status, reason, stopped), ('CANCELLED', 'user_cancel', True), self.summary)
|
||||
self.assertLess(time.monotonic() - began, 15)
|
||||
self.assertEqual(self.summary['toolCalls'], 0)
|
||||
|
||||
def test_real_output_limit_cannot_be_success(self):
|
||||
acc, status, reason, stopped = self.run_agent(AuthoredModel(length=True))
|
||||
self.assertIn(status, ('PARTIAL', 'FAILED'))
|
||||
self.assertNotEqual(reason, 'stop')
|
||||
self.assertTrue(stopped)
|
||||
|
||||
def test_real_transient_failure_retries_once_without_duplicate_text(self):
|
||||
model = AuthoredModel(transient=True)
|
||||
acc, status, reason, stopped = self.run_agent(model)
|
||||
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
||||
self.assertEqual(len(model.calls), 3)
|
||||
self.assertEqual(acc.answer, '已核实质量指标。')
|
||||
self.assertGreater(self.summary['modelTokensCharged'], 220)
|
||||
self.assertEqual(self.summary['modelRequests'], 3)
|
||||
|
||||
def test_malformed_tool_response_gets_one_native_retry_without_execution_or_rewriting(self):
|
||||
model = AuthoredModel(malformed=True)
|
||||
acc, status, reason, stopped = self.run_agent(model)
|
||||
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
||||
self.assertEqual(len(model.calls), 3)
|
||||
self.assertEqual(self.tool_calls, [{'mode': 'catalog'}])
|
||||
self.assertEqual(acc.answer, '已核实质量指标。')
|
||||
|
||||
def test_real_deadline_stops_a_silent_model_independent_of_sdk_wait(self):
|
||||
began = time.monotonic()
|
||||
acc, status, reason, stopped = self.run_agent(AuthoredModel(stall=True), deadline=began + 3)
|
||||
self.assertEqual((status, reason, stopped), ('FAILED', 'timeout', True), self.summary)
|
||||
self.assertLess(time.monotonic() - began, 13)
|
||||
|
||||
def test_authorization_watchdog_stops_and_records_only_safe_failure_metadata(self):
|
||||
model = AuthoredModel(stall=True)
|
||||
def authorize():
|
||||
if model.started.is_set():
|
||||
raise RuntimeError('private-authorization-detail')
|
||||
acc, status, reason, stopped = self.run_agent(model, authorization=authorize)
|
||||
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_unavailable', True))
|
||||
self.assertEqual(len(model.calls), 1)
|
||||
self.assertEqual(self.summary['authorizationFailure'], {
|
||||
'errorType': 'RuntimeError', 'httpStatus': None, 'retryable': False})
|
||||
self.assertNotIn('private-authorization-detail', json.dumps(self.summary))
|
||||
|
||||
def test_authorization_rejection_stops_without_claiming_a_provider_failure(self):
|
||||
from nao_quality_runtime import RuntimeErrorSafe
|
||||
model = AuthoredModel(stall=True)
|
||||
def authorize():
|
||||
if model.started.is_set():
|
||||
raise RuntimeErrorSafe('private-refusal-detail', http_status=403)
|
||||
_, status, reason, stopped = self.run_agent(model, authorization=authorize)
|
||||
self.assertEqual((status, reason, stopped), ('FAILED', 'authorization_rejected', True))
|
||||
self.assertEqual(self.summary['authorizationFailure']['httpStatus'], 403)
|
||||
self.assertNotIn('private-refusal-detail', json.dumps(self.summary))
|
||||
|
||||
def test_handled_tool_error_does_not_overturn_native_completion(self):
|
||||
from nao_quality_tools import QualityToolError
|
||||
def reject(_arguments):
|
||||
raise QualityToolError('当前范围未授权。')
|
||||
acc, status, reason, stopped = self.run_agent(AuthoredModel(), tool=reject)
|
||||
self.assertEqual((status, reason, stopped), ('COMPLETED', 'stop', True), self.summary)
|
||||
self.assertTrue(acc.tool_error)
|
||||
self.assertEqual(acc.tool_calls[0]['status'], 'failed')
|
||||
self.assertIn('当前范围未授权', acc.tool_calls[0]['outputSummary'])
|
||||
|
||||
def test_no_data_is_a_normal_tool_outcome_without_zero_metrics(self):
|
||||
acc,status,reason,stopped=self.run_agent(AuthoredModel(),tool=lambda args:{
|
||||
'kind':'quality_no_data','status':'no_data','summary':None,'rows':[],
|
||||
'message':'当前期间没有已发布来源数据。'})
|
||||
self.assertEqual((status,reason,stopped),('COMPLETED','stop',True))
|
||||
self.assertFalse(acc.tool_error)
|
||||
self.assertEqual(acc.tool_calls[0]['status'],'completed')
|
||||
self.assertIn('暂无数据',acc.tool_calls[0]['outputSummary'])
|
||||
|
||||
def test_process_cleanup_refuses_a_changed_process_identity(self):
|
||||
marker = uuid.uuid4().hex
|
||||
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
|
||||
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
path = self.root / 'process.json'
|
||||
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker, 'fingerprint': 'wrong'}))
|
||||
self.assertFalse(runtime._stop_registered(path))
|
||||
self.assertIsNone(process.poll())
|
||||
finally:
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
|
||||
def test_stop_all_reaps_only_a_matching_persisted_orphan_group(self):
|
||||
marker = uuid.uuid4().hex
|
||||
process = subprocess.Popen([str(runtime_node()), '-e', 'setInterval(()=>{},1000)', marker],
|
||||
start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
root = self.root / 'users'
|
||||
path = root / 'actor/sessions/session/private/dsh' / marker / 'process.json'
|
||||
path.parent.mkdir(parents=True)
|
||||
fingerprint = runtime._process_fingerprint(process.pid)
|
||||
self.assertIn(marker, fingerprint)
|
||||
path.write_text(json.dumps({'pid': process.pid, 'pgid': process.pid, 'marker': marker,
|
||||
'fingerprint': hashlib.sha256(fingerprint.encode()).hexdigest()}))
|
||||
result = runtime.stop_all_dsh(users_root=root)
|
||||
self.assertEqual(result, {'stopped': 1, 'unconfirmed': 0})
|
||||
process.wait(timeout=3)
|
||||
self.assertEqual(json.loads(path.read_text())['state'], 'reaped')
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
|
||||
def test_os_sandbox_denies_business_files_other_ports_and_shell(self):
|
||||
# Probe the actual generated OS policy, independent of model tool hiding.
|
||||
secret = self.root / 'business-secret.txt'
|
||||
secret.write_text('verification sentinel')
|
||||
run = self.root / 'isolated'
|
||||
run.mkdir()
|
||||
for child in ('home', 'workspace', 'tmp'):
|
||||
(run / child).mkdir()
|
||||
policy = runtime._sandbox(run, 9)
|
||||
code = '''
|
||||
const fs=require('node:fs'); const cp=require('node:child_process');
|
||||
let read=false,write=false;try{fs.readFileSync(process.argv[1]);read=true}catch{}
|
||||
try{fs.writeFileSync(process.argv[1], 'bad');write=true}catch{}
|
||||
const spawned=cp.spawnSync('/bin/sh',['-c','exit 0']);
|
||||
let network=false;try{await fetch('http://127.0.0.1:10',{signal:AbortSignal.timeout(1000)});network=true}catch{}
|
||||
console.log(JSON.stringify({read,write,shell:!spawned.error,network}));
|
||||
'''
|
||||
result = subprocess.run(['/usr/bin/sandbox-exec', '-f', str(policy), str(runtime_node()),
|
||||
'--input-type=commonjs', '-e', '(async()=>{' + code + '})()', str(secret)],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(result.stdout), {'read': False, 'write': False, 'shell': False, 'network': False})
|
||||
self.assertEqual(secret.read_text(), 'verification sentinel')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,195 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT/'scripts'), str(Path(__file__).parent)]
|
||||
from dsh_row_selection import selection_hint, select_rows, ranking_chart, validate_selection
|
||||
from dsh_model_input import canonical_proposal, planning_input, compact_tool_result
|
||||
from nao_agent_task import normalize_task, plan_task
|
||||
from nao_native_exploration import prepare_step
|
||||
from nao_quality_tools import QualityToolError, QualityTools, tool_definition
|
||||
from test_nao_agent_task import task_value
|
||||
from test_nao_quality_tools import RELEASE
|
||||
from test_native_exploration import root
|
||||
|
||||
QUESTION = '帮我筛选,10月份合格率最低TOP20产品。且重量大于1000KG,并生成报告'
|
||||
|
||||
|
||||
def selection(limit=20, stage='any'):
|
||||
value = selection_hint(QUESTION)
|
||||
value['limit'] = limit
|
||||
value['weightFilter']['stage'] = stage
|
||||
return value
|
||||
|
||||
|
||||
def population():
|
||||
rows = [{'key': f'P{i:03d}', 'label': f'产品{i:03d}', 'compositeRate': .5 + i/1000,
|
||||
'black': {'rate': .99-i/10000, 'totalWeight': 1001 if i % 2 else 10},
|
||||
'bright': {'rate': .98, 'totalWeight': 1001 if not i % 2 else 10}} for i in range(150)]
|
||||
rows.insert(0, {'key': 'LOW_WEIGHT', 'compositeRate': .01, 'black': {'totalWeight': 1000}, 'bright': {'totalWeight': 1000}})
|
||||
return {'kind': 'quality_metrics', 'releaseId': RELEASE, 'view': 'product',
|
||||
'period': {'year': 2026, 'month': 10}, 'filters': {},
|
||||
'rows': rows, 'rowCount': len(rows), 'rowsReturned': len(rows), 'rowsTruncated': False}
|
||||
|
||||
|
||||
class AggregateSelectionTests(unittest.TestCase):
|
||||
def test_captured_failure_is_bound_as_aggregate_condition_not_source_filter(self):
|
||||
captured = task_value(period={'year':2026,'month':10}, fields=['rate'], stages=['black'], dimensions=['product'], filterUpdates={'extraFilters':[{'key':'totalWeight','operator':'gt','value':1000}]})
|
||||
captured.update(needsClarification=True, clarification='请确认黑片、亮片还是任一阶段投入重量?')
|
||||
before = copy.deepcopy(captured)
|
||||
value = canonical_proposal(captured, question=QUESTION)
|
||||
task = normalize_task(value, QUESTION, {'_businessCatalog': {'years': [2026]}})
|
||||
self.assertTrue(task['needsClarification'])
|
||||
self.assertEqual(task['aggregateSelection']['weightFilter']['value'], 1000)
|
||||
self.assertNotIn('extraFilters', task['filters'])
|
||||
self.assertEqual(captured, before)
|
||||
|
||||
def test_clarification_can_resume_same_month_top_and_report_request(self):
|
||||
previous = root()
|
||||
previous.update(needsClarification=True, aggregateSelection=selection(stage=None), period={'year': 2026, 'month': 10})
|
||||
context = {'_agentTask': previous}
|
||||
for answer, expected in [('黑片or亮片', 'any'), ('我的选择:任一阶段重量大于1000KG', 'any'), ('两阶段均满足', 'all'), ('黑片和亮片都要大于1000', 'all'), ('亮片', 'bright'), ('黑片', 'black')]:
|
||||
fixed = canonical_proposal(task_value(), question=answer, context=context)
|
||||
self.assertEqual(fixed['aggregateSelection']['weightFilter']['stage'], expected)
|
||||
self.assertEqual(fixed['aggregateSelection']['requestQuestion'], QUESTION)
|
||||
self.assertEqual(fixed['period'], {'year': 2026, 'month': 10})
|
||||
self.assertFalse(fixed['needsClarification'])
|
||||
hint = planning_input({'question': QUESTION})['aggregateSelectionHint']
|
||||
self.assertIsNone(hint['weightFilter']['stage'])
|
||||
|
||||
def test_weight_only_model_alias_uses_existing_rank_contract_without_losing_conditions(self):
|
||||
question='2026年10月,按亮片投入重量由高到低列出产品Top100,不生成报告。'
|
||||
value=task_value(fields=['totalWeight'],stages=['bright'],dimensions=['product'],
|
||||
aggregateSelection={'dimension':'product','metric':'totalWeight','rateStage':'bright','direction':'desc','limit':100,'requestQuestion':question})
|
||||
fixed=canonical_proposal(value,question=question)
|
||||
self.assertNotIn('aggregateSelection',fixed)
|
||||
self.assertEqual(fixed['fields'],['totalWeight'])
|
||||
self.assertEqual(fixed['stages'],['bright'])
|
||||
predicate=copy.deepcopy(value);predicate['aggregateSelection']['weightFilter']={'metric':'totalWeight','operator':'gt','value':1000,'stage':'bright'}
|
||||
self.assertIn('aggregateSelection',canonical_proposal(predicate,question=question))
|
||||
|
||||
def test_filter_precedes_sort_and_top_and_does_not_recalculate_composite(self):
|
||||
source = population(); before = copy.deepcopy(source)
|
||||
selected = select_rows(source, selection())
|
||||
self.assertEqual(selected['eligibleRows'], 150)
|
||||
self.assertEqual([row['key'] for row in selected['rows']], [f'P{i:03d}' for i in range(20)])
|
||||
self.assertEqual(selected['rows'][0]['selectedRate'], .5)
|
||||
self.assertEqual(source, before)
|
||||
all_stages = select_rows(source, selection(stage='all'))
|
||||
self.assertEqual(all_stages['returnedRows'], 0)
|
||||
|
||||
def test_top100_and_chart_panels_keep_every_rank(self):
|
||||
source = population(); task = {'aggregateSelection': selection(100)}
|
||||
view = compact_tool_result(source, task=task)
|
||||
self.assertEqual(len(view['rows']), 100)
|
||||
self.assertEqual(view['rowCount'], 151)
|
||||
self.assertEqual(view['selectionDisplay']['eligibleRows'], 150)
|
||||
charts = ranking_chart({'result': source, 'evidenceRef': 'Q12345678'}, task, 'test-turn')
|
||||
self.assertEqual(len(charts), 4)
|
||||
self.assertEqual(sum(len(chart['categories']) for chart in charts), 100)
|
||||
self.assertTrue(all(chart['series'][0]['key'] == chart['metric'] for chart in charts))
|
||||
self.assertEqual(charts[-1]['categories'][-1]['label'], '100. 产品099')
|
||||
|
||||
def test_missing_invalid_rows_and_ties_do_not_become_zero_or_fake_rates(self):
|
||||
source = population()
|
||||
source['rows'][1]['compositeRate'] = None
|
||||
source['rows'][2]['compositeRate'] = float('nan')
|
||||
source['rows'][3]['compositeRate'] = source['rows'][4]['compositeRate']
|
||||
selected = select_rows(source, selection())
|
||||
self.assertEqual(selected['missingRateRows'], 2)
|
||||
self.assertEqual([row['key'] for row in selected['rows'][:2]], ['P002', 'P003'])
|
||||
source['rowsTruncated'] = True
|
||||
with self.assertRaises(ValueError): select_rows(source, selection())
|
||||
|
||||
def test_limit_bounds_and_batch_scope_keep_root_filters(self):
|
||||
for count in (0, 101):
|
||||
with self.assertRaises(ValueError): validate_selection(selection(count))
|
||||
task = root(); task.update(aggregateSelection=selection(100), comparisonPeriod=None)
|
||||
args, proposal = prepare_step({'mode': 'query', 'view': 'product'}, task, {'releaseId': RELEASE, 'scopes': []})
|
||||
self.assertEqual(proposal['node']['dimension'], 'product')
|
||||
self.assertEqual(args['limit'], 500)
|
||||
self.assertEqual(args['period'], task['period'])
|
||||
schema = tool_definition(native_protocol=True, task=task, release_id=RELEASE)['inputSchema']
|
||||
self.assertIn('product', schema['properties']['view']['enum'])
|
||||
task.update(stages=['black'], fields=['totalWeight'], dimensions=['product'])
|
||||
args, proposal = prepare_step({'mode': 'rank', 'dimension': 'product', 'metric': 'totalWeight', 'stage': 'black', 'limit': 100}, task, {'releaseId': RELEASE, 'scopes': []})
|
||||
self.assertEqual(args['limit'], 100)
|
||||
with self.assertRaises(QualityToolError): prepare_step({**args, 'limit': 101}, task, {'releaseId': RELEASE, 'scopes': []})
|
||||
|
||||
def test_model_clarification_cannot_enter_native_execution(self):
|
||||
value = task_value(executionMode='native_agent', needsClarification=True,
|
||||
clarification='重量筛选用黑片、亮片、任一阶段还是两阶段都满足?', clarificationRequest=['黑片','亮片','任一阶段','两阶段均满足'], period={'year': 2026, 'month': 10}, dimensions=['product'])
|
||||
context = {'_protocol': {'autonomousEnabled': True}, '_businessCatalog': {'years': [2026]}}
|
||||
with patch('nao_agent_task._model_json', return_value=value):
|
||||
task = plan_task(base_url='http://test', api_key='test', model='test', question=QUESTION,
|
||||
context=context, cancel_event=threading.Event(), checks_enabled=False)
|
||||
self.assertTrue(task['needsClarification'])
|
||||
self.assertNotEqual(task.get('executionMode'), 'native_agent')
|
||||
self.assertEqual(task['clarification'], value['clarification'])
|
||||
self.assertEqual([entry['value'] for entry in task['clarificationRequest']['options']], value['clarificationRequest'])
|
||||
|
||||
def test_unrelated_new_tasks_and_additional_ambiguity_are_not_swallowed(self):
|
||||
previous = root(); previous.update(needsClarification=True, aggregateSelection=selection(stage=None))
|
||||
context = {'_agentTask': previous}
|
||||
self.assertIsNone(selection_hint('新任务,查询综合率', context))
|
||||
value = task_value(needsClarification=True, clarification='产品有多个候选,请选择具体规格。')
|
||||
fixed = canonical_proposal(value, question='亮片', context=context)
|
||||
self.assertTrue(fixed['needsClarification'])
|
||||
self.assertEqual(fixed['clarification'], value['clarification'])
|
||||
self.assertEqual(selection_hint('改成Top100,亮片', context)['limit'], 100)
|
||||
|
||||
def test_unpaginated_metrics_do_not_require_pagination_only_rowcount(self):
|
||||
raw = {'summary': {'key':'overall','compositeRate': .95}, 'context': {},
|
||||
'rows':[{'key':'P1','compositeRate':.9}], 'coverage': {}, 'trend':[]}
|
||||
tools = object.__new__(QualityTools)
|
||||
with patch.object(tools, '_metrics', return_value=raw) as read:
|
||||
result = tools._query({'id': RELEASE}, {'year':2026,'month':10}, {}, 'product', 500)
|
||||
self.assertEqual(result['rowCount'], 1)
|
||||
self.assertFalse(result['rowsTruncated'])
|
||||
self.assertIsNone(read.call_args.kwargs['row_limit'])
|
||||
raw['context']['rowCount'] = 2
|
||||
with patch.object(tools, '_metrics', return_value=raw):
|
||||
with self.assertRaises(QualityToolError): tools._query({'id':RELEASE}, {'year':2026,'month':10}, {}, 'product', 500)
|
||||
|
||||
def test_product_batch_charts_accept_only_exact_signed_dimension_scope(self):
|
||||
from analysis_visual_evidence import protocol_visual_result
|
||||
value = population()
|
||||
value['protocolEvidence'] = {'nodeId':'products', 'scopeId':'scope-products','queryId':'q-products'}
|
||||
document = {'mode':'query','releaseId':RELEASE,'result':value,'evidenceRef':'Q12345678'}
|
||||
task = root(); task.update(period=value['period'], comparisonPeriod=None, comparisonRequired=False,
|
||||
aggregateSelection=selection(100), dimensions=['product'], fields=['rate','totalWeight'], stages=['black','bright'])
|
||||
scope = {'nodeId':'products','scopeId':'scope-products','purpose':'metrics','task':{**task,'kind':'metrics'}}
|
||||
result = protocol_visual_result([document],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':task})
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
self.assertEqual(sum(len(chart['categories']) for chart in result['visuals']), 100)
|
||||
changed = copy.deepcopy(document); changed['result']['protocolEvidence']['scopeId'] = 'wrong-scope'
|
||||
denied = protocol_visual_result([changed],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':task})
|
||||
self.assertEqual(denied['visuals'], [])
|
||||
empty=copy.deepcopy(task);empty['aggregateSelection']['weightFilter']['value']=1e12
|
||||
no_chart=protocol_visual_result([document],[scope],QUESTION,'test-turn',release_id=RELEASE,base_context={'_agentTask':empty})
|
||||
self.assertEqual(no_chart['visualState'],'not_requested')
|
||||
|
||||
def test_initial_top_comparison_read_is_never_a_single_period_weight_rank(self):
|
||||
import runpy
|
||||
initial = runpy.run_path(str(ROOT/'runtime/dsh/application/quality_native_run.py'))['initial_read_arguments']
|
||||
task = root();task.update(freshQuery=True, dimensions=['product'], fields=['rate','totalWeight'], stages=['bright'])
|
||||
self.assertEqual(initial(task,'合格率下降最多Top100产品'), {'mode':'compare'})
|
||||
task['aggregateSelection']=selection(100)
|
||||
self.assertEqual(initial(task,'合格率下降最多Top100产品'), {'mode':'compare'})
|
||||
|
||||
def test_two_period_rate_change_uses_existing_changes_contract(self):
|
||||
question='比较2026年9月和10月,列出亮片合格率下降最多的产品Top100'
|
||||
value=task_value(fields=['rate'],stages=['bright'],dimensions=['product'],
|
||||
aggregateSelection={'dimension':'product','metric':'rate','rateStage':'bright','direction':'asc','limit':100,'requestQuestion':question})
|
||||
fixed=canonical_proposal(value,question=question)
|
||||
self.assertNotIn('aggregateSelection',fixed)
|
||||
self.assertEqual(fixed['period'],{'year':2026,'month':10})
|
||||
self.assertEqual(fixed['comparisonPeriod'],{'year':2026,'month':9})
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,347 @@
|
||||
"""Exercise the staged DSH worker contract with only authored local fixtures."""
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from build_dsh_application import build
|
||||
|
||||
|
||||
class DshApplicationTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
temporary = tempfile.TemporaryDirectory(prefix='quality-dsh-application-')
|
||||
cls.addClassCleanup(temporary.cleanup)
|
||||
cls.application = Path(temporary.name) / 'worker'
|
||||
build(cls.application)
|
||||
candidate = str(cls.application / 'scripts')
|
||||
sys.path.insert(0, candidate)
|
||||
cls.addClassCleanup(sys.path.remove, candidate)
|
||||
names = [path.stem for path in (cls.application / 'scripts').glob('*.py')]
|
||||
original_modules = {name: sys.modules.pop(name) for name in names if name in sys.modules}
|
||||
|
||||
def restore_modules():
|
||||
for name in names:
|
||||
sys.modules.pop(name, None)
|
||||
sys.modules.update(original_modules)
|
||||
|
||||
cls.addClassCleanup(restore_modules)
|
||||
cls.worker = importlib.import_module('quality_analysis_runtime')
|
||||
cls.context = importlib.import_module('quality_runtime_context')
|
||||
cls.drafts = importlib.import_module('quality_draft')
|
||||
cls.reports = importlib.import_module('quality_report_deliverable')
|
||||
cls.original = importlib.import_module('quality_original_answer')
|
||||
|
||||
@staticmethod
|
||||
def request(question, **changes):
|
||||
request = {key: str(uuid.uuid4()) for key in ('turnId', 'sessionId', 'actorId', 'releaseId')}
|
||||
request.update(question=question, runtimeEngine='DSH', accessToken='fixture-only-delegation',
|
||||
context={'_protocol': {'version': 1, 'attemptId': request['sessionId'],
|
||||
'autonomousEnabled': True}})
|
||||
return {**request, **changes}
|
||||
|
||||
def test_independent_worker_preserves_delegation_and_publishes_original_report(self):
|
||||
request = self.request('请输出黑片10月质量对比报告')
|
||||
task = {'kind': 'analysis', 'executionMode': 'native_agent'}
|
||||
calls = []
|
||||
access_files = []
|
||||
answer = '## 本期质量概况\n\n| 项目 | 本期 |\n| --- | --- |\n| 例示指标 | 42 |'
|
||||
|
||||
def delegate(instance, path, **kwargs):
|
||||
calls.append(path)
|
||||
self.assertEqual(instance.access_path.stat().st_mode & 0o777, 0o600)
|
||||
access_files.append(instance.access_path)
|
||||
if path == '/api/analysis/protocol/task':
|
||||
self.assertEqual(kwargs['payload'], task)
|
||||
return {'task': task, 'attemptId': request['sessionId']}
|
||||
return {}
|
||||
|
||||
def execute(**options):
|
||||
self.assertEqual(options['authority']['task'], task)
|
||||
self.assertNotIn('accessToken', options['request'])
|
||||
self.assertEqual(options['connection'][0].split(':')[1], '//127.0.0.1')
|
||||
self.original.current().capture_native(answer)
|
||||
options['emit']({'type': 'result', 'status': 'COMPLETED', 'answer': answer,
|
||||
'agentTask': task, 'reportTitle': '10月黑片质量对比', 'protocolEvidence': [{'queryId': 'fixture'}],
|
||||
'completionMethod': 'native_autonomous_analysis',
|
||||
'answerQuality': {'narrativeFactBinding': 'verified_composition'},
|
||||
'timeline': [{'kind': 'assistant', 'text': '临时草稿'},
|
||||
{'kind': 'tool_result', 'toolCallId': 'dsh-tool-fixture'}]})
|
||||
|
||||
output = io.StringIO()
|
||||
with (patch('setup_dsh_runtime.inspect_runtime', return_value={'ready': True}),
|
||||
patch('quality_model_settings.model_configuration', return_value=('https://fixture.invalid/v1', 'fixture-key', 'fixture-model', {})),
|
||||
patch.object(self.context.DelegatedApi, 'call', delegate),
|
||||
patch.object(self.worker, '_plan', return_value=task),
|
||||
patch('quality_native_run.run_native_exploration', side_effect=execute)):
|
||||
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
|
||||
self.assertEqual(calls, ['/api/analysis/model-authorization', '/api/analysis/model-authorization',
|
||||
'/api/analysis/protocol/task'])
|
||||
self.assertTrue(all(not path.exists() for path in access_files))
|
||||
result = json.loads(output.getvalue().splitlines()[-1])
|
||||
self.assertEqual(result['engine'], 'DSH')
|
||||
self.assertEqual(result['reportMarkdown'], result['answer'])
|
||||
self.assertEqual(result['reportTitle'], '本期质量概况')
|
||||
self.assertEqual(result['timeline'], [{'kind': 'tool_result', 'toolCallId': 'dsh-tool-fixture'}])
|
||||
self.assertNotIn('fixture-only-delegation', output.getvalue())
|
||||
|
||||
def test_engine_and_attempt_are_pinned_before_access(self):
|
||||
requests = (self.request('检查指标', runtimeEngine='legacy'),
|
||||
self.request('检查指标', context={'_protocol': {'version': 1, 'attemptId': str(uuid.uuid4())}}))
|
||||
with patch.object(self.context.DelegatedApi, 'call') as delegated:
|
||||
for request in requests:
|
||||
output = io.StringIO()
|
||||
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
|
||||
self.assertEqual(json.loads(output.getvalue())['status'], 'FAILED')
|
||||
delegated.assert_not_called()
|
||||
|
||||
def test_worker_retains_original_text_when_execution_raises(self):
|
||||
request = self.request('输出质量报告')
|
||||
task = {'kind': 'analysis', 'executionMode': 'native_agent'}
|
||||
body = ' 模型原始内容,后续检查异常也要保留。\n'
|
||||
|
||||
def execute(**_options):
|
||||
self.original.current().capture_native(body)
|
||||
raise ValueError('fixture validation rejection')
|
||||
|
||||
output = io.StringIO()
|
||||
with (patch('setup_dsh_runtime.inspect_runtime', return_value={'ready': True}),
|
||||
patch('quality_model_settings.model_configuration', return_value=('https://fixture.invalid/v1', 'fixture-key', 'fixture-model', {})),
|
||||
patch.object(self.context.DelegatedApi, 'call', return_value={'task': task, 'attemptId': request['sessionId']}),
|
||||
patch.object(self.worker, '_plan', return_value=task),
|
||||
patch('quality_native_run.run_native_exploration', side_effect=execute)):
|
||||
self.worker.run_turn(request, stdin=io.StringIO(), stdout=output)
|
||||
result = json.loads(output.getvalue().splitlines()[-1])
|
||||
self.assertEqual(result['status'], 'FAILED')
|
||||
self.assertEqual(result['answer'], body)
|
||||
self.assertEqual(result['modelAnswer'], body)
|
||||
self.assertEqual(result['reportMarkdown'], body)
|
||||
|
||||
def test_native_draft_replaces_and_clears_without_emitting_private_text(self):
|
||||
frames = []
|
||||
now = [1.0]
|
||||
publisher = self.drafts.DraftPublisher(frames.append, clock=lambda: now[0])
|
||||
text = '10月黑片质量指标已进入核验,正在检查异常变化和对象范围。'
|
||||
publisher.update('fixture-attempt-id', text)
|
||||
publisher.update('fixture-attempt-id', text + '第二个片段。')
|
||||
self.assertEqual(len(frames), 1)
|
||||
self.assertNotEqual(frames[0]['dshAttemptId'], 'fixture-attempt-id')
|
||||
now[0] += 0.26
|
||||
publisher.update('fixture-attempt-id', text + '补充结论。')
|
||||
self.assertEqual(frames[1]['dshDraft'], text + '补充结论。')
|
||||
publisher.update('fixture-attempt-id', text + '内部请求归属标记')
|
||||
self.assertEqual(frames[-1], {'type': 'progress', 'dshDraft': ''})
|
||||
|
||||
def test_terminal_status_preserves_native_body_after_failed_attempt(self):
|
||||
import dsh_agent_runtime
|
||||
for status in ('PARTIAL', 'FAILED', 'CANCELLED'):
|
||||
frames = []
|
||||
body = ' ## 模型的原始判断\n\n98.04%显著低于组合乘积。\n'
|
||||
with self.original.OriginalAnswer(frames.append) as original:
|
||||
projection = dsh_agent_runtime._Projection(self.context.ConversationAccumulator(), frames.append)
|
||||
projection._text('failed-provider-attempt', body)
|
||||
projection._discard('failed-provider-attempt')
|
||||
result = {'type': 'result', 'status': status, 'answer': '程序替换文本',
|
||||
'readingSummary': {'overview': '固定摘要'}, 'stopReason': 'validation_failed'}
|
||||
original.settle(result)
|
||||
self.reports.attach(result, '输出质量分析报告')
|
||||
self.assertEqual(result['answer'], body)
|
||||
self.assertEqual(result['modelAnswer'], body)
|
||||
self.assertEqual(result['reportMarkdown'], body)
|
||||
self.assertNotIn('readingSummary', result)
|
||||
self.assertEqual(result['answerValidation']['stopReason'], 'validation_failed')
|
||||
self.assertTrue(any(frame.get('modelAnswer') == body for frame in frames))
|
||||
|
||||
def test_bounded_business_blocks_preserve_order_whitespace_and_original_title(self):
|
||||
frames = []
|
||||
texts = [' ## 先看这项\n原始数字999.9%。 ', '| 项目 | 结果 |\n| --- | --- |\n| X | 5 |', '最后一段\n']
|
||||
raw = json.dumps({'reportTitle': '模型原始标题', 'blocks': [
|
||||
{'role': 'detail', 'text': texts[0]}, {'role': 'core', 'text': texts[1]},
|
||||
{'role': 'key_evidence', 'text': texts[2]}]}, ensure_ascii=False)
|
||||
with self.original.OriginalAnswer(frames.append) as original:
|
||||
self.original.capture(raw)
|
||||
self.original.capture(json.dumps({'blocks': [{'text': '第二次修复后的替代内容'}]}))
|
||||
result = {'type': 'result', 'status': 'PARTIAL', 'answer': '正式统计固定回退',
|
||||
'reportTitle': '程序改写标题', 'answerQuality': {'narrativeComplete': False}}
|
||||
original.settle(result)
|
||||
self.assertEqual(result['answer'], '\n\n'.join(texts))
|
||||
self.assertEqual(result['reportTitle'], '模型原始标题')
|
||||
self.assertEqual(result['answerOrigin'], 'model_business_text')
|
||||
|
||||
def test_business_response_survives_invalid_composition_and_interrupted_finish(self):
|
||||
import quality_native_answer
|
||||
import threading
|
||||
raw_body = '{"blocks":[{"text":"模型未完成的原始回复,格式未闭合'
|
||||
envelope = json.dumps({'choices': [{'message': {'content': raw_body}, 'finish_reason': 'length'}]})
|
||||
with self.original.OriginalAnswer(lambda _value: None) as original:
|
||||
with io.BytesIO(envelope.encode()) as response:
|
||||
content, _, finish = quality_native_answer._request_model(
|
||||
connection=('https://fixture.invalid/v1', 'fixture-only-key', 'fixture-model'),
|
||||
messages=[{'role': 'user', 'content': '查询质量'}], cancel_event=threading.Event(),
|
||||
opener=lambda *_args, **_kwargs: response, phase='native_synthesis')
|
||||
result = {'type': 'result', 'status': 'FAILED', 'answer': '固定回退文本'}
|
||||
original.settle(result)
|
||||
self.assertEqual(content, raw_body)
|
||||
self.assertEqual(finish, 'length')
|
||||
self.assertEqual(result['answer'], raw_body)
|
||||
|
||||
def test_review_metadata_does_not_replace_native_answer_and_empty_model_stays_empty(self):
|
||||
frames = []
|
||||
with self.original.OriginalAnswer(frames.append) as original:
|
||||
original.capture_native('原始模型正文')
|
||||
self.original.capture('{"blocks":[{"text":"复核模型改写"}]}')
|
||||
result = {'type': 'result', 'status': 'PARTIAL', 'answer': '固定统计摘要'}
|
||||
original.settle(result)
|
||||
self.assertEqual(result['answer'], '原始模型正文')
|
||||
with self.original.OriginalAnswer(frames.append) as original:
|
||||
self.original.capture('{"reviews":[]}', phase='native_draft_review')
|
||||
result = {'type': 'result', 'status': 'FAILED', 'answer': '程序生成的失败正文', 'message': '模型连接失败'}
|
||||
original.settle(result)
|
||||
self.assertEqual(result['answer'], '')
|
||||
self.assertEqual(result['modelAnswer'], '')
|
||||
self.assertEqual(result['message'], '模型连接失败')
|
||||
|
||||
def test_report_request_reuses_original_body_for_any_validation_status(self):
|
||||
self.assertFalse(self.reports.requested('只列产品排名,不生成报告'))
|
||||
self.assertTrue(self.reports.requested('不分析原因,生成报告'))
|
||||
self.assertTrue(self.reports.requested('生成一份不推断原因的报告'))
|
||||
for status in ('COMPLETED', 'PARTIAL', 'FAILED', 'CANCELLED'):
|
||||
for question in ('生成一份对比报告', '请给我一份排查简报,优先列风险'):
|
||||
result = {'type': 'result', 'status': status, 'modelAnswer': '## 模型报告\n\n原始正文。',
|
||||
'answerQuality': {'narrativeComplete': False}, 'protocolEvidence': []}
|
||||
self.reports.attach(result, question)
|
||||
self.assertEqual(result['reportMarkdown'], result['modelAnswer'])
|
||||
result = {'type': 'result', 'status': 'COMPLETED', 'modelAnswer': '原文', 'reportMarkdown': '过期的报告'}
|
||||
self.reports.attach(result, '不要写报告,先核对质量数据')
|
||||
self.assertNotIn('reportMarkdown', result)
|
||||
|
||||
def test_direct_delivery_never_recomposes_or_requires_query_evidence(self):
|
||||
runner = importlib.import_module('quality_native_run')
|
||||
task = {'kind': 'analysis', 'executionMode': 'native_agent', 'filters': {},
|
||||
'period': {'year': 2026}, 'fields': ['rate'], 'stages': []}
|
||||
request = self.request('生成质量报告')
|
||||
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
|
||||
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
|
||||
accumulator = SimpleNamespace(answer='## 模型原文\n\n未经额外核验的原始回答。',
|
||||
finish_reasons=['stop'], tool_calls=[], timeline=[])
|
||||
delegate = SimpleNamespace(call=lambda *args, **kwargs: {})
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'private').mkdir()
|
||||
(root / 'evidence').mkdir()
|
||||
with (patch('dsh_agent_runtime.run_dsh_agent', return_value=(accumulator, 'COMPLETED', 'stop', True)),
|
||||
patch('quality_evidence_finalizer.synthesize_native', side_effect=AssertionError('extra model review')),
|
||||
patch('analysis_visual_evidence.protocol_visual_result', side_effect=ValueError('chart unavailable'))):
|
||||
result = runner.run_native_exploration(request=request, task=task, authority=authority,
|
||||
actor_root=root, delegated=delegate, connection=('http://127.0.0.1:1/v1', 'fixture', 'fixture'),
|
||||
cancel_event=threading.Event(), emit=lambda event: None, prior_evidence_files=frozenset())
|
||||
self.assertEqual(result['status'], 'COMPLETED')
|
||||
self.assertEqual(result['answer'], accumulator.answer)
|
||||
self.assertEqual(result['phaseTimingsMs']['synthesis'], 0)
|
||||
self.assertNotIn('answerQuality', result)
|
||||
|
||||
def test_business_data_is_read_before_model_and_seeded_in_the_native_stream(self):
|
||||
runner = importlib.import_module('quality_native_run')
|
||||
task = {'kind': 'analysis', 'freshQuery': True, 'executionMode': 'native_agent', 'filters': {},
|
||||
'period': {'year': 2026, 'month': 8}, 'fields': ['rate'], 'stages': []}
|
||||
request = self.request('2026年8月综合合格率,一句话')
|
||||
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
|
||||
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
|
||||
phases = []
|
||||
def data(_args):
|
||||
phases.append('read')
|
||||
return {'kind': 'quality_metrics', 'summary': {'compositeRate': .9557}, 'evidenceRef': 'Q12345678'}
|
||||
def model(**kwargs):
|
||||
phases.append('model')
|
||||
self.assertEqual(kwargs['context']['initialObservation']['summary']['compositeRate'], .9557)
|
||||
seeded = kwargs['initial_accumulator']
|
||||
self.assertEqual(seeded.tool_calls[0]['status'], 'completed')
|
||||
seeded.feed({'type':'text-delta','id':'original','delta':'模型原文:95.57%。'})
|
||||
return seeded, 'COMPLETED', 'stop', True
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root=Path(directory); (root/'private').mkdir(); (root/'evidence').mkdir()
|
||||
with (patch('quality_native_exploration.NativeExplorationTools.quality_data', side_effect=data),
|
||||
patch('dsh_agent_runtime.run_dsh_agent', side_effect=model)):
|
||||
result = runner.run_native_exploration(request=request, task=task, authority=authority,
|
||||
actor_root=root, delegated=SimpleNamespace(call=lambda *args, **kwargs: {}),
|
||||
connection=('http://127.0.0.1:1/v1','fixture','fixture'),cancel_event=threading.Event(),
|
||||
emit=lambda event:None,prior_evidence_files=frozenset())
|
||||
self.assertEqual(phases,['read','model'])
|
||||
self.assertEqual(result['answer'],'模型原文:95.57%。')
|
||||
self.assertEqual(result['toolCount'],1)
|
||||
|
||||
def test_all_failed_business_calls_keep_raw_answer_but_do_not_claim_completion(self):
|
||||
runner = importlib.import_module('quality_native_run')
|
||||
task = {'kind': 'analysis', 'executionMode': 'native_agent', 'filters': {},
|
||||
'period': {'year': 2026}, 'fields': ['rate'], 'stages': []}
|
||||
request = self.request('比较9月和10月')
|
||||
authority = {**{key: request[key] for key in ('sessionId', 'turnId')},
|
||||
'attemptId': request['sessionId'], 'revision': 1, 'executionId': str(uuid.uuid4())}
|
||||
accumulator = SimpleNamespace(answer='原始模型说明:本轮查询没有成功。', finish_reasons=['stop'],
|
||||
tool_calls=[{'input': {'mode': 'compare'}, 'status': 'failed'} for _ in range(3)], timeline=[])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory); (root/'private').mkdir(); (root/'evidence').mkdir()
|
||||
with patch('dsh_agent_runtime.run_dsh_agent', return_value=(accumulator, 'COMPLETED', 'stop', True)):
|
||||
result = runner.run_native_exploration(request=request, task=task, authority=authority,
|
||||
actor_root=root, delegated=SimpleNamespace(call=lambda *args, **kwargs: {}),
|
||||
connection=('http://127.0.0.1:1/v1', 'fixture', 'fixture'),
|
||||
cancel_event=threading.Event(), emit=lambda event: None, prior_evidence_files=frozenset())
|
||||
self.assertEqual(result['status'], 'FAILED')
|
||||
self.assertEqual(result['answer'], accumulator.answer)
|
||||
self.assertEqual(result['stopReason'], 'tool_queries_failed')
|
||||
|
||||
def test_unchecked_planning_calls_model_once_without_semantic_review_or_repair(self):
|
||||
planner = importlib.import_module('quality_agent_task')
|
||||
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
|
||||
'_businessCatalog': {'years': [2026], 'latestDataPeriod': '2026-10'}}
|
||||
value = {'goal': '2026年综合合格率', 'kind': 'metrics', 'freshQuery': True,
|
||||
'dimensions': [], 'stages': [], 'fields': ['rate'], 'verifySource': False,
|
||||
'fullDetails': False, 'needsClarification': False, 'filterUpdates': {}, 'period': {'year': 2026}}
|
||||
with patch.object(planner, '_model_json', return_value=value) as call:
|
||||
result = planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
|
||||
question='2026年综合合格率', context=context, cancel_event=threading.Event(),
|
||||
checks_enabled=False, semantic_review=True)
|
||||
self.assertEqual(call.call_count, 1)
|
||||
self.assertEqual(result['executionMode'], 'native_agent')
|
||||
with patch.object(planner, '_model_json', return_value={'kind': 'invalid'}) as call:
|
||||
with self.assertRaises(planner.TaskUnavailable):
|
||||
planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
|
||||
question='q', context=context, cancel_event=threading.Event(), checks_enabled=False)
|
||||
self.assertEqual(call.call_count, 1)
|
||||
|
||||
def test_direct_agent_planning_does_not_block_presentation_or_grouping_choices(self):
|
||||
planner = importlib.import_module('quality_agent_task')
|
||||
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
|
||||
'_businessCatalog': {'years': [2026], 'latestDataPeriod': {'year': 2026, 'month': 10}}}
|
||||
# The exact period/trend combination which stopped the real report.
|
||||
proposal = {'goal': '生成最近月份质量报告', 'kind': 'analysis', 'freshQuery': True,
|
||||
'dimensions': [], 'stages': ['black', 'bright'],
|
||||
'fields': ['rate', 'totalWeight', 'scrapWeight', 'scrapRate'],
|
||||
'verifySource': False, 'fullDetails': False, 'needsClarification': False,
|
||||
'filterUpdates': {}, 'period': {'year': 2026, 'month': 10},
|
||||
'trendRequired': True, 'comparisonRequired': False}
|
||||
for dimensions in ([], ['group']):
|
||||
with self.subTest(dimensions=dimensions), patch.object(
|
||||
planner, '_model_json', return_value={**proposal, 'dimensions': dimensions}) as call:
|
||||
result = planner.plan_task(base_url='http://fixture', api_key='fixture', model='fixture',
|
||||
question='生成一份最近月份的质量分析报告', context=context,
|
||||
cancel_event=threading.Event(), checks_enabled=False, semantic_review=True)
|
||||
self.assertEqual(call.call_count, 1)
|
||||
self.assertEqual(result['period'], proposal['period'])
|
||||
self.assertTrue(result['trendRequired'])
|
||||
self.assertEqual(result['dimensions'], dimensions)
|
||||
self.assertEqual(result['executionMode'], 'native_agent')
|
||||
self.assertNotIn('queryPlan', result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,80 @@
|
||||
import sys
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'verification')]
|
||||
import verify_dsh_formal as fixture
|
||||
import isolated_runtime
|
||||
|
||||
|
||||
class DshFormalFixtureTests(unittest.TestCase):
|
||||
def test_seed_is_three_months_two_stages_two_org_groups_and_six_products(self):
|
||||
rows = fixture.fixture_rows()
|
||||
self.assertEqual(len(rows), 36)
|
||||
self.assertEqual({row.month for row in rows}, {'2026-08', '2026-09', '2026-10'})
|
||||
self.assertEqual({row.stage for row in rows}, {'black', 'bright'})
|
||||
self.assertEqual({row.department for row in rows}, set(fixture.DEPARTMENTS))
|
||||
self.assertEqual({row.group for row in rows}, set(fixture.GROUPS))
|
||||
self.assertEqual({row.product for row in rows}, set(fixture.PRODUCTS))
|
||||
for month in fixture.PERIODS:
|
||||
for stage in ('black', 'bright'):
|
||||
selected = [row for row in rows if row.month == month and row.stage == stage]
|
||||
self.assertEqual({row.product for row in selected}, set(fixture.PRODUCTS))
|
||||
|
||||
def test_missing_detail_is_explicit_and_formal_rollup_stays_complete(self):
|
||||
rows = fixture.fixture_rows()
|
||||
missing = [row for row in rows if row.month == '2026-09' and row.stage == 'black'
|
||||
and row.product == '合成产品06']
|
||||
self.assertEqual(len(missing), 1)
|
||||
self.assertIn('责废尺寸重量', missing[0].raw)
|
||||
self.assertIsNone(missing[0].raw['责废尺寸重量'])
|
||||
self.assertGreater(missing[0].components['black_responsibility_scrap'], 0)
|
||||
self.assertEqual(fixture.fixture_totals(rows, 'black', '2026-10')['rows'], 6)
|
||||
self.assertEqual(fixture.fixture_totals(rows, 'bright', '2026-10')['rows'], 6)
|
||||
|
||||
def test_generated_archive_is_a_hashed_synthetic_xlsx_with_source_fields(self):
|
||||
rows = [row for row in fixture.fixture_rows()
|
||||
if row.stage == 'black' and row.month == '2026-09']
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / 'synthetic.xlsx'
|
||||
digest, size = fixture._write_workbook(path, rows)
|
||||
self.assertEqual(digest, hashlib.sha256(path.read_bytes()).hexdigest())
|
||||
self.assertEqual(size, path.stat().st_size)
|
||||
with ZipFile(path) as archive:
|
||||
self.assertTrue({'[Content_Types].xml', '_rels/.rels', 'xl/workbook.xml',
|
||||
'xl/worksheets/sheet1.xml'}.issubset(set(archive.namelist())))
|
||||
worksheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
|
||||
cells = [element.text for element in worksheet.iter() if element.tag.endswith('}t')]
|
||||
self.assertIn('责废尺寸重量', cells)
|
||||
self.assertIn('责废缺角重量', cells)
|
||||
|
||||
def test_source_physical_rows_restart_for_each_source_workbook(self):
|
||||
values = list(fixture.source_physical_rows(fixture.fixture_rows()))
|
||||
first_file = [row_no for row, row_no in values if row.stage == 'black' and row.month == '2026-08']
|
||||
next_file = [row_no for row, row_no in values if row.stage == 'bright' and row.month == '2026-08']
|
||||
self.assertEqual(first_file, list(range(2, 8)))
|
||||
self.assertEqual(next_file, list(range(2, 8)))
|
||||
|
||||
def test_private_sql_literals_are_hex_encoded(self):
|
||||
self.assertEqual(fixture._literal("模型'; DROP DATABASE nope;"),
|
||||
'CONVERT(0x' + "模型'; DROP DATABASE nope;".encode().hex() + ' USING utf8mb4)')
|
||||
self.assertIn('CAST(CONVERT(0x', fixture._json_sql({'来源': '合成值'}))
|
||||
|
||||
def test_isolated_runtime_accepts_only_scoped_engine_and_protocol_overrides(self):
|
||||
signature = isolated_runtime.isolated_api.__wrapped__.__code__.co_varnames
|
||||
self.assertIn('env_overrides', signature)
|
||||
self.assertIn('auth_credentials', signature)
|
||||
source = (ROOT / 'verification/isolated_runtime.py').read_text(encoding='utf-8')
|
||||
self.assertIn("'QUALITY_ARCHIVE_ROOT'", source)
|
||||
self.assertIn("'SERVER_SERVLET_SESSION_COOKIE_NAME'", source)
|
||||
self.assertTrue(callable(isolated_runtime.sql))
|
||||
self.assertTrue(callable(isolated_runtime.clone_database))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts')]
|
||||
from dsh_model_input import compact_tool_result, planning_input, canonical_proposal, explicit_comparison_periods, recent_trend_period, DIRECT_PLANNING_PROMPT
|
||||
from nao_quality_tools import tool_definition
|
||||
from nao_weight_ranking import rank_response
|
||||
|
||||
|
||||
class DshModelInputTests(unittest.TestCase):
|
||||
def test_recent_monthly_trend_binds_published_months_instead_of_one_month(self):
|
||||
context = {'_businessCatalog': {'periods': [f'2026-{month:02d}' for month in range(1, 11)],
|
||||
'latestDataPeriod': {'year': 2026, 'month': 10}}}
|
||||
value = {'kind': 'analysis', 'period': {'year': 2026, 'month': 10},
|
||||
'trendRequired': True, 'comparisonRequired': True}
|
||||
fixed = canonical_proposal(value, question='分析最近月份合格率的变化趋势', context=context)
|
||||
self.assertEqual(fixed['period'], {'year': 2026, 'monthStart': 5, 'monthEnd': 10})
|
||||
self.assertTrue(fixed['trendRequired'])
|
||||
self.assertFalse(fixed['comparisonRequired'])
|
||||
three = recent_trend_period('最近3个月合格率走势', context)
|
||||
self.assertEqual(three['period'], {'year': 2026, 'monthStart': 8, 'monthEnd': 10})
|
||||
self.assertEqual(recent_trend_period('最近三个月走势', context), three)
|
||||
self.assertEqual(recent_trend_period('最近3个已发布月份走势', context), three)
|
||||
self.assertFalse(recent_trend_period('最近1个月走势', context)['trendRequired'])
|
||||
self.assertIsNone(recent_trend_period('查看2026年8月至10月的近期走势', context))
|
||||
self.assertIsNone(recent_trend_period('最近12个月走势', context))
|
||||
self.assertIsNone(recent_trend_period('最近三年合格率趋势', context))
|
||||
self.assertEqual(value['period'], {'year': 2026, 'month': 10})
|
||||
def test_stage_comparison_and_monthly_trend_do_not_require_a_second_period(self):
|
||||
result = canonical_proposal({'kind': 'analysis', 'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 10}, 'dimensions': ['month'],
|
||||
'comparisonRequired': True, 'trendRequired': True, 'newTask': True},
|
||||
question='生成2026年1—10月报告,给出黑片和亮片月度对照及趋势')
|
||||
self.assertFalse(result['comparisonRequired'])
|
||||
self.assertEqual(result['dimensions'], [])
|
||||
self.assertTrue(result['trendRequired'])
|
||||
prior = {'comparisonPeriod': {'year': 2026, 'month': 9}}
|
||||
followup = canonical_proposal({'kind': 'analysis', 'comparisonRequired': True},
|
||||
question='只看黑片', context=prior)
|
||||
self.assertEqual(followup['comparisonPeriod'], prior['comparisonPeriod'])
|
||||
|
||||
def test_explicit_two_month_comparison_cannot_be_misread_as_one_range(self):
|
||||
source = {'kind': 'analysis', 'fields': ['rate'], 'comparisonPeriod': {'year': 2026, 'monthStart': 9, 'monthEnd': 10}}
|
||||
binding = {'period': {'year': 2026, 'month': 10}, 'comparisonPeriod': {'year': 2026, 'month': 9}}
|
||||
result = canonical_proposal(source, question='比较2026年9月和10月公司总体合格率')
|
||||
self.assertEqual(result['period'], binding['period'])
|
||||
self.assertEqual(result['comparisonPeriod'], binding['comparisonPeriod'])
|
||||
self.assertEqual(explicit_comparison_periods('2026年10月比9月变化多少'), binding)
|
||||
self.assertIsNone(explicit_comparison_periods('生成2026年1—10月总体质量报告'))
|
||||
self.assertIsNone(explicit_comparison_periods('不要比较2026年9月和10月'))
|
||||
self.assertEqual(explicit_comparison_periods('比较2026年1—3月和2025年1—3月')['period'],
|
||||
{'year': 2026, 'monthStart': 1, 'monthEnd': 3})
|
||||
|
||||
def test_response_field_aliases_do_not_require_another_model_call(self):
|
||||
value = {'fields': ['compositeRate', 'blackRate', 'brightRate', 'rateDeltaPP']}
|
||||
self.assertEqual(canonical_proposal(value)['fields'], ['rate'])
|
||||
self.assertEqual(value['fields'], ['compositeRate', 'blackRate', 'brightRate', 'rateDeltaPP'])
|
||||
self.assertEqual(canonical_proposal({'fields': ['unknownMetric']})['fields'], ['unknownMetric'])
|
||||
|
||||
def test_one_rank_query_preserves_auxiliary_metrics_for_the_same_selected_objects(self):
|
||||
source = {'summary': {'black': {'scrapWeight': 3, 'totalWeight': 100}},
|
||||
'rows': [{'key': 'A', 'label': '甲', 'black': {'scrapWeight': 2, 'totalWeight': 80}},
|
||||
{'key': 'B', 'label': '乙', 'black': {'scrapWeight': 1, 'totalWeight': 20}}],
|
||||
'coverage': {'sourceMonths': [{'year': 2026, 'month': 10, 'blackRows': 2}]}}
|
||||
before = copy.deepcopy(source)
|
||||
result = rank_response(source, {'id': 'fixture'}, {'year': 2026, 'month': 10}, {},
|
||||
'product', 'black', 'scrapWeight', 1, include_metrics=True)
|
||||
self.assertEqual(result['rows'][0]['key'], 'A')
|
||||
self.assertEqual(result['rows'][0]['metrics'], {'scrapWeight': 2.0, 'totalWeight': 80.0})
|
||||
self.assertEqual(result['totalWeight'], 3.0)
|
||||
self.assertEqual(source, before)
|
||||
|
||||
def test_native_tool_contract_exposes_only_executable_task_operations(self):
|
||||
task = {'period': {'year': 2026, 'month': 10}, 'stages': ['black'], 'dimensions': ['product','defect']}
|
||||
schema = tool_definition(native_protocol=True, task=task, release_id='release')['inputSchema']
|
||||
self.assertEqual(set(schema['properties']['mode']['enum']), {'catalog', 'query', 'rank', 'defects'})
|
||||
self.assertNotIn('top', tool_definition(native_protocol=True)['inputSchema']['properties']['mode']['enum'])
|
||||
self.assertEqual(schema['properties']['period']['const'], task['period'])
|
||||
self.assertEqual(schema['properties']['stage']['enum'], ['black'])
|
||||
self.assertEqual(schema['properties']['stage']['type'], 'string')
|
||||
self.assertNotIn('comparisonPeriod', schema['properties'])
|
||||
self.assertNotIn('action', schema['properties'])
|
||||
task.update(comparisonPeriod={'year': 2026, 'month': 9}, fullDetails=True)
|
||||
schema = tool_definition(native_protocol=True, task=task)['inputSchema']
|
||||
self.assertTrue({'compare', 'changes', 'details', 'investigate'} <= set(schema['properties']['mode']['enum']))
|
||||
self.assertEqual(schema['properties']['action']['enum'], ['rows'])
|
||||
overview = tool_definition(native_protocol=True, task={'kind':'analysis','dimensions':[],
|
||||
'question':'生成总体报告,给出月度趋势和报废大类'})['inputSchema']
|
||||
self.assertEqual(overview['properties']['mode']['enum'], ['catalog','query','rank'])
|
||||
|
||||
def test_compaction_keeps_numbers_nulls_coverage_and_source_immutable(self):
|
||||
summary = {'compositeRate': 0, 'black': {'rate': None, 'scrapWeight': 12.3456,
|
||||
'formalDisplay': {'scrapWeight': {'formatted': '12.35 kg'}}}}
|
||||
source = {'summary': summary, 'rows': [copy.deepcopy(summary)], 'rowCount': 1,
|
||||
'coverage': {'missingStages': ['black']}, 'calculationRules': {'internal': 'rule'},
|
||||
'metricFormulaDescriptions': {'compositeRate': 'formal rule'},
|
||||
'trend': [{'periodLabel': '2026-09', 'compositeRate': 0.9512},
|
||||
{'periodLabel': '2026-10', 'compositeRate': 0.9546}]}
|
||||
original = copy.deepcopy(source)
|
||||
view = compact_tool_result(source)
|
||||
self.assertEqual(source, original)
|
||||
self.assertEqual(view['summary']['black'], {'rate': None, 'scrapWeight': 12.3456})
|
||||
self.assertEqual(view['coverage'], original['coverage'])
|
||||
self.assertEqual(view['metricFormulaDescriptions'], original['metricFormulaDescriptions'])
|
||||
self.assertNotIn('rows', view)
|
||||
self.assertEqual(view['trend'], source['trend'])
|
||||
self.assertEqual(view['trendSummary']['compositeRate']['minimum']['period'], '2026-09')
|
||||
self.assertEqual(view['trendSummary']['compositeRate']['firstToLastPP'], 0.34)
|
||||
source['rows'].append({'compositeRate': 0.9})
|
||||
remaining = compact_tool_result(source)['rows']
|
||||
self.assertEqual(len(remaining), 2)
|
||||
self.assertEqual(remaining[0]['black']['scrapWeight'], 12.3456)
|
||||
self.assertEqual(remaining[1], {'compositeRate': 0.9})
|
||||
|
||||
def test_planning_keeps_scope_and_questions_without_duplicate_answer_bodies(self):
|
||||
payload = {'question': '只看黑片', 'previousTask': {'filters': {'department': '甲'}},
|
||||
'protocol': {'availableRankings': [{'name': '真实产品'}]},
|
||||
'previousVerifiedTurns': [{'turnId': 'old', 'question': '前问', 'answer': '重复正文'*100}],
|
||||
'previousUserQuestions': [{'question': '前问', 'status': 'COMPLETED'}],
|
||||
'lastVisibleAnswer': '重复正文'*100}
|
||||
before = copy.deepcopy(payload)
|
||||
result = planning_input(payload)
|
||||
self.assertEqual(payload, before)
|
||||
self.assertEqual(result['previousTask'], payload['previousTask'])
|
||||
self.assertEqual(result['protocol'], payload['protocol'])
|
||||
self.assertEqual(result['previousVerifiedTurns'], [{'turnId': 'old', 'question': '前问'}])
|
||||
self.assertNotIn('lastVisibleAnswer', result)
|
||||
self.assertNotIn('queryPlan=', DIRECT_PLANNING_PROMPT)
|
||||
|
||||
def test_catalog_does_not_advertise_legacy_top_or_unavailable_raw_reads(self):
|
||||
source = {'kind': 'quality_catalog', 'topDimensions': ['category', 'product', 'group'],
|
||||
'sourceFields': [{'key': 'source-field'}], 'fineDefects': {'fields': ['real-defect']},
|
||||
'investigation': {'actions': ['overview', 'fields', 'rows', 'source']}}
|
||||
view = compact_tool_result(source, task={})
|
||||
self.assertEqual(view['rankingDimensions'], ['product', 'group'])
|
||||
self.assertNotIn('topDimensions', view)
|
||||
self.assertNotIn('sourceFields', view)
|
||||
self.assertNotIn('investigation', view)
|
||||
self.assertEqual(view['fineDefects'], source['fineDefects'])
|
||||
self.assertEqual(compact_tool_result(source, task={'fullDetails': True})['sourceFields'], source['sourceFields'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import copy,json,sys,unittest
|
||||
from pathlib import Path
|
||||
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).parent)]
|
||||
from dsh_model_input import canonical_proposal,planning_input,adjacent_month_period,compact_tool_result
|
||||
from nao_native_exploration import prepare_step
|
||||
from nao_quality_tools import QualityToolError
|
||||
from test_native_exploration import root
|
||||
from test_nao_quality_tools import RELEASE
|
||||
|
||||
class SevenQuestionRegressions(unittest.TestCase):
|
||||
def test_captured_json_string_periods_are_equivalent_to_object_periods(self):
|
||||
task=root();task.update(period={'year':2026,'month':10},comparisonPeriod={'year':2026,'month':9})
|
||||
args={'mode':'compare','period':json.dumps(task['period']),'comparisonPeriod':json.dumps(task['comparisonPeriod'])}
|
||||
before=copy.deepcopy(args)
|
||||
actual=prepare_step(args,task,{'releaseId':RELEASE,'scopes':[]})
|
||||
expected=prepare_step({'mode':'compare'},task,{'releaseId':RELEASE,'scopes':[]})
|
||||
self.assertEqual(actual,expected);self.assertEqual(args,before)
|
||||
for value in ('{bad','[]','2026-10'):
|
||||
with self.assertRaises(QualityToolError):prepare_step({'mode':'compare','period':value},task,{'releaseId':RELEASE,'scopes':[]})
|
||||
def test_redundant_query_stage_uses_same_formal_query_without_widening_root(self):
|
||||
task=root();task.update(stages=['bright'],comparisonPeriod=None,visualType='donut')
|
||||
args,proposal=prepare_step({'mode':'query','stage':'bright'},task,{'releaseId':RELEASE,'scopes':[]})
|
||||
self.assertNotIn('stage',args);self.assertEqual(proposal['node']['stages'],['bright'])
|
||||
self.assertIsNone(proposal['node']['visualType'])
|
||||
with self.assertRaises(QualityToolError):prepare_step({'mode':'query','stage':'black'},task,{'releaseId':RELEASE,'scopes':[]})
|
||||
def test_company_department_ranking_cannot_invent_a_single_department(self):
|
||||
context={'_businessCatalog':{'departments':['工业电机产业部','消费电子产业部','新能源汽车磁钢']}}
|
||||
value={'kind':'analysis','filterUpdates':{'department':'工业电机产业部'},'dimensions':['group'],'stages':['bright']}
|
||||
fixed=canonical_proposal(value,question='生成2026年1—10月产业部亮片报废重量Top5对比简报,说明报废重量主要集中在哪里。',context=context)
|
||||
self.assertEqual(fixed['filterUpdates']['department'],None);self.assertEqual(fixed['dimensions'],['department'])
|
||||
explicit=canonical_proposal(value,question='生成工业电机产业部亮片报废重量Top5对比简报',context=context)
|
||||
self.assertEqual(explicit['filterUpdates']['department'],'工业电机产业部')
|
||||
def test_neighboring_months_are_part_of_the_same_requested_read_window(self):
|
||||
question='生成2026年5月质量异常诊断简报,说明当月表现、与前后月份的差异和可观察到的报废线索。数据不能证明的原因请直接说明。'
|
||||
value={'kind':'analysis','period':{'year':2026,'month':5},'comparisonRequired':False}
|
||||
fixed=canonical_proposal(value,question=question)
|
||||
self.assertEqual(fixed['period'],{'year':2026,'monthStart':4,'monthEnd':6})
|
||||
self.assertTrue(fixed['trendRequired']);self.assertFalse(fixed['comparisonRequired'])
|
||||
payload=planning_input({'question':question})
|
||||
self.assertEqual(payload['adjacentPeriodBinding']['focusPeriod'],{'year':2026,'month':5})
|
||||
self.assertEqual(value['period'],{'year':2026,'month':5})
|
||||
self.assertIsNone(adjacent_month_period('2026年5月总体表现'))
|
||||
def test_focus_month_and_complete_ranking_metadata_do_not_change_source_values(self):
|
||||
source={'kind':'quality_metrics','summary':{'black':{'scrapWeight':80}},'trend':[
|
||||
{'periodLabel':'2026-04','black':{'scrapWeight':20}},
|
||||
{'periodLabel':'2026-05','black':{'scrapWeight':35}},
|
||||
{'periodLabel':'2026-06','black':{'scrapWeight':25}}]}
|
||||
before=copy.deepcopy(source)
|
||||
projected=compact_tool_result(source,task={'question':'2026年5月诊断,与前后月份比较'})
|
||||
self.assertEqual(projected['focusMonth'],source['trend'][1]);self.assertEqual(source,before)
|
||||
rank=compact_tool_result({'kind':'quality_weight_ranking','dimension':'department','returnedRows':3,'truncated':False})
|
||||
self.assertTrue(rank['rankingDisplay']['complete']);self.assertEqual(rank['rankingDisplay']['returnedRows'],3)
|
||||
def test_fine_defect_top_does_not_inherit_a_report_composition_hint(self):
|
||||
task=root();task.update(stages=['bright'],visualType='pareto',comparisonPeriod=None)
|
||||
args,proposal=prepare_step({'mode':'defects','stage':'bright','limit':5},task,{'releaseId':RELEASE,'scopes':[]})
|
||||
self.assertEqual(proposal['node']['visualType'],'bar')
|
||||
self.assertEqual(args['limit'],5)
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,115 @@
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from analysis_visuals import build_visuals
|
||||
from nao_agent_task import bind_task
|
||||
from analysis_visual_evidence import protocol_visual_result
|
||||
|
||||
|
||||
def fixture(dimension='group', fields=None, stages=None, count=3):
|
||||
fields = fields or ['scrapWeight']
|
||||
stages = ['black', 'bright'] if stages is None else stages
|
||||
task = dict(version=1, kind='metrics', question='各对象指标排名', fields=fields,
|
||||
stages=stages, dimensions=[dimension], dimensionStages={dimension: stages},
|
||||
period={'year': 2026, 'month': 10}, filters={},
|
||||
visualPreference='auto', visualType=None, trendRequired=False,
|
||||
comparisonRequired=False)
|
||||
rows = [dict(key=f'item-{i:03}', label=f'对象{i}',
|
||||
black={'totalWeight': 100, 'scrapWeight': i + 1, 'rate': .99 - i / 100, 'scrapRate': .01 + i / 100},
|
||||
bright={'totalWeight': 200, 'scrapWeight': 2 * (i + 1), 'rate': .99 - i / 100, 'scrapRate': .01 + i / 100},
|
||||
compositeRate=.98 - i / 100, coverage={'blackRows': 1, 'brightRows': 1}) for i in range(count)]
|
||||
total = count * (count + 1) / 2
|
||||
result = dict(kind='quality_metrics', releaseId='release', release={'id': 'release'},
|
||||
view=dimension, period=task['period'], filters={}, rows=rows,
|
||||
rowCount=count, rowsReturned=count, rowsTruncated=False,
|
||||
summary={'black': {'totalWeight': count * 100, 'scrapWeight': total},
|
||||
'bright': {'totalWeight': count * 200, 'scrapWeight': 2 * total}},
|
||||
coverage={'sourceMonths': [{'year': 2026, 'month': 10, 'blackRows': count, 'brightRows': count}]})
|
||||
return task, {'tool': 'quality_data', 'mode': 'query', 'releaseId': 'release',
|
||||
'evidenceRef': 'Q1234ABCD', 'result': result}
|
||||
|
||||
|
||||
class GroupedQueryVisualTests(unittest.TestCase):
|
||||
def render(self, task, document):
|
||||
return build_visuals([document], task['question'], 'turn-grouped', context=bind_task({'releaseId': 'release'}, task))
|
||||
|
||||
def test_all_dimensions_and_stage_weights_use_the_ranking_converter(self):
|
||||
for dimension in ('department', 'group', 'product'):
|
||||
for field in ('totalWeight', 'scrapWeight'):
|
||||
with self.subTest(dimension=dimension, field=field):
|
||||
task, doc = fixture(dimension, [field])
|
||||
before = copy.deepcopy(doc)
|
||||
visuals = self.render(task, doc)['visuals']
|
||||
self.assertEqual(len(visuals), 2)
|
||||
self.assertEqual({v['stage'] for v in visuals}, {'black', 'bright'})
|
||||
self.assertEqual(visuals[0]['categories'][0]['key'], 'item-000' if field == 'totalWeight' else 'item-002')
|
||||
self.assertEqual(doc, before)
|
||||
|
||||
def test_single_stage_and_combined_scrap_keep_their_distinct_meanings(self):
|
||||
for stages, expected in ((['bright'], {'bright'}), ([], {None})):
|
||||
task, doc = fixture(stages=stages)
|
||||
visuals = self.render(task, doc)['visuals']
|
||||
self.assertEqual({v.get('stage') for v in visuals}, expected)
|
||||
|
||||
def test_full_top100_and_truncated_input(self):
|
||||
task, doc = fixture(count=123)
|
||||
charts = self.render(task, doc)['visuals']
|
||||
self.assertEqual(len(charts[0]['categories']), 100)
|
||||
self.assertEqual(charts[0]['coverage']['totalRows'], 123)
|
||||
doc['result']['rowsTruncated'] = True
|
||||
self.assertEqual(self.render(task, doc)['visuals'], [])
|
||||
|
||||
def test_query_top_count_controls_chart_rows_without_changing_query_data(self):
|
||||
task, doc = fixture(count=30)
|
||||
task['question'] = '各分组不合格重量Top20'
|
||||
charts = self.render(task, doc)['visuals']
|
||||
self.assertEqual([len(chart['categories']) for chart in charts], [20, 20])
|
||||
self.assertEqual(len(doc['result']['rows']), 30)
|
||||
|
||||
def test_published_rates_are_not_recalculated_and_bad_metrics_are_isolated(self):
|
||||
task, doc = fixture(fields=['rate', 'scrapWeight'], stages=['bright'])
|
||||
doc['result']['summary']['bright']['scrapWeight'] = -1
|
||||
visuals = self.render(task, doc)['visuals']
|
||||
rate = next(v for v in visuals if v['metric'] == 'bright_rate')
|
||||
self.assertEqual(rate['series'][0]['values'], [.99, .98, .97])
|
||||
self.assertFalse(any(v['metric'] == 'bright_scrap_weight' for v in visuals))
|
||||
|
||||
def test_no_charts_for_empty_data_or_explicit_none(self):
|
||||
task, doc = fixture(count=0)
|
||||
self.assertEqual(self.render(task, doc)['visuals'], [])
|
||||
task, doc = fixture()
|
||||
task['visualPreference'] = 'none'
|
||||
self.assertEqual(self.render(task, doc)['visuals'], [])
|
||||
|
||||
def test_each_dimension_uses_published_scrap_rates(self):
|
||||
for dimension in ('department', 'group', 'product'):
|
||||
task, doc = fixture(dimension, ['scrapRate'])
|
||||
charts = self.render(task, doc)['visuals']
|
||||
self.assertEqual({v['metric'] for v in charts}, {'black_scrap_rate', 'bright_scrap_rate'})
|
||||
self.assertEqual(charts[0]['series'][0]['values'], [.01, .02, .03])
|
||||
|
||||
def test_duplicate_rows_and_wrong_release_cannot_enter_charts(self):
|
||||
task, doc = fixture()
|
||||
doc['result']['rows'][1]['key'] = doc['result']['rows'][0]['key']
|
||||
self.assertEqual(self.render(task, doc)['visuals'], [])
|
||||
task, doc = fixture()
|
||||
doc['releaseId'] = doc['result']['releaseId'] = 'wrong-release'
|
||||
self.assertEqual(self.render(task, doc)['visuals'], [])
|
||||
|
||||
def test_broken_scope_does_not_remove_a_healthy_query_chart(self):
|
||||
task, doc = fixture()
|
||||
doc['result']['protocolEvidence'] = {'queryId': 'q1', 'nodeId': 'good', 'scopeId': 'good-scope'}
|
||||
scope = {'nodeId': 'good', 'scopeId': 'good-scope', 'purpose': 'metrics', 'task': task}
|
||||
broken = copy.deepcopy(scope)
|
||||
broken['task']['stages'] = 5
|
||||
broken['nodeId'] = 'broken'
|
||||
result = protocol_visual_result([doc], [broken, scope], task['question'], 'turn-grouped', release_id='release')
|
||||
self.assertEqual(result['visualState'], 'ready')
|
||||
self.assertEqual(len(result['visuals']), 2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1636,6 +1636,62 @@ class HistoryReportExecutionTest(unittest.TestCase):
|
||||
wrong_collective = _bind_literal_measures(wrong_collective, ranked_tokens, {}, facts=ranked)
|
||||
self.assertIn('1155.48340000、1106.25、1060.75', wrong_collective['blocks'][0]['text'])
|
||||
|
||||
def test_unknown_unit_rank_mass_requires_exact_local_metric_and_never_invents_unit(self):
|
||||
from nao_native_answer import (NativeCompositionError, _bind_exact_identifiers,
|
||||
_bind_exact_periods, _bind_exact_year_literals, _bind_literal_measures,
|
||||
_binding_inventory, _validate_blocks)
|
||||
|
||||
def ranked_fact(fact_id='R1', *, rank_group='history-rank:verified', source_turn='turn-1', value='36.00'):
|
||||
return {'id': fact_id, 'kind': 'product', 'label': '规格一',
|
||||
'text': f'2026年8月:规格一综合报废重量 = {value}。',
|
||||
'evidenceRef': 'history:' + fact_id,
|
||||
'sourceTurnId': source_turn, 'sectionIndex': 2, 'rank': 1,
|
||||
'rankGroupKey': rank_group,
|
||||
'scope': {'period': {'year': 2026, 'month': 8}, 'filters': {},
|
||||
'dimension': 'product', 'stage': 'overall'},
|
||||
'validity': {'verified': True, 'basis': 'protocol_history_snapshot'},
|
||||
'verifiedIdentifiers': [{'value': '规格一'}],
|
||||
'verifiedDisplayValues': [{'metricCode': 'scrapWeight', 'rawValue': value,
|
||||
'rawUnit': 'source_weight', 'unit': 'source_weight', 'displayValue': value}]}
|
||||
|
||||
fact = ranked_fact()
|
||||
facts = [fact]
|
||||
tokens = _binding_inventory(facts)
|
||||
|
||||
def bind(text, cited=None):
|
||||
draft = {'blocks': [{'text': text, 'factIds': cited or ['R1'], 'role': 'core'}]}
|
||||
for binder in (_bind_exact_identifiers, _bind_exact_periods, _bind_exact_year_literals):
|
||||
draft = binder(draft, tokens, {})
|
||||
return _bind_literal_measures(draft, tokens, {}, facts=facts)
|
||||
|
||||
accepted = bind('2026年8月规格一综合报废重量为36.000。')
|
||||
self.assertRegex(accepted['blocks'][0]['text'], r'\{\{N[1-9][0-9]*\}\}')
|
||||
rendered, _ = _validate_blocks(accepted, facts, tokens)
|
||||
self.assertIn('36.00', rendered[0]['text'])
|
||||
self.assertNotIn('kg', rendered[0]['text'])
|
||||
self.assertNotIn('source_weight', rendered[0]['text'])
|
||||
|
||||
for invalid in (
|
||||
'2026年8月规格一综合报废重量为36.000 kg。',
|
||||
'2026年8月规格一综合合格率97.19%,29。',
|
||||
):
|
||||
with self.subTest(invalid=invalid):
|
||||
draft = bind(invalid)
|
||||
self.assertNotIn('{{N', draft['blocks'][0]['text'])
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(draft, facts, tokens)
|
||||
|
||||
same_value_other_provenance = ranked_fact('R2', rank_group='history-rank:other',
|
||||
source_turn='turn-2')
|
||||
ambiguous_facts = [fact, same_value_other_provenance]
|
||||
ambiguous_tokens = _binding_inventory(ambiguous_facts)
|
||||
ambiguous = {'blocks': [{'text': '2026年8月规格一综合报废重量为36.00。',
|
||||
'factIds': ['R1', 'R2'], 'role': 'core'}]}
|
||||
for binder in (_bind_exact_identifiers, _bind_exact_periods, _bind_exact_year_literals):
|
||||
ambiguous = binder(ambiguous, ambiguous_tokens, {})
|
||||
ambiguous = _bind_literal_measures(ambiguous, ambiguous_tokens, {}, facts=ambiguous_facts)
|
||||
self.assertNotIn('{{N', ambiguous['blocks'][0]['text'])
|
||||
|
||||
def test_history_parent_predicate_mass_inheritance_uses_family_stage_and_exact_fact(self):
|
||||
from nao_native_answer import (NativeCompositionError, _bind_exact_identifiers,
|
||||
_bind_exact_periods, _bind_exact_year_literals, _binding_inventory, _validate_blocks)
|
||||
|
||||
@@ -96,6 +96,64 @@ class ModelGateTests(unittest.TestCase):
|
||||
self.assertEqual(self.request(model='different-model')[0], 400)
|
||||
self.assertEqual(len(self.calls), 0)
|
||||
|
||||
def test_dsh_keeps_only_its_bound_quality_tool_at_the_provider_boundary(self):
|
||||
self.gate.runtime_engine='DSH'
|
||||
names=['mcp__quality__quality_data','bash','mcp_call','story','mcp__other__quality_data']
|
||||
tools=[{'type':'function','function':{'name':name,'parameters':{'type':'object'}}} for name in names]
|
||||
self.assertEqual(self.request(extra={'tools':tools})[0],200)
|
||||
self.assertEqual([x['function']['name'] for x in self.calls[-1]['tools']],['mcp__quality__quality_data'])
|
||||
|
||||
def test_dsh_stream_separates_complete_calls_when_the_provider_reuses_an_index(self):
|
||||
from unittest.mock import Mock
|
||||
from nao_thinking_stream import ProviderStream
|
||||
self.gate.runtime_engine='DSH'
|
||||
name='mcp__quality__quality_data'
|
||||
tool={'type':'function','function':{'name':name,'parameters':{'type':'object'}}}
|
||||
frames=[{'choices':[{'index':0,'delta':{'tool_calls':[{'index':index,'id':identity,'type':'function',
|
||||
'function':{'name':name,'arguments':json.dumps({'probe':identity})}}]},'finish_reason':None}]}
|
||||
for index,identity in [(0,'a'),(1,'b'),(1,'c'),(2,'d')]]
|
||||
frames.append({'choices':[{'index':0,'delta':{},'finish_reason':'tool_calls'}]})
|
||||
chunks=[('data: '+json.dumps(frame)+'\n\n').encode() for frame in frames]+[b'data: [DONE]\n\n']
|
||||
upstream=HeldOpenProviderResponse(chunks);upstream.release.set()
|
||||
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
|
||||
code,body=self.request(extra={'stream':True,'tools':[tool]})
|
||||
self.assertEqual(code,200)
|
||||
observer=ProviderStream(lambda _:None);observer.feed(body.encode());observer.finish()
|
||||
calls=observer.completion()['choices'][0]['message']['tool_calls']
|
||||
self.assertEqual([call['id'] for call in calls],['a','b','c','d'])
|
||||
self.assertEqual([call['function']['name'] for call in calls],[name]*4)
|
||||
self.assertEqual([json.loads(call['function']['arguments']) for call in calls],[{'probe':x} for x in 'abcd'])
|
||||
|
||||
def test_revocation_interrupts_an_inflight_provider_read(self):
|
||||
from unittest.mock import Mock
|
||||
class StalledResponse:
|
||||
status=200
|
||||
headers={'Content-Type':'text/event-stream'}
|
||||
def __init__(self):
|
||||
self.reading=threading.Event();self.released=threading.Event()
|
||||
def __enter__(self):return self
|
||||
def __exit__(self,*_):self.close()
|
||||
def read1(self,_):
|
||||
self.reading.set();self.released.wait(5);return b''
|
||||
def close(self):self.released.set()
|
||||
stalled=StalledResponse();self.gate._opener=Mock()
|
||||
self.gate._opener.open.return_value=stalled
|
||||
done=threading.Event();result=[]
|
||||
def query():
|
||||
try:result.append(self.request())
|
||||
finally:done.set()
|
||||
worker=threading.Thread(target=query,daemon=True);worker.start()
|
||||
self.assertTrue(stalled.reading.wait(1))
|
||||
self.gate.revoke()
|
||||
self.assertTrue(done.wait(1),'revoked provider read was left waiting for its timeout')
|
||||
worker.join(1)
|
||||
# SSE headers may have been sent before revocation; HTTP 200 alone is
|
||||
# not a completion receipt. The interrupted body has no terminal event.
|
||||
self.assertIn(result[0][0],(200,502))
|
||||
self.assertNotIn('[DONE]',result[0][1])
|
||||
self.assertNotIn('"finish_reason": "stop"',result[0][1])
|
||||
self.assertEqual(self.request()[0],403)
|
||||
|
||||
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
|
||||
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
|
||||
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
|
||||
@@ -123,6 +181,14 @@ class ModelGateTests(unittest.TestCase):
|
||||
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
|
||||
code,body=self.request(extra={'tools':[data_tool]})
|
||||
self.assertEqual(code,502);self.assertNotIn('clarification',body)
|
||||
self.assertEqual(self.gate.rejected_tool_names,['clarification'])
|
||||
from types import SimpleNamespace
|
||||
from nao_model_gate import ToolPolicyError
|
||||
observer=SimpleNamespace(tools={0:{0:{'function':{'name':'secret=never-log','arguments':'private argument'}}}},
|
||||
choices={0:{'finish_reason':'tool_calls'}},done=True)
|
||||
with self.assertRaises(ToolPolicyError):
|
||||
self.gate.check_returned_tools(observer,{'mcp_call'},final=True)
|
||||
self.assertEqual(self.gate.rejected_tool_names,['clarification','<invalid_name>'])
|
||||
for newline in (True,False):
|
||||
prefix=b'data: {"choices":[{"index":0,"delta":{"content":"Preserved text"}}]}\n\n'
|
||||
bad=('data: '+json.dumps({'choices':[{'index':0,'delta':{'tool_calls':[tool]},'finish_reason':'tool_calls'}]})+ ('\n\n' if newline else '')).encode()
|
||||
@@ -215,6 +281,20 @@ class ModelGateTests(unittest.TestCase):
|
||||
self.assertNotIn('reasoning_effort',body)
|
||||
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
|
||||
|
||||
def test_mimo_disables_thinking_only_for_native_tool_requests(self):
|
||||
self.gate._model='mimo-v2.6-flash'
|
||||
self.gate.runtime_engine='DSH'
|
||||
self.gate._thinking={'enabled':True,'effort':'auto','protocol':'thinking'}
|
||||
tool={'type':'function','function':{'name':'mcp__quality__quality_data','parameters':{'type':'object'}}}
|
||||
for native,tools in ((False,[]),(True,[tool]),(True,[])):
|
||||
with self.subTest(native=native,tools=bool(tools)):
|
||||
self.gate.native_agent=native
|
||||
self.assertEqual(self.request(model=self.gate._model,extra={'tools':tools})[0],200)
|
||||
expected=not(native and tools)
|
||||
self.assertEqual(self.calls[-1]['thinking'],{'type':'enabled' if expected else 'disabled'})
|
||||
self.assertTrue(self.gate._thinking['enabled'])
|
||||
self.assertTrue(self.gate.request_audit[-1]['requestedEnabled'])
|
||||
|
||||
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
|
||||
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
|
||||
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
|
||||
|
||||
@@ -295,6 +295,54 @@ class RawPlanExecutionTests(unittest.TestCase):
|
||||
self.assertNotIn('recordList',result)
|
||||
self.assertEqual({doc['mode'] for doc in documents},{'investigate'})
|
||||
|
||||
def test_autonomous_source_and_records_only_plan_uses_finalizer_and_keeps_raw_coverage(self):
|
||||
task=self.root_task(purposes=('source','records'))
|
||||
api=InvestigationApi(total_rows=1)
|
||||
for purpose in ('source','records'):
|
||||
api.add_scope(purpose,purpose+'-scope',purpose,task)
|
||||
captured={}
|
||||
def synthesize_stub(**kwargs):
|
||||
captured['evidence']=copy.deepcopy(kwargs['evidence'])
|
||||
captured['question']=kwargs['question']
|
||||
return {'answer':'仅确认了原表核验,原问题仍缺少正式分析依据。',
|
||||
'evidenceSufficient':False,'missingEvidence':['原问题所需正式分析依据未取得。'],
|
||||
'modelDeclaredEvidenceSufficient':None,'usage':{},'finishReason':'stop',
|
||||
'selectedFactIds':[]}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize',side_effect=synthesize_stub) as synth:
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
|
||||
'question':task['question'],'_autonomousAnswer':True},task=task,
|
||||
authority={**identity(),'task':task},actor_root=root,delegated=api,
|
||||
connection=('http://offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertTrue(synth.called)
|
||||
source_docs=[doc for doc in captured['evidence'] if doc.get('mode')=='investigate'
|
||||
and (doc.get('result') or {}).get('action')=='source']
|
||||
self.assertEqual(len(source_docs),1)
|
||||
self.assertFalse(any((doc.get('result') or {}).get('action')=='rows'
|
||||
for doc in captured['evidence']))
|
||||
self.assertEqual(source_docs[0]['result']['current']['sourceVerification']['verifiedRows'],1)
|
||||
self.assertIn('已按稳定分页读全1条',result['answer'])
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertIn('原问题所需正式分析依据未取得。',result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
def test_autonomous_records_only_plan_stays_deterministic(self):
|
||||
task=self.root_task(purposes=('records',))
|
||||
api=InvestigationApi(total_rows=1)
|
||||
api.add_scope('records','records-scope','records',task)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize') as synth:
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
|
||||
'question':task['question'],'_autonomousAnswer':True},task=task,
|
||||
authority={**identity(),'task':task},actor_root=root,delegated=api,
|
||||
connection=('http://offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
self.assertFalse(synth.called)
|
||||
self.assertEqual(result['status'],'COMPLETED')
|
||||
self.assertIn('已按本轮服务器授权范围核对原始数据。',result['answer'])
|
||||
|
||||
def test_unselected_column_filters_accept_canonical_empty_context_echo(self):
|
||||
api=InvestigationApi(total_rows=2)
|
||||
result,_,_=self.run_query(api,with_column_filters=False)
|
||||
@@ -468,6 +516,45 @@ class MixedRawPlanExecutionTests(unittest.TestCase):
|
||||
self.assertIn('原表值与存储记录一致',result['answer'])
|
||||
self.assertNotIn('PRIVATE_RAW_VALUE',result['answer'])
|
||||
|
||||
def test_autonomous_mixed_synthesis_receives_source_proof_without_record_rows(self):
|
||||
plan=[
|
||||
{'nodeId':'records','purpose':'records','fields':[],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
{'nodeId':'verify','purpose':'source','fields':[],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
{'nodeId':'metric','purpose':'metrics','fields':['scrapRate'],'stages':['bright'],
|
||||
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
|
||||
'comparisonRequired':False},
|
||||
]
|
||||
value=task_value(kind='details',dimensions=[],stages=['bright'],fields=['scrapRate'],
|
||||
verifySource=True,fullDetails=True,period={'year':2026,'month':9},queryPlan=plan,
|
||||
visualPreference='none',visualType=None,comparisonRequired=False,trendRequired=False)
|
||||
task=normalize_task(value,'查询亮片报废率,并核验原表及读取明细',{'_protocol':{'version':1}})
|
||||
api=MixedRawMetricsApi(task,task['queryPlan'])
|
||||
captured={}
|
||||
def synthesize_stub(**kwargs):
|
||||
captured['evidence']=copy.deepcopy(kwargs['evidence'])
|
||||
return {'answer':'已基于正式指标和原表核验完成说明。','evidenceSufficient':True,
|
||||
'missingEvidence':[],'modelDeclaredEvidenceSufficient':True,'usage':{},
|
||||
'finishReason':'stop','selectedFactIds':[]}
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
|
||||
with patch('nao_evidence_finalizer.synthesize',side_effect=synthesize_stub):
|
||||
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':task['question'],
|
||||
'_autonomousAnswer':True},task=task,authority={**identity(),'task':task},
|
||||
actor_root=root,delegated=api,connection=('http://offline','synthetic','model'),
|
||||
cancel_event=threading.Event(),emit=lambda _:None)
|
||||
docs=captured['evidence']
|
||||
self.assertTrue(any(doc.get('mode')=='query' for doc in docs))
|
||||
source_docs=[doc for doc in docs if doc.get('mode')=='investigate'
|
||||
and (doc.get('result') or {}).get('action')=='source']
|
||||
self.assertTrue(source_docs)
|
||||
self.assertFalse(any((doc.get('result') or {}).get('action')=='rows' for doc in docs))
|
||||
self.assertNotIn('PRIVATE_RAW_VALUE',json.dumps(docs,ensure_ascii=False))
|
||||
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
"""Authored SSE exercises the real observer and installed SDK without network I/O."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from nao_thinking_stream import ProviderStream
|
||||
from nao_tool_stream import ToolStreamNormalizer, ToolStreamProtocolError
|
||||
|
||||
TOOL = 'mcp__quality__quality_data'
|
||||
|
||||
|
||||
def frame(*calls, choice=0, finish=None, **delta):
|
||||
if calls:
|
||||
delta['tool_calls'] = list(calls)
|
||||
return {'choices': [{'index': choice, 'delta': delta, 'finish_reason': finish}]}
|
||||
|
||||
|
||||
def call(index, identity=None, arguments=None, name=None):
|
||||
value = {'index': index}
|
||||
if identity is not None:
|
||||
value.update(id=identity, type='function')
|
||||
function = {}
|
||||
if name is not None:
|
||||
function['name'] = name
|
||||
if arguments is not None:
|
||||
function['arguments'] = arguments
|
||||
if function:
|
||||
value['function'] = function
|
||||
return value
|
||||
|
||||
|
||||
def wire(rows, *, done=True, newline=b'\n'):
|
||||
result = b''.join(b'data: ' + json.dumps(row, ensure_ascii=False).encode() + newline * 2 for row in rows)
|
||||
return result + (b'data: [DONE]' + newline * 2 if done else b'')
|
||||
|
||||
|
||||
def collision_rows():
|
||||
return [frame(call(0, 'a', '{"mode":"catalog"}', TOOL)),
|
||||
frame(call(1, 'b', '{"mode":"compare"}', TOOL)),
|
||||
frame(call(1, 'c', '{"mode":', TOOL)),
|
||||
frame(call(1, arguments='"changes"}')),
|
||||
frame(call(2, 'd', '{"mode":"metrics"}', TOOL)),
|
||||
frame(finish='tool_calls')]
|
||||
|
||||
|
||||
class ToolStreamTests(unittest.TestCase):
|
||||
def normalize(self, data, pieces=None):
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
output = b''
|
||||
for piece in pieces if pieces is not None else [data]:
|
||||
output += normalizer.feed(piece)
|
||||
return output + normalizer.finish()
|
||||
|
||||
def completion(self, data):
|
||||
observer = ProviderStream(lambda _text: None)
|
||||
observer.feed(data)
|
||||
observer.finish()
|
||||
return observer.completion()['choices']
|
||||
|
||||
def test_raw_collision_is_a_red_capable_real_observer_fixture(self):
|
||||
tools = self.completion(wire(collision_rows()))[0]['message']['tool_calls']
|
||||
self.assertEqual(len(tools), 3)
|
||||
self.assertEqual(tools[1]['function']['name'], TOOL + TOOL)
|
||||
with self.assertRaises(ValueError):
|
||||
json.loads(tools[1]['function']['arguments'])
|
||||
|
||||
def test_collision_preserves_three_calls_and_following_wire_index_cannot_collide(self):
|
||||
result = self.completion(self.normalize(wire(collision_rows())))[0]['message']['tool_calls']
|
||||
self.assertEqual([value['id'] for value in result], ['a', 'b', 'c', 'd'])
|
||||
self.assertEqual([value['function']['name'] for value in result], [TOOL] * 4)
|
||||
self.assertEqual([json.loads(value['function']['arguments'])['mode'] for value in result],
|
||||
['catalog', 'compare', 'changes', 'metrics'])
|
||||
rows = [json.loads(line[5:]) for line in self.normalize(wire(collision_rows())).splitlines()
|
||||
if line.startswith(b'data:') and b'[DONE]' not in line]
|
||||
self.assertEqual([item['index'] for row in rows for choice in row['choices']
|
||||
for item in choice['delta'].get('tool_calls', [])], [0, 1, 2, 2, 3])
|
||||
|
||||
def test_legal_parallel_interleaved_fragments_preserve_exact_wire(self):
|
||||
rows = [frame(call(0, 'a', '{"value":', TOOL), call(1, 'b', '{"value":', TOOL)),
|
||||
frame(call(1, arguments='"亮片🙂"}')),
|
||||
frame(call(0, arguments='"黑片"}')), frame(finish='tool_calls')]
|
||||
data = wire(rows, newline=b'\r\n')
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
result = self.completion(data)[0]['message']['tool_calls']
|
||||
self.assertEqual([json.loads(item['function']['arguments'])['value'] for item in result], ['黑片', '亮片🙂'])
|
||||
|
||||
def test_every_byte_boundary_and_headers_heartbeats_are_preserved(self):
|
||||
data = b': heartbeat\n\nevent: message\n' + wire(collision_rows())
|
||||
expected = self.normalize(data)
|
||||
self.assertEqual(self.normalize(data, [bytes([byte]) for byte in data]), expected)
|
||||
self.assertTrue(expected.startswith(b': heartbeat\n\nevent: message\n'))
|
||||
|
||||
def test_reindexed_frame_preserves_name_id_argument_string_and_other_fields(self):
|
||||
arguments = ' {"quoted":"a\\\"b","path":"x\\\\y","unicode":"亮片🙂"} '
|
||||
original = frame(call(0, 'identity/亮片', arguments, TOOL), content='原文🙂')
|
||||
original.update(id='completion-original', extra={'unchanged': True})
|
||||
data = wire([frame(call(0, 'first', '{}', TOOL)), original, frame(finish='tool_calls')])
|
||||
normalized = self.normalize(data)
|
||||
rows = [json.loads(line[5:]) for line in normalized.splitlines()
|
||||
if line.startswith(b'data:') and b'[DONE]' not in line]
|
||||
expected = json.loads(json.dumps(original))
|
||||
expected['choices'][0]['delta']['tool_calls'][0]['index'] = 1
|
||||
self.assertEqual(rows[1], expected)
|
||||
self.assertEqual(rows[1]['choices'][0]['delta']['tool_calls'][0]['function']['arguments'], arguments)
|
||||
|
||||
def test_feed_returns_valid_events_before_the_rest_of_the_stream_arrives(self):
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
first = wire([frame(call(0, 'a', '{"mode":', TOOL))], done=False)
|
||||
self.assertEqual(normalizer.feed(first), first)
|
||||
final = wire([frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')])
|
||||
self.assertEqual(normalizer.feed(final) + normalizer.finish(), final)
|
||||
|
||||
def test_fragmented_name_and_delayed_initial_id_remain_one_call(self):
|
||||
rows = [frame(call(0, arguments='{', name='mcp__quality__')),
|
||||
frame(call(0, 'a', arguments='"mode":', name='quality_data')),
|
||||
frame(call(0, arguments='"catalog"}')), frame(finish='tool_calls')]
|
||||
data = wire(rows)
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
self.assertEqual(self.completion(data)[0]['message']['tool_calls'][0]['id'], 'a')
|
||||
|
||||
def test_message_snapshot_and_non_sse_json_are_not_rewritten(self):
|
||||
snapshot = {'choices': [{'index': 0, 'message': {'tool_calls': [call(1, 'a', '{}', TOOL),
|
||||
call(1, 'b', '{}', TOOL)]},
|
||||
'finish_reason': 'tool_calls'}]}
|
||||
data = wire([snapshot])
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
ordinary = json.dumps(snapshot).encode()
|
||||
self.assertEqual(self.normalize(ordinary), ordinary)
|
||||
|
||||
def test_choices_have_independent_index_and_id_namespaces(self):
|
||||
rows = [frame(call(0, 'a', '{}', TOOL), choice=0), frame(call(0, 'a', '{}', TOOL), choice=1),
|
||||
frame(call(0, 'b', '{}', TOOL), choice=0), frame(call(1, 'b', '{}', TOOL), choice=1),
|
||||
frame(choice=0, finish='tool_calls'), frame(choice=1, finish='tool_calls')]
|
||||
choices = self.completion(self.normalize(wire(rows)))
|
||||
self.assertEqual([[item['id'] for item in choice['message']['tool_calls']] for choice in choices],
|
||||
[['a', 'b'], ['a', 'b']])
|
||||
|
||||
def test_repeated_active_id_continues_but_repeated_name_is_not_silently_deduplicated(self):
|
||||
data = wire([frame(call(0, 'a', '{"mode":', TOOL)),
|
||||
frame(call(0, 'a', '"catalog"}')), frame(finish='tool_calls')])
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'a', '', TOOL))])
|
||||
|
||||
def reject(self, rows, *, code=None, done=False):
|
||||
with self.assertRaises(ToolStreamProtocolError) as caught:
|
||||
self.normalize(wire(rows, done=done))
|
||||
if code:
|
||||
self.assertEqual(caught.exception.code, code)
|
||||
self.assertNotIn('sensitive-fixture', str(caught.exception))
|
||||
|
||||
def test_argument_errors_are_preserved_for_sdk_without_aborting_later_calls(self):
|
||||
for old in ('', '{"mode":', '[]', 'null', '42', '{}{}', '{"x":NaN}', '{"x":Infinity}',
|
||||
'{"x":1,"x":2}'):
|
||||
with self.subTest(old=old):
|
||||
result = self.completion(self.normalize(wire([
|
||||
frame(call(0, 'a', old, TOOL)), frame(call(0, 'b', '{}', TOOL)),
|
||||
frame(finish='tool_calls')])))[0]['message']['tool_calls']
|
||||
self.assertEqual([item['id'] for item in result], ['a', 'b'])
|
||||
self.assertEqual([item['function']['arguments'] for item in result], [old, '{}'])
|
||||
|
||||
def test_reuse_inside_one_event_keeps_each_original_call(self):
|
||||
data = wire([frame(call(0, 'a', '{}', TOOL), call(0, 'b', '{}', TOOL), call(0, 'c', '{}', TOOL)),
|
||||
frame(finish='tool_calls')])
|
||||
result = self.completion(self.normalize(data))[0]['message']['tool_calls']
|
||||
self.assertEqual([item['id'] for item in result], ['a', 'b', 'c'])
|
||||
|
||||
def test_retired_call_is_not_recovered_from_its_complete_arguments_or_name(self):
|
||||
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"x":', TOOL)),
|
||||
frame(call(0, 'a', '1}'))], code='reused_call_id')
|
||||
|
||||
def test_malformed_frame_and_invalid_tool_payload_do_not_leak_input(self):
|
||||
for data in (b'data: {sensitive-fixture}\n\n', b'data: []\n\n',
|
||||
b'data: {"choices":[],"choices":[]}\n\n'):
|
||||
with self.assertRaises(ToolStreamProtocolError) as caught:
|
||||
self.normalize(data)
|
||||
self.assertNotIn('sensitive-fixture', str(caught.exception))
|
||||
for changed in (call(0, 'a', '{}', TOOL) | {'type': 'custom'},
|
||||
call(0, 'a', '{}', TOOL) | {'function': {'name': TOOL, 'arguments': {}}}):
|
||||
self.reject([frame(changed)])
|
||||
|
||||
def test_partial_name_never_authorizes_a_rotation(self):
|
||||
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(call(0, 'b', '{}', TOOL))],
|
||||
code='incomplete_previous_call')
|
||||
|
||||
def test_unknown_tool_prefix_and_final_incomplete_name_fail_closed(self):
|
||||
self.reject([frame(call(0, 'a', '{}', 'sensitive-fixture'))], code='unadvertised_tool')
|
||||
self.reject([frame(call(0, 'a', '{}', 'mcp__quality__')), frame(finish='tool_calls')],
|
||||
code='unadvertised_tool')
|
||||
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', 'shell'))],
|
||||
code='unadvertised_tool')
|
||||
|
||||
def test_retired_id_cannot_return_or_move_to_another_wire_index(self):
|
||||
first = [frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{}', TOOL))]
|
||||
for last in (call(0, 'a', '', None), call(1, 'a', '{}', TOOL), call(1, 'b', '{}', TOOL)):
|
||||
with self.subTest(last=last):
|
||||
self.reject(first + [frame(last)], code='reused_call_id')
|
||||
|
||||
def test_empty_or_invalid_id_and_missing_or_invalid_index_fail_closed(self):
|
||||
base = call(0, 'a', '{}', TOOL)
|
||||
for changed in ({**base, 'id': ''}, {**base, 'id': 3}, {**base, 'index': True},
|
||||
{**base, 'index': -1}, {**base, 'index': 0.5},
|
||||
{key: value for key, value in base.items() if key != 'index'}):
|
||||
with self.subTest(changed=changed):
|
||||
self.reject([frame(changed)])
|
||||
|
||||
def test_mimo_null_identity_and_name_are_continuations_of_an_identified_call(self):
|
||||
rows = [frame(call(0, 'call_mimo', '', TOOL)),
|
||||
frame({'index': 0, 'id': None, 'type': 'function',
|
||||
'function': {'name': None, 'arguments': '{"mode": '}}),
|
||||
frame({'index': 0, 'id': None, 'type': 'function',
|
||||
'function': {'name': None, 'arguments': '"catalog"}'}}),
|
||||
frame(finish='tool_calls')]
|
||||
result = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0]['id'], 'call_mimo')
|
||||
self.assertEqual(result[0]['function']['name'], TOOL)
|
||||
self.assertEqual(json.loads(result[0]['function']['arguments']), {'mode': 'catalog'})
|
||||
|
||||
def test_mimo_observed_namespace_alias_keeps_call_identity_and_arguments(self):
|
||||
# Captured with the real task input: MiMo Flash omitted only "mcp__".
|
||||
arguments = '{"mode": "query"}'
|
||||
for fragments in (['quality__quality_data'], ['quality__', 'quality_data'],
|
||||
['q', 'uality__quality_data']):
|
||||
with self.subTest(fragments=fragments):
|
||||
rows = [frame(call(0, 'mimo_alias', '', fragments[0]))]
|
||||
rows += [frame(call(0, name=part)) for part in fragments[1:]]
|
||||
rows += [frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': arguments}}),
|
||||
frame(finish='tool_calls')]
|
||||
tools = self.completion(self.normalize(wire(rows)))[0]['message']['tool_calls']
|
||||
self.assertEqual(len(tools), 1)
|
||||
self.assertEqual(tools[0]['id'], 'mimo_alias')
|
||||
self.assertEqual(tools[0]['function'], {'name': TOOL, 'arguments': arguments})
|
||||
|
||||
def test_namespace_alias_only_resolves_to_an_offered_tool(self):
|
||||
for name in ('quality_data', 'quality__shell', 'quality__quality_data_extra',
|
||||
'functions.mcp__quality__quality_data'):
|
||||
self.reject([frame(call(0, 'a', '{}', name))], code='unadvertised_tool')
|
||||
normalizer = ToolStreamNormalizer({'other_tool'})
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.feed(wire([frame(call(0, 'a', '{}', 'quality__quality_data'))]))
|
||||
|
||||
def test_null_identity_cannot_start_a_call_or_change_its_tool(self):
|
||||
self.reject([frame({'index': 0, 'id': None, 'function': {'name': TOOL, 'arguments': '{}'}})],
|
||||
code='invalid_call_id')
|
||||
self.reject([frame(call(0, 'a', '', TOOL)),
|
||||
frame({'index': 0, 'id': None, 'function': {'name': 'shell', 'arguments': '{}'}})],
|
||||
code='unadvertised_tool')
|
||||
|
||||
def test_calls_after_choice_finish_or_done_fail_closed(self):
|
||||
self.reject([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
|
||||
frame(call(1, 'b', '{}', TOOL))], code='call_after_finish')
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls')]))
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.feed(wire([frame(call(1, 'b', '{}', TOOL))]))
|
||||
|
||||
def test_missing_finish_and_failed_normalizer_cannot_be_resumed(self):
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
normalizer.feed(wire([frame(call(0, 'a', '{}', TOOL))], done=False))
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.finish()
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.feed(wire([frame(finish='tool_calls')]))
|
||||
|
||||
def test_final_line_without_newline_and_usage_tail_are_preserved(self):
|
||||
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(finish='tool_calls'),
|
||||
{'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2}}]).rstrip(b'\n')
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
|
||||
def test_sparse_wire_indices_remain_unchanged_until_they_would_collide(self):
|
||||
data = wire([frame(call(4, 'a', '{}', TOOL)), frame(call(9, 'b', '{}', TOOL)), frame(finish='tool_calls')])
|
||||
self.assertEqual(self.normalize(data), data)
|
||||
|
||||
def test_one_failed_feed_cannot_return_an_earlier_valid_line_or_be_resumed(self):
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
data = wire([frame(call(0, 'a', '{}', TOOL)), frame(call(0, 'b', '{"sensitive-fixture":', 'shell'))], done=False)
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.feed(data)
|
||||
with self.assertRaises(ToolStreamProtocolError):
|
||||
normalizer.finish()
|
||||
|
||||
def test_oversized_unterminated_frame_fails_closed(self):
|
||||
normalizer = ToolStreamNormalizer({TOOL})
|
||||
normalizer.MAX_FRAME_BYTES = 32
|
||||
with self.assertRaises(ToolStreamProtocolError) as caught:
|
||||
normalizer.feed(b'data: ' + b'x' * 33)
|
||||
self.assertEqual(caught.exception.code, 'frame_too_large')
|
||||
|
||||
|
||||
@unittest.skipUnless((ROOT / 'runtime/dsh/bin/node').is_file()
|
||||
and (ROOT / 'runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js').is_file(),
|
||||
'requires the installed pinned DSH SDK')
|
||||
class InstalledSdkTests(unittest.TestCase):
|
||||
def sdk(self, data):
|
||||
script = r'''
|
||||
import fs from 'node:fs';
|
||||
import { stream } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/api/openai-completions.js';
|
||||
import { normalizeContext } from './runtime/dsh/node_modules/@earendil-works/pi-ai/dist/utils/transcript.js';
|
||||
const wire = fs.readFileSync(0, 'utf8');
|
||||
const model = {id:'m', name:'m', api:'openai-completions', provider:'test', baseUrl:'http://offline.invalid/v1',
|
||||
reasoning:false, input:['text'], cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:8192,maxTokens:1024};
|
||||
const context = normalizeContext({messages:[{role:'user',content:'authored offline fixture',timestamp:0}]});
|
||||
let fetches = 0;
|
||||
const events = stream(model,context,{apiKey:'authored-no-network',fetch:async()=>{
|
||||
fetches++; return new Response(wire,{status:200,headers:{'Content-Type':'text/event-stream'}});
|
||||
}});
|
||||
const calls = []; let terminal;
|
||||
for await (const event of events) {
|
||||
if (event.type === 'toolcall_end') calls.push({id:event.toolCall.id,name:event.toolCall.name,
|
||||
arguments:event.toolCall.arguments,invalid:event.toolCall.argumentsError !== undefined});
|
||||
if (event.type === 'done' || event.type === 'error') terminal = event.type;
|
||||
}
|
||||
console.log(JSON.stringify({calls,terminal,fetches}));
|
||||
'''
|
||||
result = subprocess.run([str(ROOT / 'runtime/dsh/bin/node'), '--input-type=module', '-e', script],
|
||||
cwd=ROOT, input=data, capture_output=True, timeout=15, check=True)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
def test_real_sdk_raw_collision_fails_and_normalized_calls_keep_ids_and_arguments(self):
|
||||
data = wire(collision_rows())
|
||||
bad = self.sdk(data)
|
||||
self.assertEqual(len(bad['calls']), 3)
|
||||
self.assertTrue(bad['calls'][1]['invalid'])
|
||||
good = self.sdk(ToolStreamTests().normalize(data))
|
||||
self.assertEqual(good['terminal'], 'done')
|
||||
self.assertEqual(good['fetches'], 1)
|
||||
self.assertEqual([value['id'] for value in good['calls']], ['a', 'b', 'c', 'd'])
|
||||
self.assertTrue(all(value['name'] == TOOL and not value['invalid'] for value in good['calls']))
|
||||
self.assertEqual([value['arguments']['mode'] for value in good['calls']],
|
||||
['catalog', 'compare', 'changes', 'metrics'])
|
||||
|
||||
def test_real_sdk_still_rejects_a_final_incomplete_argument_object(self):
|
||||
data = wire([frame(call(0, 'a', '{"mode":', TOOL)), frame(finish='tool_calls')])
|
||||
result = self.sdk(ToolStreamTests().normalize(data))
|
||||
self.assertTrue(result['calls'][0]['invalid'])
|
||||
|
||||
def test_real_sdk_keeps_argument_error_and_subsequent_independent_call(self):
|
||||
data = wire([frame(call(0, 'bad', '{"mode":', TOOL)),
|
||||
frame(call(0, 'next', '{"mode":"query"}', TOOL)), frame(finish='tool_calls')])
|
||||
result = self.sdk(ToolStreamTests().normalize(data))
|
||||
self.assertEqual(result['terminal'], 'done')
|
||||
self.assertEqual([item['id'] for item in result['calls']], ['bad', 'next'])
|
||||
self.assertTrue(result['calls'][0]['invalid'])
|
||||
self.assertFalse(result['calls'][1]['invalid'])
|
||||
self.assertEqual(result['calls'][1]['arguments'], {'mode': 'query'})
|
||||
|
||||
def test_real_sdk_receives_the_advertised_name_for_observed_mimo_alias(self):
|
||||
data = wire([frame(call(0, 'mimo_alias', '', 'quality__quality_data')),
|
||||
frame({'index': 0, 'id': None, 'function': {'name': None, 'arguments': '{"mode":"query"}'}}),
|
||||
frame(finish='tool_calls')])
|
||||
result = self.sdk(ToolStreamTests().normalize(data))
|
||||
self.assertEqual(result['terminal'], 'done')
|
||||
self.assertEqual(result['calls'], [{'id': 'mimo_alias', 'name': TOOL,
|
||||
'arguments': {'mode': 'query'}, 'invalid': False}])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import copy
|
||||
import sys
|
||||
import unittest
|
||||
@@ -28,6 +29,27 @@ def bundle(value,queries):
|
||||
|
||||
|
||||
class VisualRecoveryTests(unittest.TestCase):
|
||||
def test_native_product_ranking_recovers_from_exact_frozen_batch_without_queryplan(self):
|
||||
value=task();value.update(executionMode='native_agent',kind='analysis',trendRequired=False,
|
||||
dimensions=['product'],stages=['black'],fields=['rate','totalWeight'])
|
||||
value['aggregateSelection']={'dimension':'product','metric':'rate','rateStage':None,'direction':'asc',
|
||||
'limit':100,'requestQuestion':'筛选合格率最低Top100产品并生成报告',
|
||||
'weightFilter':{'metric':'totalWeight','stage':'black','operator':'gt','value':1.0}}
|
||||
raw={**value,'_view':'product'};saved=query(raw,BoundRankingApi())
|
||||
for row in saved['response']['rows']:row['compositeRate']=.9
|
||||
scope={'nodeId':'product-batch','scopeId':'scope-products','purpose':'metrics','limit':500,'task':{**value,'kind':'metrics'}}
|
||||
for key in ('nodeId','scopeId'):saved[key]=saved['proof'][key]=scope[key]
|
||||
request=bundle(value,[saved]);request['scopes']=[scope]
|
||||
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
|
||||
result=rebuild(request)
|
||||
self.assertEqual(result['visualState'],'ready')
|
||||
self.assertTrue(result['visuals'])
|
||||
self.assertEqual(result['visuals'][0]['dimension'],'product')
|
||||
self.assertEqual(result['queryRefs'][0]['queryIds'],[saved['queryId']])
|
||||
from decimal import Decimal
|
||||
decimal_wire = json.loads(json.dumps(request), parse_float=Decimal)
|
||||
self.assertEqual(rebuild(decimal_wire)['visualState'], 'ready')
|
||||
|
||||
def test_trend_is_rebuilt_from_only_the_frozen_response_without_api_or_model(self):
|
||||
value=task({'product':'18X15X4+T'});saved=query(value);original=copy.deepcopy(saved)
|
||||
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -12,7 +13,7 @@ from urllib.error import HTTPError
|
||||
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
||||
from nao_evidence_finalizer import synthesize_native
|
||||
from nao_native_answer import build_native_report, numeric_inventory, _validate_blocks, _validate_summary_selection, _within_filters, _bind_exact_year_literals, _select_fact_pool, _binding_inventory, _bind_exact_identifiers, _bind_exact_periods, _bind_literal_measures, _metric_semantics, NativeCompositionError
|
||||
from nao_native_answer import build_native_report, numeric_inventory, _validate_blocks, _validate_summary_selection, _within_filters, _bind_exact_year_literals, _select_fact_pool, _binding_inventory, _bind_exact_identifiers, _bind_exact_periods, _bind_literal_measures, _metric_semantics, NativeCompositionError, synthesize_native_answer
|
||||
from nao_native_answer import _affirmative_production_cause, _source_relation_gap, _review_supported_remainder
|
||||
from test_agent_answer import evidence
|
||||
|
||||
@@ -65,6 +66,514 @@ def token(payload, value, fact_id='F1'):
|
||||
|
||||
|
||||
class NativeAnswerTests(unittest.TestCase):
|
||||
def test_stage_scrap_total_cannot_bind_to_a_named_formal_category(self):
|
||||
fact = {'id':'F1','kind':'stage','label':'黑片正式指标',
|
||||
'text':'2026年9月黑片报废重量14.00 kg。','evidenceRef':'Q00000001',
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'14','displayValue':'14.00',
|
||||
'unit':'kg','metricField':'scrapWeight','metricRole':'scrapWeight','stageHint':'黑片'}]}
|
||||
tokens = _binding_inventory([fact])
|
||||
for category in ('责废','料废'):
|
||||
text = f'正式{category}14.00 kg。'
|
||||
parsed = _bind_literal_measures({'blocks':[{'text':text,'factIds':['F1']}]},tokens,{},facts=[fact])
|
||||
with self.subTest(category=category):
|
||||
self.assertEqual(parsed['blocks'][0]['text'],text)
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(parsed,[fact],tokens)
|
||||
self.assertIsNotNone(_source_relation_gap({'text':text,'factIds':['F1']},[fact]))
|
||||
|
||||
def test_source_proof_projects_category_total_and_missing_selected_field_separately(self):
|
||||
period = {'year':2026,'month':9}
|
||||
result = {'kind':'quality_investigation','action':'source','releaseId':'release-1',
|
||||
'period':period,'filters':{},'stage':'black',
|
||||
'current':{'context':{'releaseId':'release-1','stage':'black','year':2026,'month':9},
|
||||
'totalRows':1,'aggregateComplete':True,'selectedFields':['责废尺寸重量'],
|
||||
'categories':[{'category':'责废','formalWeight':'7',
|
||||
'formalCoverage':{'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
|
||||
'invalidNumericRows':0,'negativeRows':0},
|
||||
'fields':[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':0,
|
||||
'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
|
||||
'negativeRows':0,'numericSum':'3'}}]}],
|
||||
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,
|
||||
'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0,
|
||||
'files':[{'fields':[{'field':'责废尺寸重量','sourceCoverage':{'rows':1,
|
||||
'numericRows':0,'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
|
||||
'negativeRows':0,'numericSum':'3'}}]}]}},
|
||||
'comparisonPeriod':None}
|
||||
document = {'releaseId':'release-1','evidenceRef':'Q00000001','mode':'investigate','result':result}
|
||||
context = {'releaseId':'release-1','period':period,'filters':{},
|
||||
'_agentTask':{'kind':'analysis','period':period,'comparisonPeriod':None,
|
||||
'filters':{},'stages':['black'],'executionMode':'native_agent'}}
|
||||
report = build_native_report([document],context,'核对责废原件和正式分量')
|
||||
category = next(fact for fact in report['facts'] if fact.get('kind') == 'category')
|
||||
missing = next(fact for fact in report['facts'] if fact.get('kind') == 'source_coverage')
|
||||
self.assertIn('责废',category['text'])
|
||||
self.assertIn('7.00 kg',category['text'])
|
||||
self.assertIn('空值',missing['text'])
|
||||
self.assertIn('不能按零处理',missing['text'])
|
||||
self.assertNotIn('3',missing['text'])
|
||||
typed = category['verifiedNativeNumericValues'][0]
|
||||
self.assertEqual(typed['metricField'],'categoryWeight')
|
||||
self.assertEqual(typed['categoryHint'],'责废')
|
||||
source = next(fact for fact in report['facts'] if fact.get('kind') == 'source')
|
||||
reconciliation = next(fact for fact in report['facts'] if fact.get('kind') == 'source_reconciliation')
|
||||
self.assertIn('不证明正式类别与来源细项已对账', source['text'])
|
||||
self.assertIn('未完整对账', reconciliation['text'])
|
||||
self.assertFalse(reconciliation['_nativeSourceCategoryReconciliation']['reconciled'])
|
||||
source_metrics = {value['metricField'] for value in source['verifiedNativeNumericValues']}
|
||||
self.assertEqual(source_metrics, {'sourceVerifiedRows','sourceExpectedRows','sourceMissingRows','sourceMismatchRows'})
|
||||
tokens = _binding_inventory(report['facts'])
|
||||
parsed = _bind_literal_measures({'blocks':[{'text':'正式责废7.00 kg。','factIds':[category['id']]}]},
|
||||
tokens,{},facts=report['facts'])
|
||||
blocks, _ = _validate_blocks(parsed,report['facts'],tokens)
|
||||
self.assertIn('7.00 kg',blocks[0]['text'])
|
||||
self.assertIsNone(_source_relation_gap(blocks[0],report['facts']))
|
||||
|
||||
proof_text = '黑片原表已核验行数与应核验行数均为1,未核验0条,不一致0条。'
|
||||
proof = _bind_literal_measures({'blocks':[{'text':proof_text,'factIds':[source['id']]}]},tokens,{})
|
||||
self.assertEqual(proof['blocks'][0]['text'].count('{{N'),3)
|
||||
proof_blocks, _ = _validate_blocks(proof,report['facts'],tokens)
|
||||
self.assertIn('已核验行数与应核验行数均为1',proof_blocks[0]['text'])
|
||||
self.assertIn('未核验0条,不一致0条',proof_blocks[0]['text'])
|
||||
exact_proof = _bind_literal_measures({'blocks':[{'text':'已核验1行/应核验1行;未核验0条;不一致0条。',
|
||||
'factIds':[source['id']]}]},tokens,{})
|
||||
self.assertEqual(exact_proof['blocks'][0]['text'].count('{{N'),4)
|
||||
_validate_blocks(exact_proof,report['facts'],tokens)
|
||||
mismatch = _bind_literal_measures({'blocks':[{'text':'未核验1条。','factIds':[source['id']]}]},tokens,{})
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(mismatch,report['facts'],tokens)
|
||||
|
||||
uneven_source = {'id':'F90','text':'已核验1行/应核验2行;未核验1条;不一致0条。',
|
||||
'evidenceRef':'Q00000090','verifiedNativeNumericValues':[
|
||||
{'sourceOrdinal':0,'rawValue':'1','displayValue':'1','unit':'metadata_count',
|
||||
'metricField':'sourceVerifiedRows','sourceVerificationCount':True},
|
||||
{'sourceOrdinal':1,'rawValue':'2','displayValue':'2','unit':'metadata_count',
|
||||
'metricField':'sourceExpectedRows','sourceVerificationCount':True},
|
||||
{'sourceOrdinal':2,'rawValue':'1','displayValue':'1','unit':'metadata_count',
|
||||
'metricField':'sourceMissingRows','sourceVerificationCount':True},
|
||||
{'sourceOrdinal':3,'rawValue':'0','displayValue':'0','unit':'metadata_count',
|
||||
'metricField':'sourceMismatchRows','sourceVerificationCount':True}]}
|
||||
uneven_tokens = _binding_inventory([uneven_source])
|
||||
exact_uneven = _bind_literal_measures({'blocks':[{'text':'已核验1行/应核验2行;未核验1条;不一致0条。',
|
||||
'factIds':['F90']}]},uneven_tokens,{})
|
||||
self.assertEqual(exact_uneven['blocks'][0]['text'].count('{{N'),4)
|
||||
_validate_blocks(exact_uneven,[uneven_source],uneven_tokens)
|
||||
false_equal = _bind_literal_measures({'blocks':[{'text':'已核验行数与应核验行数均为1。',
|
||||
'factIds':['F90']}]},uneven_tokens,{})
|
||||
self.assertNotIn('{{N',false_equal['blocks'][0]['text'])
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(false_equal,[uneven_source],uneven_tokens)
|
||||
|
||||
scoped_source = {'id':'F91','kind':'source','label':'黑片原表核验',
|
||||
'text':'2026年9月黑片规格A已核验1行/应核验2行;未核验1条;不一致0条。',
|
||||
'evidenceRef':'Q00000091','scope':{'period':{'year':2026,'month':9},
|
||||
'stage':'black','filters':{'product':'规格A'}},
|
||||
'verifiedNativeNumericValues':[
|
||||
{'sourceOrdinal':2,'rawValue':'1','displayValue':'1','unit':'metadata_count',
|
||||
'metricField':'sourceVerifiedRows','sourceVerificationCount':True,'stageHint':'黑片',
|
||||
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
|
||||
{'sourceOrdinal':3,'rawValue':'2','displayValue':'2','unit':'metadata_count',
|
||||
'metricField':'sourceExpectedRows','sourceVerificationCount':True,'stageHint':'黑片',
|
||||
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
|
||||
{'sourceOrdinal':4,'rawValue':'1','displayValue':'1','unit':'metadata_count',
|
||||
'metricField':'sourceMissingRows','sourceVerificationCount':True,'stageHint':'黑片',
|
||||
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}},
|
||||
{'sourceOrdinal':5,'rawValue':'0','displayValue':'0','unit':'metadata_count',
|
||||
'metricField':'sourceMismatchRows','sourceVerificationCount':True,'stageHint':'黑片',
|
||||
'period':{'year':2026,'month':9},'ownerScope':{'product':'规格A'}}]}
|
||||
scoped_tokens = _binding_inventory([scoped_source])
|
||||
def bind_scoped(text):
|
||||
parsed = {'blocks':[{'text':text,'factIds':['F91']}]}
|
||||
parsed = _bind_exact_periods(parsed,scoped_tokens,{})
|
||||
parsed = _bind_exact_year_literals(parsed,scoped_tokens,{})
|
||||
return _bind_literal_measures(parsed,scoped_tokens,{},facts=[scoped_source])
|
||||
matched = bind_scoped('2026年9月黑片规格A已核验1行。')
|
||||
_validate_blocks(matched,[scoped_source],scoped_tokens)
|
||||
for wrong_scope in ('2026年9月亮片规格A已核验1行。',
|
||||
'2026年8月黑片规格A已核验1行。',
|
||||
'2026年9月黑片规格B已核验1行。'):
|
||||
with self.subTest(wrong_scope=wrong_scope):
|
||||
mismatch = bind_scoped(wrong_scope)
|
||||
self.assertFalse(any('{{' + key + '}}' in mismatch['blocks'][0]['text']
|
||||
for key, token_value in scoped_tokens.items() if token_value.get('sourceVerificationCount')))
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(mismatch,[scoped_source],scoped_tokens)
|
||||
|
||||
def test_formal_category_source_relation_needs_same_scope_server_reconciliation(self):
|
||||
period = {'year':2026,'month':9}
|
||||
base = {'kind':'quality_investigation','action':'source','releaseId':'release-1',
|
||||
'period':period,'filters':{'product':'规格06'},'stage':'black',
|
||||
'current':{'context':{'releaseId':'release-1','stage':'black','year':2026,'month':9,
|
||||
'product':'规格06'},
|
||||
'totalRows':1,'aggregateComplete':True,'selectedFields':['责废尺寸重量'],
|
||||
'categories':[{'category':'责废','formalWeight':'7','rawTotalWeight':None,
|
||||
'rawTotalCoverage':{'rows':1,'numericRows':0,'keyAbsentRows':0,'nullOrBlankRows':1,
|
||||
'invalidNumericRows':0,'negativeRows':0},
|
||||
'formalCoverage':{'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
|
||||
'invalidNumericRows':0,'negativeRows':0},
|
||||
'fields':[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':0,
|
||||
'nullOrBlankRows':1,'keyAbsentRows':0,'invalidNumericRows':0,
|
||||
'negativeRows':0,'numericSum':None}}],
|
||||
'subitemWeightSum':'0','differenceFormalMinusSubitems':'7',
|
||||
'comparableRows':1,'missingOrInvalidFormalRows':0,'rawFormalMismatchRows':1,
|
||||
'noFineFields':False,'requiresFurtherInvestigation':True}],
|
||||
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,
|
||||
'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0}},
|
||||
'comparisonPeriod':None}
|
||||
document = {'releaseId':'release-1','evidenceRef':'Q00000001','mode':'investigate','result':base}
|
||||
context = {'releaseId':'release-1','period':period,'filters':{'product':'规格06'},
|
||||
'_agentTask':{'kind':'analysis','period':period,'comparisonPeriod':None,
|
||||
'filters':{'product':'规格06'},'stages':['black'],'executionMode':'native_agent'}}
|
||||
report = build_native_report([document],context,'核对规格06责废原件与正式分量')
|
||||
category = next(fact for fact in report['facts'] if fact.get('kind') == 'category')
|
||||
source = next(fact for fact in report['facts'] if fact.get('kind') == 'source')
|
||||
fact_ids = [category['id'],source['id']]
|
||||
|
||||
raw_only = '责废7.00 kg;原件所选字段与存储记录匹配,但无法确认正式责废分量与来源细项相等。'
|
||||
self.assertIsNone(_source_relation_gap({'text':raw_only,'factIds':fact_ids},report['facts']))
|
||||
raw_label_only = '原件中的责废原始值与原表一致。'
|
||||
self.assertIsNone(_source_relation_gap({'text':raw_label_only,'factIds':fact_ids},report['facts']))
|
||||
raw_amount_audit = '原件中的责废原始值7.00 kg已通过原件核验。'
|
||||
self.assertIsNone(_source_relation_gap({'text':raw_amount_audit,'factIds':fact_ids},report['facts']))
|
||||
for negative_claim in ('正式责废重量7.00 kg未经原件核验。',
|
||||
'正式责废重量7.00 kg尚未通过原件核验。'):
|
||||
self.assertIsNone(_source_relation_gap({'text':negative_claim,'factIds':fact_ids},report['facts']))
|
||||
explicit_difference = '正式责废重量7.00 kg,与原件细项不一致。'
|
||||
self.assertIsNone(_source_relation_gap({'text':explicit_difference,'factIds':fact_ids},report['facts']))
|
||||
invalid_claims = (
|
||||
'正式责废重量7.00 kg,与原始Excel核验结果一致。',
|
||||
'正式**责废**重量7.00 kg,与**原件**核验结果相符。',
|
||||
'正式责废重量7.00 kg已通过原件核验。',
|
||||
'原件核验确认正式责废重量7.00 kg。',
|
||||
'责废7.00 kg已通过原件核验。',
|
||||
'| 正式责废分量 | 原件对账状态 |\n|---|---|\n| 7.00 kg | 与Excel一致 |',
|
||||
)
|
||||
for text in invalid_claims:
|
||||
with self.subTest(text=text):
|
||||
self.assertIsNotNone(_source_relation_gap({'text':text,'factIds':fact_ids},report['facts']))
|
||||
|
||||
reconciled = copy.deepcopy(base)
|
||||
category_data = reconciled['current']['categories'][0]
|
||||
category_data.update(rawTotalWeight='7',subitemWeightSum='7',differenceFormalMinusSubitems='0',
|
||||
rawFormalMismatchRows=0,requiresFurtherInvestigation=False,
|
||||
rawTotalCoverage={'rows':1,'numericRows':1,'keyAbsentRows':0,'nullOrBlankRows':0,
|
||||
'invalidNumericRows':0,'negativeRows':0},
|
||||
fields=[{'field':'责废尺寸重量','coverage':{'rows':1,'numericRows':1,
|
||||
'nullOrBlankRows':0,'keyAbsentRows':0,'invalidNumericRows':0,'negativeRows':0,
|
||||
'numericSum':'7'}}])
|
||||
reconciled_report = build_native_report([{**document,'result':reconciled}],context,
|
||||
'核对规格06责废原件与正式分量')
|
||||
reconciled_category = next(fact for fact in reconciled_report['facts'] if fact.get('kind') == 'category')
|
||||
reconciled_source = next(fact for fact in reconciled_report['facts'] if fact.get('kind') == 'source')
|
||||
self.assertTrue(reconciled_category['_nativeSourceCategoryReconciliation']['reconciled'])
|
||||
valid_claim = '正式责废重量7.00 kg,与原表细项对账一致。'
|
||||
self.assertIsNone(_source_relation_gap({'text':valid_claim,
|
||||
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
|
||||
stage_qualified_claim = '黑片正式责废分量与原表细项对账一致。'
|
||||
self.assertIsNone(_source_relation_gap({'text':stage_qualified_claim,
|
||||
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
|
||||
typed_amount_claim = '责废7.00 kg已通过原件核验。'
|
||||
self.assertIsNone(_source_relation_gap({'text':typed_amount_claim,
|
||||
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
|
||||
independent_following_clause = '黑片正式责废分量与原表细项对账一致,综合指标仍使用正式口径。'
|
||||
self.assertIsNone(_source_relation_gap({'text':independent_following_clause,
|
||||
'factIds':[reconciled_category['id'],reconciled_source['id']]},reconciled_report['facts']))
|
||||
for wrong_scope_claim in (
|
||||
'亮片正式责废分量与原表细项对账一致。',
|
||||
'黑片2026年8月正式责废分量与原表细项对账一致。'):
|
||||
with self.subTest(wrong_scope_claim=wrong_scope_claim):
|
||||
self.assertIsNotNone(_source_relation_gap({'text':wrong_scope_claim,
|
||||
'factIds':[reconciled_category['id'],reconciled_source['id']]},
|
||||
reconciled_report['facts']))
|
||||
|
||||
wrong_scope_source = copy.deepcopy(reconciled_source)
|
||||
wrong_scope_source['_nativeSourceCategoryReconciliations'][0]['stage'] = 'bright'
|
||||
wrong_scope_category = copy.deepcopy(reconciled_category)
|
||||
wrong_scope_category.pop('_nativeSourceCategoryReconciliation',None)
|
||||
self.assertIsNotNone(_source_relation_gap({'text':valid_claim,
|
||||
'factIds':[wrong_scope_category['id'],wrong_scope_source['id']]},
|
||||
[wrong_scope_category,wrong_scope_source]))
|
||||
|
||||
def approving_review(request, timeout):
|
||||
return response({'reportTitle':'原件与正式责废核验','reviews':[{'sourceId':'D1',
|
||||
'role':'core','factIds':fact_ids,'supported':True,'statisticalClaimsSupported':True}],
|
||||
'missingGoalDescriptions':[],'visualIds':[]})
|
||||
unsafe_drafts = (
|
||||
'正式责废重量7.00 kg,与原始Excel核验结果一致。',
|
||||
'正式责废重量7.00 kg已通过原件核验。',
|
||||
'原件核验确认正式责废重量7.00 kg。',
|
||||
'责废7.00 kg已通过原件核验。',
|
||||
)
|
||||
for draft in unsafe_drafts:
|
||||
with self.subTest(reviewed_draft=draft), patch('nao_native_answer.build_native_report',
|
||||
return_value=copy.deepcopy(report)):
|
||||
reviewed = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
|
||||
model='fixture',question='核对规格06责废正式分量与原件',evidence=[],
|
||||
cancel_event=threading.Event(),context={'_nativeExecutionId':'exec',
|
||||
'_dshDraftReview':True,'_autonomousAnswer':True,
|
||||
'_agentTask':{'kind':'analysis','period':period,'filters':{'product':'规格06'},
|
||||
'stages':['black']}},opener=approving_review,
|
||||
native_answer='## 重点\n\n'+draft)
|
||||
self.assertNotIn(draft,reviewed['answer'])
|
||||
self.assertTrue(reviewed['compositionDiagnostics'].get('sourceRelationGaps'))
|
||||
|
||||
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
|
||||
negative_reviewed=synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
|
||||
model='fixture',question='核对规格06责废正式分量与原件',evidence=[],
|
||||
cancel_event=threading.Event(),context={'_nativeExecutionId':'exec',
|
||||
'_dshDraftReview':True,'_autonomousAnswer':True,
|
||||
'_agentTask':{'kind':'analysis','period':period,'filters':{'product':'规格06'},
|
||||
'stages':['black']}},opener=approving_review,
|
||||
native_answer='## 重点\n\n正式责废重量7.00 kg未经原件核验。')
|
||||
self.assertFalse(negative_reviewed['compositionDiagnostics'].get('sourceRelationGaps'))
|
||||
|
||||
def test_native_review_repair_requires_remainder_verdicts_and_nonoverlapping_clause_ids(self):
|
||||
fact = {'id':'F1','kind':'stage','label':'黑片正式指标',
|
||||
'text':'黑片正式报废重量14.00 kg。','evidenceRef':'Q00000001',
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'14','displayValue':'14.00',
|
||||
'unit':'kg','metricField':'scrapWeight','stageHint':'黑片',
|
||||
'period':{'year':2026,'month':9}}]}
|
||||
report = {'facts':[fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
|
||||
'scopes':[],'sections':[],'_presentationSources':[],'rejectedEvidenceRefs':[],
|
||||
'notices':[],'calculationRules':[],'metricFormulaDescriptions':[],
|
||||
'unverifiedEvidenceWarnings':[]}
|
||||
draft = '## 重点\n\n黑片正式报废重量14.00 kg。无法支持的观察句。'
|
||||
calls = []
|
||||
unsupported = '无法支持的观察句。'
|
||||
def provider(request, timeout):
|
||||
messages = json.loads(request.data)['messages']
|
||||
payload = json.loads(messages[1]['content'])
|
||||
calls.append(messages)
|
||||
block = payload['draftBlocks'][0]
|
||||
bad_id = next(item['clauseId'] for item in block['clauseCatalog']
|
||||
if item.get('text') == unsupported)
|
||||
if len(calls) == 1:
|
||||
parent_id = next(item['clauseId'] for item in block['clauseCatalog']
|
||||
if item.get('kind') == 'block')
|
||||
# Parent and child overlap, and the required remainder verdicts
|
||||
# are deliberately missing to exercise the bounded repair.
|
||||
row = {'sourceId':block['sourceId'],'role':'core','factIds':['F1'],
|
||||
'supported':False,'statisticalClaimsSupported':False,
|
||||
'unsupportedClauseIds':[parent_id,bad_id]}
|
||||
return response({'reportTitle':'黑片报废依据','reviews':[row],
|
||||
'missingGoalDescriptions':[],'visualIds':[]})
|
||||
row = {'sourceId':block['sourceId'],'role':'core','factIds':['F1'],
|
||||
'supported':False,'statisticalClaimsSupported':False,'unsupportedClauseIds':[bad_id],
|
||||
'remainingClaimsSupported':True,'remainingStatisticalClaimsSupported':True}
|
||||
return response({'reportTitle':'黑片报废依据','reviews':[row],
|
||||
'missingGoalDescriptions':[],'visualIds':[]})
|
||||
|
||||
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
|
||||
final = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
|
||||
model='fixture',question='解释黑片正式报废重量',evidence=[],cancel_event=threading.Event(),
|
||||
context={'_nativeExecutionId':'exec','_dshDraftReview':True,
|
||||
'_agentTask':{'kind':'analysis','period':{'year':2026,'month':9},
|
||||
'filters':{},'stages':['black']}},opener=provider,native_answer=draft)
|
||||
|
||||
self.assertEqual(len(calls),2)
|
||||
self.assertTrue(final['compositionDiagnostics'].get('reviewClauseRemovals'))
|
||||
self.assertNotIn(unsupported,final['answer'])
|
||||
self.assertIn('14.00 kg',final['answer'])
|
||||
for field in ('remainingClaimsSupported','remainingStatisticalClaimsSupported'):
|
||||
self.assertIn(field,calls[0][0]['content'])
|
||||
self.assertIn(field,calls[1][-1]['content'])
|
||||
self.assertIn('互不重叠',calls[1][-1]['content'])
|
||||
self.assertIn('父节点与其子句/表格行不可同时选择',calls[1][-1]['content'])
|
||||
|
||||
def test_pp_table_headers_bind_same_stage_roles_when_rounded_values_collide(self):
|
||||
from nao_native_answer import (_native_atomic_inventory,_native_unique_span,
|
||||
_native_resolve_span,_prepare_native_draft,_native_draft_review_messages)
|
||||
period={'year':2026,'month':9}
|
||||
facts=[
|
||||
{'id':'F1','kind':'stage','label':'黑片自身率变化',
|
||||
'text':'黑片自身率上升0.15个百分点。','evidenceRef':'Q00000011',
|
||||
'scope':{'period':period,'filters':{},'stage':'black'},
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'0.14687895375728394',
|
||||
'displayValue':'0.15','unit':'pp','metricField':'rateChange',
|
||||
'metricRole':'OwnStageRateChangePp','stageHint':'黑片','period':period,'ownerScope':{}}]},
|
||||
{'id':'F2','kind':'stage','label':'黑片对综合率贡献',
|
||||
'text':'黑片对综合率变化贡献为0.15个百分点。','evidenceRef':'Q00000012',
|
||||
'scope':{'period':period,'filters':{},'stage':'black'},
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'0.14521243189527',
|
||||
'displayValue':'0.15','unit':'pp','metricField':'stageContribution',
|
||||
'metricRole':'ContributionPp','stageHint':'黑片','period':period,'ownerScope':{}}]},
|
||||
]
|
||||
context={'_agentTask':{'kind':'analysis','period':period,'filters':{},'stages':['black']}}
|
||||
draft=('| 阶段 | 自身率变化 (pp) | 对综合率贡献 (pp) |\n'
|
||||
'|---|---:|---:|\n| 黑片 | 0.15 | 0.15 |')
|
||||
prepared,metadata=_prepare_native_draft(draft,facts,context)
|
||||
self.assertEqual(metadata['protectedMetricSpans'],2)
|
||||
by_start={item['start']:item for item in prepared[0]['_protected']}
|
||||
own_start=draft.index('0.15')
|
||||
contribution_start=draft.rindex('0.15')
|
||||
self.assertEqual(by_start[own_start]['factId'],'F1')
|
||||
self.assertEqual(by_start[contribution_start]['factId'],'F2')
|
||||
|
||||
tokens=_native_atomic_inventory(facts)
|
||||
fact_map={fact['id']:fact for fact in facts}
|
||||
own_token=next(token for token in tokens.values() if token.get('metricRole')=='OwnStageRateChangePp')
|
||||
contribution_token=next(token for token in tokens.values() if token.get('metricRole')=='ContributionPp')
|
||||
for start,correct,wrong in ((own_start,own_token,contribution_token),
|
||||
(contribution_start,contribution_token,own_token)):
|
||||
match=re.match(r'0\.15',draft[start:])
|
||||
self.assertIsNotNone(_native_resolve_span(draft,start,start+len(match[0]),correct,
|
||||
fact_map,tokens,context=context,require_positive=True))
|
||||
self.assertIsNone(_native_resolve_span(draft,start,start+len(match[0]),wrong,
|
||||
fact_map,tokens,context=context,require_positive=True))
|
||||
for wrong_stage in ('亮片','综合'):
|
||||
wrong=draft.replace('| 黑片 |','| '+wrong_stage+' |')
|
||||
self.assertIsNone(_native_unique_span(wrong,re.search(r'0\.15',wrong),fact_map,tokens,context,True))
|
||||
|
||||
messages=_native_draft_review_messages(prepared,facts,'说明黑片自身率变化与综合贡献',context,{}, {})
|
||||
payload=json.loads(messages[1]['content'])
|
||||
catalog=payload['bindingCatalog']
|
||||
for token in (own_token,contribution_token):
|
||||
item=catalog[next(key for key,value in catalog.items() if value['factId']==token['factId'])]
|
||||
self.assertEqual(item['metricRole'],token['metricRole'])
|
||||
self.assertEqual(item['metricField'],token['metricField'])
|
||||
self.assertEqual(item['stage'],'黑片')
|
||||
self.assertEqual(item['ownerScope'],{})
|
||||
self.assertEqual(item['period'],period)
|
||||
self.assertNotEqual(own_token['rawValue'],contribution_token['rawValue'])
|
||||
|
||||
def test_dsh_draft_review_flag_preserves_draft_and_unflagged_execution_keeps_presentation(self):
|
||||
fact = {'id':'F1','kind':'stage','label':'黑片正式指标','text':'黑片报废重量14.00 kg。',
|
||||
'evidenceRef':'Q00000001','verifiedNativeNumericValues':[{'sourceOrdinal':0,
|
||||
'rawValue':'14','displayValue':'14.00','unit':'kg','metricField':'scrapWeight',
|
||||
'metricRole':'scrapWeight','stageHint':'黑片','period':{'year':2026,'month':9}}]}
|
||||
report = {'facts':[fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
|
||||
'scopes':[],'sections':[],'_presentationSources':[{'markdown':'### 黑片\n\n报废重量14.00 kg。',
|
||||
'evidenceRefs':['Q00000001'],'scope':{}}]}
|
||||
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)), \
|
||||
patch('nao_native_answer._synthesize_verified_report',return_value={'answer':'checked'}) as finalize:
|
||||
synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
|
||||
question='解释黑片报废重量',evidence=[],cancel_event=threading.Event(),
|
||||
context={'_nativeExecutionId':'exec','_dshDraftReview':True,
|
||||
'_agentTask':{'kind':'analysis'}},native_answer='## 重点\n\n黑片报废重量14.00 kg。')
|
||||
self.assertEqual(finalize.call_args.kwargs['native_answer'],'## 重点\n\n黑片报废重量14.00 kg。')
|
||||
self.assertNotIn('presentationSections',finalize.call_args.kwargs['report'])
|
||||
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)), \
|
||||
patch('nao_native_answer.native_sections',return_value=['### 黑片\n\n报废重量14.00 kg。']), \
|
||||
patch('nao_native_answer._synthesize_verified_report',return_value={'answer':'checked'}) as finalize:
|
||||
synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
|
||||
question='解释黑片报废重量',evidence=[],cancel_event=threading.Event(),
|
||||
context={'_nativeExecutionId':'exec','_agentTask':{'kind':'analysis'}},
|
||||
native_answer='## 重点\n\n黑片报废重量14.00 kg。')
|
||||
self.assertIsNone(finalize.call_args.kwargs['native_answer'])
|
||||
self.assertTrue(finalize.call_args.kwargs['report']['presentationSections'])
|
||||
|
||||
def test_stage_contribution_shorthand_binds_only_the_named_stage_and_contribution_role(self):
|
||||
from nao_native_answer import _native_atomic_inventory, _native_resolve_span
|
||||
period={'year':2026,'month':10}
|
||||
facts=[]
|
||||
for fact_id,field,role,raw,label in [('F1','stageContribution','ContributionPp','-0.14687895375728394','对综合率变化贡献'),
|
||||
('F2','rateChange','OwnStageRateChangePp','-0.14841364717368','自身率变化')]:
|
||||
facts.append({'id':fact_id,'kind':'stage','label':'黑片'+label,'text':'黑片'+label+'-0.15个百分点。',
|
||||
'evidenceRef':'Q00000001','scope':{'period':period,'filters':{},'stage':'black'},
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':raw,'displayValue':'-0.15',
|
||||
'unit':'pp','metricField':field,'metricRole':role,'stageHint':'黑片','period':period,'ownerScope':{}}]})
|
||||
tokens=_native_atomic_inventory(facts);known={f['id']:f for f in facts}
|
||||
contribution=next(t for t in tokens.values() if t.get('metricRole')=='ContributionPp')
|
||||
own=next(t for t in tokens.values() if t.get('metricRole')=='OwnStageRateChangePp')
|
||||
context={'_agentTask':{'kind':'analysis','period':period,'filters':{},'stages':['black'],'fields':['rate']}}
|
||||
text='黑片阶段贡献约 -0.15 pp。';match=re.search('-0.15',text)
|
||||
self.assertIsNotNone(_native_resolve_span(text,*match.span(),contribution,known,tokens,context=context,require_positive=True))
|
||||
self.assertIsNone(_native_resolve_span(text,*match.span(),own,known,tokens,context=context,require_positive=True))
|
||||
for wrong in ('亮片阶段贡献约 -0.15 pp。','黑片对亮片合格率变化贡献约 -0.15 pp。'):
|
||||
match=re.search('-0.15',wrong)
|
||||
self.assertIsNone(_native_resolve_span(wrong,*match.span(),contribution,known,tokens,context=context,require_positive=True))
|
||||
|
||||
def test_native_formal_presentation_gets_one_bounded_repair_before_independent_review(self):
|
||||
period={'year':2026,'month':10}
|
||||
fact={'id':'F1','kind':'stage','label':'黑片合格率','text':'黑片合格率98.84%。','evidenceRef':'Q00000001',
|
||||
'scope':{'period':period,'filters':{},'stage':'black'}}
|
||||
report={'facts':[fact],'missingEvidence':[],'requiredFactIds':['F1'],'priorityFactIds':['F1'],'scopes':[],
|
||||
'sections':[],'notices':[],'rejectedEvidenceRefs':[],'_presentationSources':[{'markdown':'### 黑片合格率\n\n| 指标 | 正式值 |\n|---|---|\n| 黑片合格率 | 98.84% |',
|
||||
'evidenceRefs':['Q00000001'],'scope':{'period':period,'stage':'black'}}]}
|
||||
phases=[];generations=[]
|
||||
def provider(request,timeout):
|
||||
body=json.loads(request.data)
|
||||
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
||||
phases.append('evidence_review')
|
||||
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
||||
phases.append('native_synthesis');generations.append(body)
|
||||
if len(generations)==2:
|
||||
self.assertIn('校验错误',body['messages'][-1]['content'])
|
||||
self.assertNotIn('{{N',body['messages'][-1]['content'])
|
||||
value='99.99' if len(generations)==1 else '98.84'
|
||||
return response({'reportTitle':'2026年10月黑片合格率核对','blocks':[{'role':'core',
|
||||
'text':'黑片合格率'+value+'%。','factIds':['F1']}],'visualIds':[]})
|
||||
with patch('nao_native_answer.build_native_report',return_value=report):
|
||||
result=synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',
|
||||
question='生成黑片合格率报告',evidence=[],cancel_event=threading.Event(),opener=provider,
|
||||
context={'_nativeExecutionId':'exec','_agentTask':{'kind':'analysis','period':period,
|
||||
'filters':{},'fields':['rate'],'stages':['black']}},native_answer='未核验的调查草稿')
|
||||
self.assertTrue(result['evidenceSufficient'],result['compositionDiagnostics'])
|
||||
self.assertEqual(phases,['native_synthesis','native_synthesis','evidence_review'])
|
||||
self.assertNotIn('99.99',result['answer'])
|
||||
self.assertIn('98.84%',result['answer'])
|
||||
|
||||
def test_dsh_draft_cannot_keep_category_value_from_stage_total_after_review_true(self):
|
||||
fact = {'id':'F1','kind':'stage','label':'黑片正式指标','text':'黑片报废重量14.00 kg。',
|
||||
'evidenceRef':'Q00000001','verifiedNativeNumericValues':[{'sourceOrdinal':0,
|
||||
'rawValue':'14','displayValue':'14.00','unit':'kg','metricField':'scrapWeight',
|
||||
'metricRole':'scrapWeight','stageHint':'黑片','period':{'year':2026,'month':9}}]}
|
||||
category_fact = {'id':'F2','kind':'category','label':'责废',
|
||||
'text':'黑片正式大类分量:责废为7.00 kg。','evidenceRef':'Q00000002',
|
||||
'verifiedNativeNumericValues':[{'sourceOrdinal':0,'rawValue':'7','displayValue':'7.00',
|
||||
'unit':'kg','metricField':'categoryWeight','metricRole':'categoryWeight',
|
||||
'stageHint':'黑片','categoryHint':'责废','period':{'year':2026,'month':9}}]}
|
||||
report = {'facts':[fact,category_fact],'missingEvidence':[],'requiredFactIds':[],'priorityFactIds':[],
|
||||
'scopes':[],'sections':[],'_presentationSources':[],'rejectedEvidenceRefs':[],
|
||||
'notices':[],'calculationRules':[],'metricFormulaDescriptions':[],
|
||||
'unverifiedEvidenceWarnings':[]}
|
||||
context = {'_nativeExecutionId':'exec','_dshDraftReview':True,
|
||||
'_autonomousAnswer':True,'_agentTask':{'kind':'analysis','period':{'year':2026,'month':9},
|
||||
'filters':{},'stages':['black']}}
|
||||
invalid_drafts = ('黑片正式责废重量为**14.00 kg**。',
|
||||
'黑片正式**责废**重量为14.00 kg。', '黑片正式责废(14.00 kg)。')
|
||||
for draft in invalid_drafts:
|
||||
def provider(request, timeout):
|
||||
return response({'reportTitle':'黑片报废依据','reviews':[{'sourceId':'D1','role':'core',
|
||||
'factIds':['F1'],'supported':True,'statisticalClaimsSupported':True}],
|
||||
'missingGoalDescriptions':[],'visualIds':[]})
|
||||
with self.subTest(draft=draft), patch('nao_native_answer.build_native_report',
|
||||
return_value=copy.deepcopy(report)):
|
||||
final = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
|
||||
model='fixture',question='核对黑片责废重量',evidence=[],cancel_event=threading.Event(),
|
||||
context=context,opener=provider,native_answer=draft)
|
||||
self.assertEqual(final['compositionDiagnostics']['compositionMode'],'native_draft_review')
|
||||
normalized_answer = re.sub(r'[*_`()()\[\]【】]', '', final['answer'])
|
||||
self.assertNotRegex(normalized_answer, r'责废[^。;\n]{0,20}14\.00\s*kg')
|
||||
|
||||
valid_draft = '黑片正式责废重量为7.00 kg。'
|
||||
def valid_provider(request, timeout):
|
||||
return response({'reportTitle':'责废正式分量','reviews':[{'sourceId':'D1','role':'core',
|
||||
'factIds':['F2'],'supported':True,'statisticalClaimsSupported':True}],
|
||||
'missingGoalDescriptions':[],'visualIds':[]})
|
||||
with patch('nao_native_answer.build_native_report',return_value=copy.deepcopy(report)):
|
||||
accepted = synthesize_native_answer(base_url='https://offline.invalid/v1',api_key='fixture',
|
||||
model='fixture',question='核对黑片责废重量',evidence=[],cancel_event=threading.Event(),
|
||||
context=context,opener=valid_provider,native_answer=valid_draft)
|
||||
self.assertTrue(accepted['evidenceSufficient'],accepted['compositionDiagnostics'])
|
||||
self.assertIn('7.00 kg',accepted['answer'])
|
||||
|
||||
tokens = _binding_inventory([fact,category_fact])
|
||||
table_cases = (
|
||||
('| 对象 | 责废重量(kg) |\n|---|---:|\n| 黑片 | 14.00 |', ['F1'], False),
|
||||
('| 大类 | 黑片报废重量(kg) |\n|---|---:|\n| 责废 | 7.00 |', ['F2'], True),
|
||||
)
|
||||
for table, fact_ids, supported in table_cases:
|
||||
with self.subTest(table=table):
|
||||
parsed = _bind_literal_measures({'blocks':[{'text':table,'factIds':fact_ids}]},tokens,{},facts=[fact,category_fact])
|
||||
if supported:
|
||||
validated, _ = _validate_blocks(parsed,[fact,category_fact],tokens)
|
||||
self.assertIn('7.00',validated[0]['text'])
|
||||
self.assertIsNone(_source_relation_gap({'text':validated[0]['text'],'factIds':fact_ids},[fact,category_fact]))
|
||||
else:
|
||||
self.assertNotIn('{{N',parsed['blocks'][0]['text'])
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_validate_blocks(parsed,[fact,category_fact],tokens)
|
||||
|
||||
def test_short_markdown_tables_and_verified_chart_selection_survive_finalization(self):
|
||||
report = {'facts': [{'id': 'F1', 'text': '合格率95.00%。', 'evidenceRef': 'Q00000001'}],
|
||||
'requiredFactIds': ['F1'], 'priorityFactIds': [], 'scopes': [],
|
||||
@@ -87,6 +596,50 @@ class NativeAnswerTests(unittest.TestCase):
|
||||
self.assertEqual(result['readingSummary']['visualIds'], ['verified-chart'])
|
||||
self.assertFalse(result['compositionRepairAttempted'])
|
||||
|
||||
def test_data_presentation_repair_request_carries_natural_names_and_structure_contract(self):
|
||||
from nao_native_answer import _synthesize_verified_report
|
||||
fact = {'id':'F1','kind':'product','label':'产品04',
|
||||
'text':'2026年9月产品04综合投入重量120.00 kg。','evidenceRef':'Q00000001',
|
||||
'scope':{'period':{'year':2026,'month':9},'filters':{'product':'产品04'},
|
||||
'dimension':'product','stage':'overall'}}
|
||||
report = {'facts':[fact],'requiredFactIds':['F1'],'priorityFactIds':[],'scopes':[],
|
||||
'missingEvidence':[],'rejectedEvidenceRefs':[],'notices':[],
|
||||
'presentationSections':[{'markdown':'### 第一章:重点规格\n\n| 规格 | 综合投入重量 |\n|---|---:|\n| 产品04 | 120.00 kg |',
|
||||
'factIds':['F1'],'evidenceRefs':['Q00000001'],'scope':fact['scope']}]}
|
||||
calls = []
|
||||
def provider(request, timeout):
|
||||
messages = json.loads(request.data)['messages']
|
||||
calls.append(messages)
|
||||
if len(calls) == 1:
|
||||
return response({'reportTitle':'质量分析简报','summarySelection':{'leadIndex':0,'evidenceIndexes':[]},
|
||||
'blocks':[{'role':'core','text':'产品04综合投入重量{{N30}} kg。','factIds':['F1']}],
|
||||
'visualIds':[]})
|
||||
return response({'reportTitle':'质量分析简报','summarySelection':{'leadIndex':0,'evidenceIndexes':[]},
|
||||
'blocks':[{'role':'core','text':'{{F1}}','factIds':['F1']}],'visualIds':[]})
|
||||
result = _synthesize_verified_report(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
||||
question='按用户指定三章结构分析产品04。',report=report,cancel_event=threading.Event(),
|
||||
context={'_agentTask':{'kind':'analysis'}},opener=provider)
|
||||
self.assertEqual(len(calls),2)
|
||||
self.assertTrue(result['compositionRepairAttempted'])
|
||||
generation_rules = calls[0][0]['content']
|
||||
repair_rules = calls[1][-1]['content']
|
||||
for instruction in ('平台会按已核验来源自动插入完整数据表','不重复列出全部投入规模、分母',
|
||||
'产品、组织和客户必须写完整原名','保留用户明确指定的章节数量、顺序和标题',
|
||||
'简短概括放在第一章内','每章Markdown标题与该章正文必须放在同一个block',
|
||||
'不能把标题单独做成factIds为空的block','N/I/P开头的内部诊断编号'):
|
||||
self.assertIn(instruction,generation_rules)
|
||||
for instruction in ('平台会插入完整已核验排名表与投入/分母数据','避免在text中逐行重述排名或重复完整规模/分母',
|
||||
'产品、组织和客户写完整原名','保留用户指定的章节数量、顺序、标题',
|
||||
'简短概括并入第一章','每章Markdown标题与该章正文必须留在同一个block',
|
||||
'不要保留空factIds标题','N/I/P编号只是定位问题的诊断线索'):
|
||||
self.assertIn(instruction,repair_rules)
|
||||
self.assertIn('校验错误中的N/I/P编号只是定位问题的诊断线索',repair_rules)
|
||||
self.assertIn('不可用编号替代产品名、组织名或指标名',repair_rules)
|
||||
self.assertIn('保留用户指定的章节数量、顺序、标题',repair_rules)
|
||||
self.assertNotIn('N30',repair_rules)
|
||||
self.assertNotIn('改用已有证据的token',repair_rules)
|
||||
self.assertTrue(result['evidenceSufficient'],result['compositionDiagnostics'])
|
||||
|
||||
def test_strong_defect_comparison_needs_same_scope_counterpart_and_deletes_only_clause(self):
|
||||
scope = {'filters': {'department': '新能源汽车磁钢'}, 'period': {'year': 2026, 'month': 9},
|
||||
'comparisonPeriod': {'year': 2026, 'month': 8}, 'dimension': 'defect', 'stage': 'black'}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Native tool loop boundaries, using actual tool adapters and captured-like APIs."""
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -10,7 +11,7 @@ from pathlib import Path
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
||||
from nao_native_exploration import prepare_step, NativeExplorationTools, _proofs, native_model_view
|
||||
from nao_agent_task import normalize_task, plan_task
|
||||
from nao_quality_tools import QualityToolError
|
||||
from nao_quality_tools import QualityToolError, tool_definition
|
||||
from test_nao_protocol_execution import BoundApi, identity
|
||||
from test_nao_quality_tools import RELEASE
|
||||
from test_nao_agent_task import task_value
|
||||
@@ -59,6 +60,80 @@ class Delegated(BoundApi):
|
||||
|
||||
class NativeExplorationTests(unittest.TestCase):
|
||||
def state(self): return {'releaseId': RELEASE, 'scopes': []}
|
||||
|
||||
def test_report_rank_is_available_without_a_preplanned_rank_dimension(self):
|
||||
task=root();task.update(dimensions=['defect'],comparisonPeriod=None,
|
||||
question='生成2026年1—10月总体质量分析报告,重点说明总体表现、月度变化和主要报废结构。')
|
||||
schema=tool_definition(native_protocol=True,task=task,release_id=RELEASE)['inputSchema']
|
||||
self.assertIn('rank',schema['properties']['mode']['enum'])
|
||||
self.assertNotIn('investigate',schema['properties']['mode']['enum'])
|
||||
|
||||
def test_captured_rank_calls_explain_missing_metric_before_proposing_any_scope(self):
|
||||
for dimension in ('department','product'):
|
||||
with self.subTest(dimension=dimension), self.assertRaisesRegex(QualityToolError,'缺少metric'):
|
||||
prepare_step({'mode':'rank','stage':'black','dimension':dimension,'limit':10},root(),self.state())
|
||||
|
||||
def test_public_rank_trace_retains_the_actual_metric_without_credentials(self):
|
||||
from nao_quality_runtime import public_tool_call
|
||||
call=public_tool_call('mcp_call',{'tool':'quality_data','arguments':{
|
||||
'mode':'rank','metric':'totalWeight','stage':'black','dimension':'product','token':'private'}})
|
||||
self.assertEqual(call['input']['metric'],'totalWeight')
|
||||
self.assertEqual(call['label'],'查询重量排名')
|
||||
self.assertNotIn('token',call['input'])
|
||||
|
||||
def test_batch_overview_uses_one_bound_scope_and_overall_alias_cannot_widen_parent(self):
|
||||
task=root();task['filters']={'department':'A'}
|
||||
args,proposal=prepare_step({'mode':'query','view':'group','filters':{'department':'__ALL__'}},task,self.state())
|
||||
self.assertEqual(args['filters'],{'department':'A'})
|
||||
self.assertEqual(args['period'],task['period'])
|
||||
self.assertEqual(args['limit'],100)
|
||||
self.assertEqual(proposal['node']['purpose'],'metrics')
|
||||
self.assertEqual(proposal['node']['dimension'],'group')
|
||||
self.assertEqual(proposal['node']['selector'],{'kind':'base'})
|
||||
|
||||
def test_inactive_rows_are_omitted_from_model_projection_without_recalculating_summary(self):
|
||||
tools=NativeExplorationTools(delegated=None,evidence_dir=Path('/unused'),task=root(),identity={},
|
||||
state_path=Path('/unused/state'),directory={'departments':['有效产业部']})
|
||||
raw={'view':'department','summary':{'compositeRate':.95},'rows':[{'key':'有效产业部'},
|
||||
{'key':'欧洲磁钢'},{'key':'美洲产业部'}],'rowCount':3,'rowsTruncated':False}
|
||||
view=tools.active_result(raw)
|
||||
self.assertEqual(view['rows'],[{'key':'有效产业部'}])
|
||||
self.assertEqual(view['summary'],raw['summary'])
|
||||
self.assertEqual(len(raw['rows']),3)
|
||||
|
||||
def test_no_published_rows_are_cached_as_no_data_with_original_proof(self):
|
||||
from nao_quality_tools import SourceCoverageUnavailable
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p=Path(folder);api=Delegated()
|
||||
tools=NativeExplorationTools(delegated=api,evidence_dir=p/'evidence',task=root(),
|
||||
identity={**identity(),'releaseId':RELEASE},state_path=p/'state.json')
|
||||
raw={'coverage':{'sourceMonths':[]},'protocolEvidence':{**identity(),'queryId':'empty-query'}}
|
||||
with patch('nao_quality_tools.QualityTools.quality_data',side_effect=SourceCoverageUnavailable(root()['period'],raw)) as query:
|
||||
first=tools.quality_data({'mode':'query'})
|
||||
second=tools.quality_data({'mode':'query'})
|
||||
self.assertEqual(query.call_count,1)
|
||||
self.assertEqual(first['status'],'no_data')
|
||||
self.assertIsNone(first['summary'])
|
||||
self.assertEqual(first['protocolEvidence'],raw['protocolEvidence'])
|
||||
self.assertTrue(second['reusedEvidence'])
|
||||
|
||||
@staticmethod
|
||||
def source_task():
|
||||
value = root()
|
||||
value.update(kind='analysis', fields=[], dimensions=[], dimensionStages={}, stages=['black'],
|
||||
period={'year': 2026, 'month': 9}, comparisonPeriod=None,
|
||||
comparisonRequired=False, trendRequired=False, verifySource=True,
|
||||
fullDetails=False)
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def source_catalog():
|
||||
return {'releaseId': RELEASE, 'sourceFields': [
|
||||
{'key': 'src_responsibility_detail', 'label': '责废细项重量', 'stages': ['black'],
|
||||
'columns': {'black': '原表责废细项重量', 'bright': None}},
|
||||
{'key': 'src_black_total', 'label': '黑片报废合计', 'stages': ['black'],
|
||||
'columns': {'black': '原表黑片报废合计', 'bright': None}},
|
||||
]}
|
||||
def test_model_view_removes_only_transport_and_identical_signal_duplicates(self):
|
||||
row={'key':'甲','label':'甲','rank':1,'current':{'rate':0,'scrapWeight':None},
|
||||
'comparison':{'rate':None,'scrapWeight':0}}
|
||||
@@ -85,6 +160,15 @@ class NativeExplorationTests(unittest.TestCase):
|
||||
changed=native_model_view(source)
|
||||
self.assertIn('rateDrop',changed['prioritySignals']['scale'][0])
|
||||
self.assertIsNone(changed['prioritySignals']['scale'][0]['rateDrop'])
|
||||
def test_official_category_share_names_its_actual_denominator_without_mutating_signed_values(self):
|
||||
source={'kind':'quality_metrics','summary':{'bright':{'rate':.98,'scrapWeight':100,
|
||||
'defects':[{'name':'加工废','weight':30,'rate':.6},{'name':'镀废','weight':20,'rate':.4}]}}}
|
||||
original=copy.deepcopy(source);view=native_model_view(source)['summary']['bright']
|
||||
self.assertEqual(view['rate'],.98)
|
||||
self.assertEqual(view['defects'][0]['shareOfListedCategoryWeight'],.6)
|
||||
self.assertNotIn('rate',view['defects'][0])
|
||||
self.assertEqual(view['categoryShareBasis'],'sum_of_listed_category_weights')
|
||||
self.assertEqual(source,original)
|
||||
def test_explicit_cause_cannot_silently_become_a_static_recipe(self):
|
||||
from nao_native_planning import explicit_investigation
|
||||
self.assertTrue(explicit_investigation('为什么下降?请自行深挖'))
|
||||
@@ -292,6 +376,178 @@ class NativeExplorationTests(unittest.TestCase):
|
||||
self.assertEqual(args['limit'], 100)
|
||||
self.assertEqual(proposal['node']['purpose'], 'source')
|
||||
|
||||
def test_native_source_fields_map_only_exact_current_catalog_keys_labels_and_columns(self):
|
||||
task = self.source_task()
|
||||
catalog = self.source_catalog()
|
||||
args, proposal = prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': ['责废细项重量']}, task, self.state(), source_catalog=catalog)
|
||||
self.assertEqual(args['fields'], ['src_responsibility_detail'])
|
||||
self.assertEqual(proposal['node']['sourceFields'], {'black': ['src_responsibility_detail']})
|
||||
|
||||
args, proposal = prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': ['src_black_total']}, task, self.state(), source_catalog=catalog)
|
||||
self.assertEqual(args['fields'], ['src_black_total'])
|
||||
self.assertEqual(proposal['node']['sourceFields'], {'black': ['src_black_total']})
|
||||
|
||||
duplicate_label = copy.deepcopy(catalog)
|
||||
duplicate_label['sourceFields'].append({'key': 'src_duplicate', 'label': '责废细项重量',
|
||||
'stages': ['black'], 'columns': {'black': '另一原列', 'bright': None}})
|
||||
for bad_catalog, bad_field, error in (
|
||||
(catalog, '未经授权的字段', '不在当前releaseId'),
|
||||
(catalog, 'src_bright_only', '不在当前releaseId'),
|
||||
(duplicate_label, '责废细项重量', '不唯一')):
|
||||
with self.subTest(field=bad_field), self.assertRaisesRegex(QualityToolError, error):
|
||||
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': [bad_field]}, task, self.state(), source_catalog=bad_catalog)
|
||||
stale = {**catalog, 'releaseId': 'older-release'}
|
||||
with self.assertRaisesRegex(QualityToolError, '不属于当前releaseId'):
|
||||
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': ['src_responsibility_detail']}, task, self.state(), source_catalog=stale)
|
||||
|
||||
def test_source_scope_rejects_unsupported_actions_and_category_without_registering(self):
|
||||
task = self.source_task()
|
||||
for action in ('overview', 'fields'):
|
||||
with self.subTest(action=action), self.assertRaisesRegex(QualityToolError, '只支持action=source或rows'):
|
||||
prepare_step({'mode': 'investigate', 'action': action, 'stage': 'black'}, task, self.state())
|
||||
with self.assertRaisesRegex(QualityToolError, '不支持category筛选.*categoryReconciliation'):
|
||||
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'category': '责废'}, task, self.state())
|
||||
with self.assertRaisesRegex(QualityToolError, '未签发sourceRevisionId'):
|
||||
prepare_step({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'sourceRevisionId': 'revision-2'}, task, self.state())
|
||||
|
||||
# Category filtering remains a supported defect query and is not silently stripped.
|
||||
_, proposal = prepare_step({'mode': 'defects', 'stage': 'black', 'dimension': 'defect',
|
||||
'category': '责废'}, root(), self.state())
|
||||
self.assertEqual(proposal['node']['category'], '责废')
|
||||
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder); api = Delegated()
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=task,
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
|
||||
with self.assertRaisesRegex(QualityToolError, 'categoryReconciliation'):
|
||||
tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'category': '责废'})
|
||||
self.assertEqual(api.steps, [])
|
||||
|
||||
def test_dsh_schema_uses_only_protocol_supported_investigation_actions(self):
|
||||
legacy = tool_definition()
|
||||
native = tool_definition(native_protocol=True)
|
||||
self.assertEqual(legacy['inputSchema']['properties']['action']['enum'],
|
||||
['overview', 'rows', 'fields', 'source'])
|
||||
self.assertEqual(native['inputSchema']['properties']['action']['enum'], ['source', 'rows'])
|
||||
self.assertIn('正式指标概览使用query', native['description'])
|
||||
self.assertIn('字段目录使用catalog', native['description'])
|
||||
self.assertIn('categoryReconciliation', native['inputSchema']['properties']['category']['description'])
|
||||
query_rule = next(rule for rule in native['inputSchema']['allOf']
|
||||
if rule.get('if', {}).get('properties', {}).get('mode', {}).get('const') == 'query')
|
||||
query_forbidden = [clause['not']['required'][0] for clause in query_rule['then']['allOf']]
|
||||
self.assertIn('stage', query_forbidden)
|
||||
self.assertIn('category', query_forbidden)
|
||||
native_raw_scope = next(rule for rule in native['inputSchema']['allOf']
|
||||
if rule.get('if', {}).get('properties', {}).get('action', {}).get('enum') == ['source', 'rows'])
|
||||
self.assertEqual([clause['not']['required'][0] for clause in native_raw_scope['then']['allOf']],
|
||||
['category', 'sourceRevisionId'])
|
||||
|
||||
def test_source_query_signs_catalog_stable_key_and_sends_only_signed_original_column(self):
|
||||
source_fields = [{'key': 'src_responsibility_detail', 'label': '责废细项重量',
|
||||
'stages': ['black'], 'columns': {'black': '原表责废细项重量', 'bright': None}}]
|
||||
fine_catalog = {'kind': 'fine_defect_catalog', 'grain': 'source_defect_weight',
|
||||
'fields': [{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
|
||||
'category': '责废', 'available': True}],
|
||||
'categories': [{'stage': 'black', 'category': '责废', 'available': True}]}
|
||||
|
||||
class SourceApi(Delegated):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.registered = []
|
||||
self.source_payloads = []
|
||||
self.registered_scopes = []
|
||||
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
if path == '/api/analysis/model-authorization':
|
||||
self.authorization_calls += 1
|
||||
return {}
|
||||
if path == '/api/releases':
|
||||
return {'currentReleaseId': RELEASE, 'items': [
|
||||
{'id': RELEASE, 'name': 'Published', 'status': 'PUBLISHED'}]}
|
||||
if path == '/api/options':
|
||||
return {'years': [2026], 'departments': [], 'groups': [], 'products': [], 'projectGroups': []}
|
||||
if path == '/api/analysis/tools/catalog':
|
||||
return {'kind': 'analysis_catalog', 'visibleWeightFields': [], 'visibleDefectFields': [],
|
||||
'dimensions': [], 'topDimensions': [], 'grouping': ['overall'], 'limitations': [],
|
||||
'sourceFields': copy.deepcopy(source_fields), 'fineDefects': copy.deepcopy(fine_catalog)}
|
||||
if path == '/api/analysis/protocol/exploration/steps':
|
||||
self.registered.append(copy.deepcopy(payload))
|
||||
node = payload['node']
|
||||
task = NativeExplorationTests.source_task()
|
||||
task['stages'] = copy.deepcopy(node['stages'])
|
||||
selected = node.get('sourceFields')
|
||||
if selected:
|
||||
task['sourceFields'] = copy.deepcopy(selected)
|
||||
task['investigationFields'] = {
|
||||
stage: [next(row['columns'][stage] for row in source_fields if row['key'] == key)
|
||||
for key in keys]
|
||||
for stage, keys in selected.items()}
|
||||
registered = {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
|
||||
'task': task, 'scope': {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
|
||||
'purpose': node['purpose'], 'task': task}}
|
||||
self.registered_scopes.append(registered)
|
||||
return copy.deepcopy(registered)
|
||||
if path == '/api/analysis/investigate':
|
||||
self.source_payloads.append(copy.deepcopy(payload))
|
||||
period = {key: payload[key] for key in ('year', 'month', 'quarter', 'monthStart', 'monthEnd')
|
||||
if key in payload}
|
||||
context = {**period, 'releaseId': RELEASE, 'stage': payload['stage'], 'category': None,
|
||||
'sourceRevisionId': None, 'includeUnassigned': False,
|
||||
'department': payload.get('department'), 'group': payload.get('group'),
|
||||
'product': payload.get('product'), 'projectGroup': payload.get('projectGroup'),
|
||||
'extraFilters': payload.get('extraFilters', {}), 'columnFilters': payload.get('columnFilters', []),
|
||||
'exclusions': payload.get('exclusions', {})}
|
||||
proof = {**identity(), 'queryId': 'source-query', 'responseHash': 'a' * 64,
|
||||
'nodeId': node_id, 'scopeId': scope_id}
|
||||
return {'kind': 'quality_investigation', 'action': 'source', 'releaseId': RELEASE,
|
||||
'context': context, 'totalRows': 1, 'aggregateComplete': True,
|
||||
'categories': [{'category': '责废', 'categoryReconciliation': {'status': 'INCOMPLETE'}}],
|
||||
'lineage': [], 'selectedFields': copy.deepcopy(payload.get('fields')),
|
||||
'sourceVerification': {'expectedRows': 1, 'complete': True, 'verifiedRows': 1,
|
||||
'missingRows': 0, 'mismatchRows': 0, 'allSelectedValuesMatchStored': True},
|
||||
'protocolEvidence': proof}
|
||||
if path == '/api/analysis/protocol/queries':
|
||||
return {'scopeCoverage': [{'nodeId': row['nodeId'], 'scopeId': row['scopeId'],
|
||||
'complete': True, 'missingEvidence': []} for row in self.registered_scopes], 'rootGoalGaps': []}
|
||||
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder)
|
||||
api = SourceApi()
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=self.source_task(),
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
|
||||
tools.quality_data({'mode': 'catalog'})
|
||||
result = tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': ['责废细项重量']})
|
||||
|
||||
self.assertEqual(len(api.registered), 1)
|
||||
signed_node = api.registered[0]['node']
|
||||
self.assertEqual(signed_node['sourceFields'], {'black': ['src_responsibility_detail']})
|
||||
self.assertEqual(api.source_payloads[0]['fields'], ['原表责废细项重量'])
|
||||
self.assertNotIn('category', api.source_payloads[0])
|
||||
self.assertEqual(result['action'], 'source')
|
||||
self.assertEqual(result['category'], None)
|
||||
self.assertEqual(_proofs(result)[0]['nodeId'], signed_node['nodeId'])
|
||||
|
||||
def test_native_source_fields_require_catalog_from_current_release_before_registration(self):
|
||||
class NoCatalog(Delegated):
|
||||
pass
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder); api = NoCatalog()
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=path/'evidence', task=self.source_task(),
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=path/'state.json')
|
||||
with self.assertRaisesRegex(QualityToolError, '当前releaseId的catalog'):
|
||||
tools.quality_data({'mode': 'investigate', 'action': 'source', 'stage': 'black',
|
||||
'fields': ['src_responsibility_detail']})
|
||||
self.assertEqual(api.steps, [])
|
||||
|
||||
def test_multilevel_child_inherits_verified_parent(self):
|
||||
state = self.state(); state['scopes'] = [{'filters': {'group': '甲'}, 'queryIds': ['q1']}]
|
||||
_, step = prepare_step({'mode': 'defects', 'stage': 'black', 'filters': {'group': '甲', 'product': '乙'}}, root(), state)
|
||||
@@ -314,6 +570,64 @@ class NativeExplorationTests(unittest.TestCase):
|
||||
tools.quality_data({'mode': 'compare', 'dimension': 'product'})
|
||||
self.assertEqual(api.steps, [])
|
||||
|
||||
def test_catalog_reads_only_authorized_metadata_and_formal_reads_still_register_scope(self):
|
||||
fine_catalog = {
|
||||
'kind': 'fine_defect_catalog', 'grain': 'source_defect_weight',
|
||||
'fields': [
|
||||
{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
|
||||
'category': '责废', 'sourceField': '责废尺寸重量',
|
||||
'grain': 'source_defect_weight', 'available': True},
|
||||
{'stage': 'bright', 'key': '镀废重量', 'label': '镀废重量',
|
||||
'category': '镀废', 'sourceField': '镀废重量',
|
||||
'grain': 'source_defect_weight', 'available': True},
|
||||
],
|
||||
'categories': [
|
||||
{'stage': 'black', 'category': '责废', 'available': True},
|
||||
{'stage': 'bright', 'category': '镀废', 'available': True},
|
||||
],
|
||||
}
|
||||
|
||||
class CatalogApi(Delegated):
|
||||
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
|
||||
result = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
|
||||
if path == '/api/analysis/tools/catalog':
|
||||
result = {**result, 'fineDefects': copy.deepcopy(fine_catalog)}
|
||||
return result
|
||||
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
p = Path(folder); api = CatalogApi()
|
||||
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
|
||||
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
|
||||
|
||||
catalog = tools.quality_data({'mode': 'catalog'})
|
||||
self.assertEqual(catalog['periods']['availableMonthsInLatestYear'], [])
|
||||
self.assertEqual(catalog['periods']['availableMonthsSource'], 'not_scanned')
|
||||
self.assertEqual(catalog['fineDefects']['endpoint'], 'GET /api/analysis/tools/catalog')
|
||||
self.assertEqual(catalog['fineDefects']['fields'], fine_catalog['fields'])
|
||||
self.assertEqual(catalog['fineDefects']['scanned'], False)
|
||||
self.assertNotIn('protocolEvidence', catalog)
|
||||
|
||||
narrowed = tools.quality_data({'mode': 'catalog', 'stage': 'black'})
|
||||
self.assertEqual([item['stage'] for item in narrowed['fineDefects']['fields']], ['black'])
|
||||
self.assertEqual([item['stage'] for item in narrowed['fineDefects']['categories']], ['black'])
|
||||
paths = [call[1].split('?', 1)[0] for call in api.calls]
|
||||
self.assertTrue(set(paths) <= {
|
||||
'/api/analysis/model-authorization', '/api/releases', '/api/analysis/tools/catalog', '/api/options'
|
||||
}, paths)
|
||||
self.assertNotIn('/api/metrics', paths)
|
||||
self.assertNotIn('/api/analysis/defects', paths)
|
||||
|
||||
for _ in range(3):
|
||||
duplicate = tools.quality_data({'mode': 'catalog'})
|
||||
self.assertTrue(duplicate['reusedEvidence'])
|
||||
state = json.loads((p/'state.json').read_text())
|
||||
self.assertEqual(state['noProgress'], 0)
|
||||
self.assertFalse(state.get('halted', False))
|
||||
|
||||
tools.quality_data({'mode': 'compare'})
|
||||
self.assertEqual(len(api.steps), 1)
|
||||
self.assertTrue(any(path.startswith('/api/metrics') for _, path, _ in api.calls))
|
||||
|
||||
def test_actual_compare_preserves_both_proofs_and_duplicate_reads_are_cached(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
api = Delegated(); p = Path(folder)
|
||||
|
||||
@@ -21,5 +21,13 @@ class NativeLayoutCapacityTests(unittest.TestCase):
|
||||
with self.assertRaises(NativeCompositionError):_native_draft_blocks(draft)
|
||||
def test_layout_only_cannot_become_a_business_answer(self):
|
||||
with self.assertRaises(NativeCompositionError):_native_draft_blocks('## 标题\n\n---\n\n* * *')
|
||||
def test_full_report_packs_short_paragraphs_without_removing_or_reordering_text(self):
|
||||
draft='## 期间汇总\n\n'+'\n\n'.join('第'+str(i)+'段已核实内容。' for i in range(30))
|
||||
blocks=_native_draft_blocks(draft,pack_report=True)
|
||||
self.assertLessEqual(len(blocks),24)
|
||||
self.assertEqual('\n\n'.join(b['text'] for b in blocks),draft)
|
||||
self.assertTrue(all(len(b['text'])<=2200 for b in blocks))
|
||||
with self.assertRaises(NativeCompositionError):
|
||||
_native_draft_blocks('| 完整表格 |\n'+'字'*2201,pack_report=True)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Core safety requirements that must remain in the DSH planning contract."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from nao_native_planning import NATIVE_ROOT_PROMPT, STATIC_NODE_CONTRACT
|
||||
|
||||
|
||||
class NativePlanningContractTests(unittest.TestCase):
|
||||
def test_ambiguous_catalog_prefix_requires_a_user_choice(self):
|
||||
self.assertIn('边界前缀对应多个候选时必须needsClarification=true', NATIVE_ROOT_PROMPT)
|
||||
self.assertIn('清除该产品filter', NATIVE_ROOT_PROMPT)
|
||||
self.assertIn('不能把前缀直接传为产品筛选值', NATIVE_ROOT_PROMPT)
|
||||
|
||||
def test_source_receipts_do_not_substitute_for_formal_metric_reconciliation(self):
|
||||
self.assertIn('两者都不提供正式指标或正式缺陷分类', STATIC_NODE_CONTRACT)
|
||||
self.assertIn('还必须规划能返回对应正式值的metrics节点和/或正式defects节点', STATIC_NODE_CONTRACT)
|
||||
self.assertIn('不得用“原件核验通过”或“记录已读全”的回执代替对账结果', STATIC_NODE_CONTRACT)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -71,6 +71,36 @@ class NativeStructureTests(unittest.TestCase):
|
||||
self.assertIn('规格'+name,location['inheritedText'])
|
||||
self.assertEqual(sum('规格'+other in location['inheritedText'] for other in '甲乙丙'),1)
|
||||
|
||||
def test_closed_annotation_siblings_do_not_leak_subjects_even_when_values_match(self):
|
||||
tokens={'N1':{'ownerScope':{'group':'合成组织甲'}},
|
||||
'N2':{'ownerScope':{'group':'合成组织乙'}}}
|
||||
context={'_agentTask':{'period':{'year':2026,'month':9},'filters':{}},
|
||||
'_nativeScopeCatalog':[{'group':'合成组织甲'},{'group':'合成组织乙'}]}
|
||||
for second_rate in ('-0.28','-0.29'):
|
||||
text=('合成组织甲(率降-0.28pp,报废增量+12kg)和'
|
||||
f'合成组织乙(率降{second_rate}pp,报废增量+13kg)。')
|
||||
for span in (second_rate,'+13'):
|
||||
start=text.rindex(span)
|
||||
location=native_span_structure(text).locate(start,start+len(span))
|
||||
with self.subTest(second_rate=second_rate,span=span):
|
||||
self.assertIn('合成组织乙',location['inheritedText'])
|
||||
self.assertNotIn('合成组织甲',location['inheritedText'])
|
||||
slots=_native_span_slots(text,start,start+len(span),context,tokens,
|
||||
'pp' if span.startswith('-') else 'kg',location)
|
||||
self.assertFalse(slots['scopeConflict'],slots)
|
||||
self.assertEqual(slots['scope'].get('group'),'合成组织乙')
|
||||
|
||||
nested='范围:父部门(合成组织乙(率降-0.28pp))'
|
||||
start=nested.index('-0.28')
|
||||
location=native_span_structure(nested).locate(start,start+5)
|
||||
nested_context={'_agentTask':{'period':{'year':2026,'month':9},
|
||||
'filters':{'department':'父部门'}},
|
||||
'_nativeScopeCatalog':[{'department':'父部门','group':'合成组织乙'}]}
|
||||
nested_tokens={'N1':{'ownerScope':{'department':'父部门','group':'合成组织乙'}}}
|
||||
slots=_native_span_slots(nested,start,start+5,nested_context,nested_tokens,'pp',location)
|
||||
self.assertFalse(slots['scopeConflict'],slots)
|
||||
self.assertEqual(slots['scope'],{'department':'父部门','group':'合成组织乙'})
|
||||
|
||||
def test_versions_and_explicit_ancestors_are_separate(self):
|
||||
blocks=[{'sourceId':'D1','text':'## 产业部:部门甲\n\n分组比较:'},
|
||||
{'sourceId':'D2','text':'| 分组 | 本期 |\n|---|---|\n| 分组乙 | 10 |'}]
|
||||
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
import copy
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
@@ -11,7 +12,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import nao_quality_runtime as runtime
|
||||
from nao_product_identification import identify_products
|
||||
from nao_product_identification import identify_products, resolve_clarification_choice
|
||||
|
||||
ACTOR = '11111111-1111-4111-8111-111111111111'
|
||||
SESSION = '22222222-2222-4222-8222-222222222222'
|
||||
@@ -20,6 +21,89 @@ TURN = '55555555-5555-4555-8555-555555555555'
|
||||
|
||||
|
||||
class ProductClarificationRuntimeTests(unittest.TestCase):
|
||||
def test_catalog_prefix_with_multiple_candidates_forces_explicit_choice(self):
|
||||
question = ('查询2026年10月合成产品0的黑片合格率;如果名称不能唯一确定,'
|
||||
'请让我明确选择,不要自行猜产品。')
|
||||
authorized = [f'合成产品0{i}' for i in range(1, 7)]
|
||||
options = {'products': authorized, 'organizationPaths': []}
|
||||
match = identify_products(question, options)['productMatch']
|
||||
self.assertEqual(match['status'], 'ambiguous')
|
||||
self.assertEqual(match['query'], '合成产品0')
|
||||
self.assertEqual(match['candidateCount'], 6)
|
||||
self.assertFalse(match['truncated'])
|
||||
self.assertEqual([item['name'] for item in match['candidates']], authorized)
|
||||
|
||||
# Even if the planner mistakenly forwards the partial name as a filter
|
||||
# and omits needsClarification, the deterministic authorized catalog
|
||||
# overrides it while preserving the user's metric and scope.
|
||||
from nao_agent_task import plan_task
|
||||
planned = {
|
||||
'goal': question, 'kind': 'metrics', 'freshQuery': True,
|
||||
'dimensions': [], 'dimensionStages': {}, 'stages': ['black'],
|
||||
'fields': ['rate'], 'period': {'year': 2026, 'month': 10},
|
||||
'comparisonRequired': False, 'trendRequired': False,
|
||||
'verifySource': False, 'fullDetails': False,
|
||||
'visualPreference': 'none', 'responseStyle': 'standard',
|
||||
'needsClarification': False, 'clarification': '',
|
||||
'filterUpdates': {'product': '合成产品0'},
|
||||
}
|
||||
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
|
||||
'_businessCatalog': {'productMatch': match}}
|
||||
with patch('nao_agent_task._model_json', return_value=planned):
|
||||
pending = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question=question, context=context, cancel_event=threading.Event())
|
||||
self.assertTrue(pending['needsClarification'])
|
||||
self.assertNotIn('product', pending['filters'])
|
||||
self.assertEqual(pending['period'], {'year': 2026, 'month': 10})
|
||||
self.assertEqual(pending['fields'], ['rate'])
|
||||
self.assertEqual(pending['stages'], ['black'])
|
||||
self.assertEqual([item['value'] for item in pending['clarificationRequest']['options']], authorized)
|
||||
|
||||
selected = resolve_clarification_choice('选择合成产品03', options, pending)
|
||||
self.assertEqual(selected['productMatch']['status'], 'exact')
|
||||
self.assertTrue(selected['productMatch']['selectedFromClarification'])
|
||||
self.assertEqual(selected['productMatch']['candidates'][0]['name'], '合成产品03')
|
||||
|
||||
def test_prefix_matching_stays_with_current_authorized_catalog_and_candidate_cap(self):
|
||||
question = '查询合成产品0的黑片合格率'
|
||||
names = [f'合成产品0{i:02d}' for i in range(25)]
|
||||
match = identify_products(question, {'products': names})['productMatch']
|
||||
self.assertEqual(match['status'], 'ambiguous')
|
||||
self.assertEqual(match['candidateCount'], 25)
|
||||
self.assertEqual(len(match['candidates']), 20)
|
||||
self.assertTrue(match['truncated'])
|
||||
self.assertEqual(set(item['name'] for item in match['candidates']), set(names[:20]))
|
||||
|
||||
not_authorized = identify_products('查询合成产品99的黑片合格率', {'products': names})['productMatch']
|
||||
self.assertEqual(not_authorized['status'], 'none')
|
||||
self.assertEqual(not_authorized['candidates'], [])
|
||||
|
||||
unique = identify_products('查询合成产品0的黑片合格率', {'products': ['合成产品01']})['productMatch']
|
||||
self.assertEqual(unique['status'], 'unique')
|
||||
self.assertEqual([item['name'] for item in unique['candidates']], ['合成产品01'])
|
||||
|
||||
# A unique partial prefix is bound to the full authorized directory
|
||||
# name, never sent to the API as a partial product filter.
|
||||
from nao_agent_task import plan_task
|
||||
question = '查询合成产品0的黑片合格率'
|
||||
planned = {
|
||||
'goal': question, 'kind': 'metrics', 'freshQuery': True,
|
||||
'dimensions': [], 'dimensionStages': {}, 'stages': ['black'],
|
||||
'fields': ['rate'], 'period': {'year': 2026, 'month': 10},
|
||||
'comparisonRequired': False, 'trendRequired': False,
|
||||
'verifySource': False, 'fullDetails': False,
|
||||
'visualPreference': 'none', 'responseStyle': 'standard',
|
||||
'needsClarification': False, 'clarification': '',
|
||||
'filterUpdates': {'product': '合成产品0'},
|
||||
}
|
||||
context = {'_protocol': {'version': 1, 'autonomousEnabled': True},
|
||||
'_businessCatalog': {'productMatch': unique}}
|
||||
with patch('nao_agent_task._model_json', return_value=planned):
|
||||
task = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
|
||||
question=question, context=context, cancel_event=threading.Event())
|
||||
self.assertFalse(task['needsClarification'])
|
||||
self.assertEqual(task['filters']['product'], '合成产品01')
|
||||
|
||||
def exercise(self, *, restart=False, revoke=False, other_release=False):
|
||||
opts = {'products': ['18X15X4+T', '18X15X4+Z'], 'organizationPaths': []}
|
||||
pending_match = identify_products('今年18x15x4合格率变化趋势', opts)['productMatch']
|
||||
|
||||
Reference in New Issue
Block a user