接入 DSH 专用运行时和对话界面,保留模型原始回答,支持会话分页、连续追问与显式报告。 补齐组合筛选、各维度 Top100、分组查询到排名图的转换,图表故障静默省略并基于保存数据自动恢复。 优化报告风格、图表顺序和表格换行,更新接入说明及构建输入。 验证:独立目录前后端构建通过;118 项 Java、102 项前端、102 项图表、14 项 DSH 应用和 13 项组合筛选检查通过。 仅提交相关源码、锁定依赖与测试;不包含凭据、会话、业务数据或本地产物。
275 lines
16 KiB
Python
275 lines
16 KiB
Python
"""Synthetic scale benchmark in a disposable database and a separate loopback API.
|
|
|
|
The live release and data are never modified. Replicas retain the sample's 15
|
|
source periods and full JSON column widths; they are not real annual business data.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import http.cookiejar
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
import secrets
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.parse import urlencode
|
|
from urllib.request import Request, build_opener, HTTPCookieProcessor
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import backup
|
|
import project
|
|
import managed_test_resources as test_resources
|
|
|
|
OUT = ROOT / 'output/completion'
|
|
|
|
|
|
def ident(value):
|
|
if not value.replace('_', '').isalnum():
|
|
raise ValueError('Invalid isolated database identifier')
|
|
return '`' + value + '`'
|
|
|
|
|
|
def sql(statement, database=None, empty=False):
|
|
create = re.fullmatch(r'CREATE DATABASE `?([a-z0-9_]+)`? CHARACTER SET [a-z0-9_]+(?: COLLATE [a-z0-9_]+)?;?', statement.strip(), flags=re.I)
|
|
if create:
|
|
return test_resources.create(ROOT, create[1], backup.client, statement, empty=empty)
|
|
drop = re.fullmatch(r'DROP DATABASE(?: IF EXISTS)? `?([a-z0-9_]+)`?;?', statement.strip(), flags=re.I)
|
|
if drop:
|
|
return test_resources.drop(ROOT, drop[1], backup.client)
|
|
result = backup.client(statement, database)
|
|
user = re.fullmatch(r"DROP USER(?: IF EXISTS)? '([a-z0-9_]+)'@'127\.0\.0\.1';?", statement.strip(), flags=re.I)
|
|
if user:
|
|
test_resources.user_dropped(ROOT, user[1])
|
|
return result
|
|
|
|
|
|
def clone_database(source, target):
|
|
tables = sql('SHOW TABLES;', source).splitlines()
|
|
for table in tables:
|
|
sql(f'CREATE TABLE {ident(target)}.{ident(table)} LIKE {ident(source)}.{ident(table)};')
|
|
if table not in ('fact_black', 'fact_bright'):
|
|
sql(f'INSERT INTO {ident(target)}.{ident(table)} SELECT * FROM {ident(source)}.{ident(table)};')
|
|
# A copied database must never resume the live installation's pending jobs.
|
|
for table in ('task', 'import_run'):
|
|
if table in tables:
|
|
sql(f"UPDATE {ident(table)} SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING','RETRY_WAIT','CANCEL_REQUESTED');", target)
|
|
for table in ('backup_plan', 'quality_inspection_strategy'):
|
|
if table in tables:
|
|
sql(f'UPDATE {ident(table)} SET active=FALSE;', target)
|
|
if 'analysis_turn' in tables and 'status' in [row.split('\t')[0] for row in sql('SHOW COLUMNS FROM analysis_turn;', target).splitlines()]:
|
|
sql("UPDATE analysis_turn SET status='CANCELLED' WHERE status IN ('QUEUED','RUNNING');", target)
|
|
return tables
|
|
|
|
|
|
def seed_facts(source, target, stage, count):
|
|
table = 'fact_' + stage
|
|
columns = [line.split('\t')[0] for line in sql(f'SHOW COLUMNS FROM {ident(table)};', source).splitlines()]
|
|
columns.remove('id')
|
|
# Replicate only the unique builds in the selected current release. Including
|
|
# older transform builds would inflate table size without exercising the
|
|
# promised number of query-visible records.
|
|
selected = "EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id)"
|
|
seed_count = int(sql(f'SELECT COUNT(*) FROM {ident(table)} f WHERE {selected};', source).strip())
|
|
if not seed_count:
|
|
raise RuntimeError('Source stage has no seed rows')
|
|
offset = int(sql(f'SELECT MAX(physical_row_no)+1 FROM {ident(table)};', source).strip())
|
|
repeats = math.ceil(count / seed_count)
|
|
if offset * (repeats + 1) > 2_147_483_647:
|
|
raise ValueError('Synthetic row identity exceeds the supported integer range')
|
|
sql('CREATE TEMPORARY TABLE scale_numbers (n INT PRIMARY KEY);' +
|
|
'INSERT INTO scale_numbers VALUES ' + ','.join(f'({i})' for i in range(repeats)) + ';' +
|
|
f'INSERT INTO {ident(table)} (' + ','.join(ident(c) for c in columns) + ') SELECT ' +
|
|
','.join(f'f.{ident(c)} + n.n*{offset}' if c == 'physical_row_no' else f'f.{ident(c)}' for c in columns) +
|
|
f' FROM {ident(source)}.{ident(table)} f CROSS JOIN scale_numbers n WHERE ' +
|
|
selected.replace('FROM release_member', 'FROM ' + ident(source) + '.release_member').replace('JOIN current_release', 'JOIN ' + ident(source) + '.current_release') +
|
|
f' ORDER BY n.n,f.id LIMIT {int(count)};', target)
|
|
return int(sql(f'SELECT COUNT(*) FROM {ident(table)};', target).strip())
|
|
|
|
|
|
class BenchApi:
|
|
def __init__(self, port, config):
|
|
self.base = f'http://127.0.0.1:{port}'
|
|
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
|
token = self.get('/api/auth/csrf')['token']
|
|
request = Request(self.base + '/api/auth/login',
|
|
data=json.dumps({'username': config['adminUsername'], 'password': config['adminPassword']}).encode(),
|
|
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
|
|
with self.opener.open(request, timeout=30) as response:
|
|
response.read()
|
|
|
|
def get(self, path, params=None):
|
|
request = Request(self.base + path + ('?' + urlencode(params) if params else ''),
|
|
headers={'Accept': 'application/json'})
|
|
with self.opener.open(request, timeout=180) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def percentile(values, fraction):
|
|
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None
|
|
|
|
|
|
def run(args):
|
|
if not args.run:
|
|
print('Plan only. Use --run to create an isolated synthetic database, benchmark, and remove it.')
|
|
return 0
|
|
if args.black < 1 or args.bright < 1 or args.black + args.bright > 2_000_000:
|
|
raise ValueError('Synthetic scale must be between two rows and two million rows')
|
|
if args.port == 5281 or not 1 <= args.concurrency <= 20:
|
|
raise ValueError('Use a separate loopback port and at most twenty workers')
|
|
with socket.socket() as probe:
|
|
if probe.connect_ex(('127.0.0.1', args.port)) == 0:
|
|
raise RuntimeError('The requested benchmark port is already in use')
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
config = project.config()
|
|
tag = datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
|
database = 'quality_scale_' + tag
|
|
account = 'qscale_' + uuid.uuid4().hex[:12]
|
|
password = secrets.token_urlsafe(32)
|
|
report = {'status': 'RUNNING', 'kind': 'SYNTHETIC_SAMPLE_REPLICATION',
|
|
'sourcePeriods': '15 provided black/bright periods retained; not a full real year',
|
|
'requestedRows': {'black': args.black, 'bright': args.bright},
|
|
'concurrency': args.concurrency, 'database': database,
|
|
'liveDatabaseModified': False, 'cleanup': {'database': False, 'account': False, 'process': False}}
|
|
result_path = OUT / 'scale-benchmark.json'
|
|
process = None
|
|
created_db = created_user = False
|
|
started = time.monotonic()
|
|
try:
|
|
report['livePointerBefore'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
|
sql(f'CREATE DATABASE {ident(database)} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
created_db = True
|
|
clone_database(config['mysqlDatabase'], database)
|
|
report['rows'] = {stage: seed_facts(config['mysqlDatabase'], database, stage, count)
|
|
for stage, count in (('black', args.black), ('bright', args.bright))}
|
|
report['currentReleaseRows'] = {stage: int(sql("SELECT COUNT(*) FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm JOIN current_release cr ON cr.release_id=rm.release_id AND cr.dataset_id='quality' WHERE rm.transform_build_id=f.build_id);", database).strip()) for stage in ('black', 'bright')}
|
|
if report['currentReleaseRows'] != report['rows']:
|
|
raise RuntimeError('Not all synthetic rows are visible in the benchmark release')
|
|
existing_tables = set(sql('SHOW TABLES;', database).splitlines())
|
|
# A copied 750-row materialization must never stand in for the 800k facts.
|
|
for materialization in ('metric_rollup', 'metric_rollup_build'):
|
|
if materialization in existing_tables:
|
|
sql('DELETE FROM ' + ident(materialization) + ';', database)
|
|
sql(f"CREATE USER '{account}'@'127.0.0.1' IDENTIFIED BY '{password}';")
|
|
created_user = True
|
|
test_resources.attach_user(ROOT, database, account)
|
|
sql(f"GRANT ALL PRIVILEGES ON {ident(database)}.* TO '{account}'@'127.0.0.1';")
|
|
jars = [p for p in (ROOT / 'backend/target').glob('*.jar') if not p.name.endswith('.original')]
|
|
if not jars:
|
|
raise RuntimeError('Build the backend before benchmarking')
|
|
jar = max(jars, key=lambda p: p.stat().st_mtime)
|
|
env = project.environment()
|
|
env.update({'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{config["mysqlPort"]}/{database}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
|
|
'MYSQL_USER': account, 'MYSQL_PASSWORD': password, 'PORT': str(args.port), 'SERVER_PORT': str(args.port),
|
|
'QUALITY_GOVERNANCE_BACKGROUND_ENABLED': 'false', 'QUALITY_BACKUP_DATABASE': database})
|
|
with (OUT / 'scale-backend.log').open('wb') as log:
|
|
process = subprocess.Popen([str(project.RUNTIME / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx768m', '-jar', str(jar)],
|
|
cwd=ROOT, env=env, stdout=log, stderr=log, start_new_session=True)
|
|
test_resources.attach_child(ROOT, database, process.pid)
|
|
preparing = time.monotonic()
|
|
project.wait_http(f'http://127.0.0.1:{args.port}/actuator/health', seconds=90)
|
|
if 'metric_rollup_build' in set(sql('SHOW TABLES;', database).splitlines()):
|
|
deadline = time.monotonic() + 300
|
|
while True:
|
|
status = sql("SELECT COUNT(DISTINCT rm.transform_build_id),COUNT(DISTINCT rb.build_id),COALESCE(SUM(rb.source_row_count),0) FROM current_release cr JOIN release_member rm ON rm.release_id=cr.release_id LEFT JOIN metric_rollup_build rb ON rb.build_id=rm.transform_build_id WHERE cr.dataset_id='quality';", database).strip().split('\t')
|
|
visible, rolled, rolled_rows = map(int, status)
|
|
if visible == rolled and rolled_rows == args.black + args.bright:
|
|
report['rollupPreparation'] = {'visibleBuilds':visible,'rolledBuilds':rolled,'sourceRows':rolled_rows,'seconds':round(time.monotonic()-preparing,4)}
|
|
break
|
|
if process.poll() is not None or time.monotonic() >= deadline:
|
|
raise RuntimeError('Synthetic rollup preparation did not cover all query-visible facts')
|
|
time.sleep(.25)
|
|
clients = [BenchApi(args.port, config) for _ in range(args.concurrency)]
|
|
common = {'year': 2026, 'releaseId': report['livePointerBefore'], 'includeUnassigned': 'true'}
|
|
cases = [('/api/metrics', {**common, 'view': view}) for view in ('overall', 'department', 'group', 'product')]
|
|
cases.extend(('/api/analytics/top', {**common, 'stage': stage, 'dimension': dimension, 'limit': 20})
|
|
for stage in ('black', 'bright') for dimension in ('category', 'product'))
|
|
def verify_visible_counts(result):
|
|
if 'summary' in result:
|
|
coverage = result.get('summary', {}).get('coverage', {})
|
|
valid = coverage.get('blackRows') == args.black and coverage.get('brightRows') == args.bright
|
|
else:
|
|
months = result.get('coverage', {}).get('sourceMonths', [])
|
|
valid = (sum(row.get('blackRows', 0) for row in months) == args.black
|
|
and sum(row.get('brightRows', 0) for row in months) == args.bright
|
|
and result.get('cumulativeThresholdApplied') is False and bool(result.get('resultId')))
|
|
if not valid: raise RuntimeError('The benchmark did not aggregate all requested synthetic records')
|
|
cold = []
|
|
for path, params in cases:
|
|
tick = time.monotonic()
|
|
result = clients[0].get(path, params)
|
|
if 'summary' not in result and 'resultId' not in result:
|
|
raise RuntimeError('Benchmark endpoint did not return metric data')
|
|
verify_visible_counts(result)
|
|
cold.append({'view': params.get('view', 'TOP_' + params.get('stage','') + '_' + params.get('dimension','')),
|
|
'seconds': round(time.monotonic() - tick, 4), 'rows': len(result.get('rows', []))})
|
|
report['firstQueryMeasurements'] = cold
|
|
# MySQL process/buffer state is shared with the local installation, so this
|
|
# is a first API query, not a claim that the physical cache was cold.
|
|
def worker(index):
|
|
timings = []
|
|
for n in range(args.rounds):
|
|
path, params = cases[(index + n) % len(cases)]
|
|
tick = time.monotonic()
|
|
result = clients[index].get(path, params)
|
|
if 'summary' not in result and 'resultId' not in result:
|
|
raise RuntimeError('Concurrent query returned no summary')
|
|
verify_visible_counts(result)
|
|
timings.append(time.monotonic() - tick)
|
|
return timings
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
timings = [item for group in pool.map(worker, range(args.concurrency)) for item in group]
|
|
report['concurrentSummary'] = {'requests': len(timings), 'p50Seconds': round(percentile(timings, .5), 4),
|
|
'p95Seconds': round(percentile(timings, .95), 4), 'maxSeconds': round(max(timings), 4),
|
|
'initialTwoSecondTargetMet': percentile(timings, .95) <= 2}
|
|
report['firstQueryTwoSecondTargetMet'] = all(item['seconds'] <= 2 for item in cold)
|
|
report['status'] = 'PASS' if report['concurrentSummary']['initialTwoSecondTargetMet'] and report['firstQueryTwoSecondTargetMet'] else 'PERFORMANCE_TARGET_NOT_MET'
|
|
report['limits'] = ['Synthetic replicas do not prove real annual data quality', 'No mixed AI/import concurrency in this run',
|
|
'The host MySQL buffer pool was not flushed', 'Production environment and business acceptance remain separate']
|
|
except Exception as failure:
|
|
report['status'] = 'ERROR'
|
|
report['errorType'] = type(failure).__name__
|
|
# Database/HTTP clients can include credentials in exception details.
|
|
report['error'] = 'Isolated benchmark failed; inspect the private local log without publishing credentials.'
|
|
finally:
|
|
if process is not None:
|
|
if process.poll() is None:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
try: process.wait(timeout=30)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait(timeout=10)
|
|
report['cleanup']['process'] = True
|
|
if created_db:
|
|
sql(f'DROP DATABASE {ident(database)};')
|
|
report['cleanup']['database'] = True
|
|
if created_user:
|
|
sql(f"DROP USER '{account}'@'127.0.0.1';")
|
|
report['cleanup']['account'] = True
|
|
report['livePointerAfter'] = sql("SELECT release_id FROM current_release WHERE dataset_id='quality';", config['mysqlDatabase']).strip()
|
|
report['livePointerUnchanged'] = report.get('livePointerBefore') == report['livePointerAfter']
|
|
report['elapsedSeconds'] = round(time.monotonic() - started, 3)
|
|
result_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
|
print(json.dumps({key: report.get(key) for key in ('status', 'rows', 'concurrentSummary', 'cleanup', 'livePointerUnchanged', 'elapsedSeconds')}, ensure_ascii=False))
|
|
return 0 if report['status'] == 'PASS' and report['livePointerUnchanged'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--run', action='store_true')
|
|
parser.add_argument('--black', type=int, default=500_000)
|
|
parser.add_argument('--bright', type=int, default=300_000)
|
|
parser.add_argument('--concurrency', type=int, default=10)
|
|
parser.add_argument('--rounds', type=int, default=3)
|
|
parser.add_argument('--port', type=int, default=5283)
|
|
sys.exit(run(parser.parse_args()))
|