329 lines
20 KiB
Python
329 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Real, isolated MySQL application-update acceptance on the macOS build host.
|
|
|
|
Never reads the source deployment configuration or connects to its MySQL port.
|
|
The test copies the application and creates its own database server and ports.
|
|
Only HTTP transport and the target-runtime verify action are adapted for the
|
|
host; signature verification, backup, file replacement, startup and rollback
|
|
are the production updater implementation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import http.cookiejar
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import traceback
|
|
from unittest.mock import patch
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
|
|
|
|
import update_manager as update
|
|
|
|
SOURCE = Path(__file__).resolve().parents[2]
|
|
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
|
|
def free_port():
|
|
with socket.socket() as test:
|
|
test.bind(('127.0.0.1', 0))
|
|
return test.getsockname()[1]
|
|
|
|
|
|
def backend_port():
|
|
for port in (5285, 5284, 5283):
|
|
try:
|
|
with socket.socket() as test:
|
|
test.bind(('127.0.0.1', port))
|
|
return port
|
|
except OSError:
|
|
continue
|
|
raise RuntimeError('No unused standalone test backend port is available; no existing instance was touched.')
|
|
|
|
|
|
def query(root, sql, database=None):
|
|
config = update.read(root / '.runtime/deployment.json')
|
|
command = [str(root / 'runtimes/mysql/bin/mysql'), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
|
|
'--port=' + str(config['mysqlPort']), '--user=root', '--connect-timeout=5', '--batch', '--skip-column-names',
|
|
'--default-character-set=utf8mb4']
|
|
if database:
|
|
command.append(database)
|
|
env = os.environ.copy()
|
|
env['MYSQL_PWD'] = config['rootPassword']
|
|
result = subprocess.run(command, input=sql, encoding='utf-8', capture_output=True, env=env, timeout=60)
|
|
if result.returncode:
|
|
(root / '.runtime/update/test-sql-error.log').write_text(result.stderr)
|
|
raise RuntimeError('SQL failed in the isolated test server; see its private test log.')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def web(root, path='/'):
|
|
config = update.read(root / '.runtime/deployment.json')
|
|
with OPENER.open(f'http://127.0.0.1:{config["webPort"]}' + path, timeout=10) as response:
|
|
return response.read()
|
|
|
|
|
|
def login(root, username, password):
|
|
config = update.read(root / '.runtime/deployment.json')
|
|
base = f'http://127.0.0.1:{config["webPort"]}'
|
|
cookies = http.cookiejar.CookieJar()
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies))
|
|
with opener.open(base + '/api/auth/csrf', timeout=10) as response:
|
|
token = json.load(response)['token']
|
|
request = urllib.request.Request(base + '/api/auth/login', data=json.dumps({'username': username, 'password': password}).encode(),
|
|
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
|
|
with opener.open(request, timeout=15) as response:
|
|
assert response.status == 200
|
|
with opener.open(base + '/api/auth/me', timeout=10) as response:
|
|
assert response.status == 200
|
|
|
|
|
|
def prepare(work):
|
|
# MySQL's Unix socket has a 103-byte path limit on this host.
|
|
root = Path(tempfile.mkdtemp(prefix='qc-upd-', dir='/tmp')).resolve()
|
|
(work / 'isolated-root.txt').write_text(str(root) + '\n')
|
|
for name in ('app/web', 'scripts', 'deploy/offline', 'runtimes/python/bin', '.runtime/update',
|
|
'runtimedata/logs', 'runtimedata/sources', 'runtimedata/archive', 'backend/src/main/resources'):
|
|
(root / name).mkdir(parents=True, exist_ok=True)
|
|
for name in ('control.py', 'gateway.py', 'update_manager.py'):
|
|
shutil.copy2(SOURCE / 'deploy/offline' / name, root / 'deploy/offline' / name)
|
|
shutil.copy2(SOURCE / 'deploy/offline/runtime_platform.py', root / 'scripts/runtime_platform.py')
|
|
# No AI processes run in this acceptance test; preserve supervisor cleanup's
|
|
# normal subprocess path while keeping it confined to the disposable root.
|
|
(root / 'scripts/nao_quality_runtime.py').write_text('"""Isolated test: no AI processes have been started."""\n')
|
|
shutil.copy2(SOURCE / 'backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar', root / 'app/backend.jar')
|
|
shutil.copy2(SOURCE / 'backend/src/main/resources/analysis-runtime-policy.json', root / 'backend/src/main/resources/analysis-runtime-policy.json')
|
|
(root / 'app/web/index.html').write_text('<!doctype html><html><body><div id="root">isolated-update-v1</div></body></html>')
|
|
(root / 'app/web/obsolete.js').write_text('/* obsolete test application asset */')
|
|
(root / 'runtimes/java').symlink_to(SOURCE / '.runtime/jdk/Contents/Home', target_is_directory=True)
|
|
(root / 'runtimes/mysql').symlink_to(SOURCE / '.runtime/mysql', target_is_directory=True)
|
|
(root / 'runtimes/python/bin/python3').symlink_to(sys.executable)
|
|
key = Ed25519PrivateKey.generate()
|
|
(root / update.KEY).write_bytes(key.public_key().public_bytes(Encoding.PEM, PublicFormat.SubjectPublicKeyInfo))
|
|
config = {'mysqlPort': free_port(), 'backendPort': backend_port(), 'webPort': free_port(), 'webBind': '127.0.0.1',
|
|
'mysqlDatabase': 'quality_platform', 'archiveDirectory': 'runtimedata/archive',
|
|
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
|
|
'adminUsername': 'update-test-admin', 'adminPassword': secrets.token_urlsafe(24)}
|
|
assert config['mysqlPort'] != 53306
|
|
update.atomic(root / '.runtime/deployment.json', config)
|
|
(root / '.runtime/deployment.json').chmod(0o600)
|
|
private_marker = secrets.token_hex(20)
|
|
update.atomic(root / '.runtime/analysis-api-settings.json', {'testPrivateMarker': private_marker})
|
|
manifest = {'platform': 'darwin', 'architecture': 'arm64', 'updateVersion': 'host-initial', 'updateSequence': 1,
|
|
'runtimes': [{'name': 'java', 'hostTestRuntime': True}, {'name': 'mysql', 'hostTestRuntime': True},
|
|
{'name': 'python', 'version': sys.version.split()[0], 'hostTestRuntime': True}],
|
|
'pythonWheels': [], 'files': []}
|
|
for path in sorted(root.rglob('*')):
|
|
if path.is_file() and not path.is_symlink() and not path.relative_to(root).as_posix().startswith(('.runtime/', 'runtimedata/', 'runtimes/')):
|
|
manifest['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': update.digest(path)})
|
|
update.atomic(root / 'manifest.json', manifest)
|
|
return root, key, config, private_marker
|
|
|
|
|
|
def release_fixture(root, work, key, version, sequence, bad=False):
|
|
folder = work / ('fixture-' + version)
|
|
folder.mkdir()
|
|
program = {}
|
|
for entry in update.read(root / 'manifest.json')['files']:
|
|
name = entry['path']
|
|
if update.program_path(name) and name != 'app/web/obsolete.js':
|
|
program[name] = root / name
|
|
index = folder / 'index.html'
|
|
index.write_text(f'<!doctype html><html><body><div id="root">isolated-update-{version}</div></body></html>')
|
|
program['app/web/index.html'] = index
|
|
if bad:
|
|
bad_jar = folder / 'bad-backend.jar'
|
|
# The real new Java process runs Flyway, changes data, then fails. This
|
|
# proves rollback restores the dump, rather than merely restarting Java.
|
|
migration = ("UPDATE platform_metadata SET metadata_value='FAILED-UPDATE-DATA' WHERE metadata_key='update_integration_sentinel';\n"
|
|
"UPDATE app_user SET password_hash='FAILED-UPDATE-HASH' WHERE username='update-test-admin';\n"
|
|
"CREATE TABLE isolated_update_failure_marker(id INT PRIMARY KEY);\n"
|
|
"SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Intentional isolated updater integration failure';\n")
|
|
with zipfile.ZipFile(root / 'app/backend.jar') as source, zipfile.ZipFile(bad_jar, 'w') as target:
|
|
for member in source.infolist():
|
|
target.writestr(member, source.read(member))
|
|
target.writestr('BOOT-INF/classes/db/migration/V999__isolated_update_failure.sql', migration)
|
|
program['app/backend.jar'] = bad_jar
|
|
archive = folder / ('quality-update-' + version + '.zip')
|
|
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as target:
|
|
for name, path in sorted(program.items()):
|
|
target.write(path, name)
|
|
release = {'schema': 1, 'platform': 'windows-x64', 'version': version, 'sequence': sequence,
|
|
'runtimeFingerprint': update.runtime_fingerprint(update.read(root / 'manifest.json')),
|
|
'archive': {'file': archive.name, 'size': archive.stat().st_size, 'sha256': update.digest(archive)},
|
|
'files': [{'path': name, 'size': path.stat().st_size, 'sha256': update.digest(path)} for name, path in sorted(program.items())]}
|
|
payload = update.canonical(release)
|
|
envelope = {'payload': base64.b64encode(payload).decode(), 'signature': base64.b64encode(key.sign(payload)).decode()}
|
|
(folder / 'latest.json').write_bytes(update.canonical(envelope))
|
|
return release, envelope, archive
|
|
|
|
|
|
def run_update(root, release, envelope, archive):
|
|
real_controller = update.controller
|
|
verify_checks = []
|
|
|
|
def host_controller(target, action, database_only=False, timeout=1100):
|
|
assert target == root
|
|
if action != 'verify':
|
|
return real_controller(target, action, database_only, timeout)
|
|
# Host adaptation: this ARM Mac cannot satisfy Windows/x64 runtime probes.
|
|
# All installed program hashes and the native Java/MySQL executables are
|
|
# still checked. Real full application startup is checked below by serve.
|
|
for entry in update.read(root / 'manifest.json')['files']:
|
|
assert update.digest(root / entry['path']) == entry['sha256'], entry['path']
|
|
for command in ([root / 'runtimes/java/bin/java', '-version'],
|
|
[root / 'runtimes/mysql/bin/mysqld', '--no-defaults', '--version']):
|
|
result = subprocess.run([str(a) for a in command], capture_output=True, timeout=30)
|
|
assert result.returncode == 0
|
|
verify_checks.append(True)
|
|
|
|
def transport(url, target=None, expected_size=None, expected_hash=None):
|
|
if url == update.LATEST and target is None:
|
|
return update.canonical(envelope)
|
|
assert url == update.BASE + archive.name and target is not None
|
|
assert expected_size == archive.stat().st_size and expected_hash == update.digest(archive)
|
|
with archive.open('rb') as reader, target.open('xb') as writer:
|
|
shutil.copyfileobj(reader, writer)
|
|
return b''
|
|
|
|
launcher = subprocess.Popen([sys.executable, '-c', 'pass'])
|
|
launcher.wait(timeout=10)
|
|
token = secrets.token_hex(16)
|
|
update.atomic(root / '.runtime/update/request.json', {'token': token, 'created': time.time()})
|
|
# Patch only this test process. Production Windows/source enablement remains
|
|
# unchanged, including in the real backend processes started by control.py.
|
|
with patch.object(update, 'enabled', return_value=True), patch.object(update, 'download', side_effect=transport), \
|
|
patch.object(update, 'controller', side_effect=host_controller):
|
|
update.worker(launcher.pid, token, root)
|
|
assert verify_checks, update.read(root / '.runtime/update/status.json').get('message')
|
|
return update.read(root / '.runtime/update/status.json')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--work', type=Path, required=True)
|
|
parser.add_argument('--expected-tables', type=int, default=83)
|
|
args = parser.parse_args()
|
|
work = args.work.resolve()
|
|
work.mkdir(parents=True, exist_ok=False)
|
|
root = None
|
|
report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'passed': False,
|
|
'environment': 'macOS ARM host, disposable native MySQL + Java + Python runtimes',
|
|
'productionServicesTouched': False, 'productionDatabasePortUsed': False,
|
|
'networkTransport': 'local signed fixtures substituted only in test process',
|
|
'targetRuntimeVerify': 'host adapter: all manifest hashes plus native Java/MySQL probes; actual backend restarts',
|
|
'windowsNativeProcessDetachmentValidated': False, 'windowsNativeRuntimeCompatibilityValidated': False,
|
|
'checks': {}}
|
|
try:
|
|
print('Preparing isolated update acceptance root.', flush=True)
|
|
root, key, config, private_marker = prepare(work)
|
|
report['isolatedRoot'] = str(root)
|
|
report['ports'] = {k: config[k] for k in ('mysqlPort', 'backendPort', 'webPort')}
|
|
print('Starting isolated MySQL and Java; applying real Flyway schema.', flush=True)
|
|
process = update.controller(root, 'serve')
|
|
update.wait_ready(root, process)
|
|
database = config['mysqlDatabase']
|
|
count = int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{database}';"))
|
|
assert count == args.expected_tables, f'Expected {args.expected_tables} tables; actual isolated schema has {count}.'
|
|
report['checks']['tableCount'] = count
|
|
sentinel = secrets.token_hex(24)
|
|
query(root, f"INSERT INTO platform_metadata(metadata_key,metadata_value) VALUES ('update_integration_sentinel','{sentinel}');", database)
|
|
account_hash = query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database)
|
|
assert account_hash.startswith('$2')
|
|
login(root, config['adminUsername'], config['adminPassword'])
|
|
report['checks']['initialLogin'] = True
|
|
config['restoredFromMigration'] = 'isolated-update-migration-fixture'
|
|
update.atomic(root / '.runtime/deployment.json', config)
|
|
update.atomic(root / '.runtime/migration-receipt.json', {'migrationId': config['restoredFromMigration'],
|
|
'database': database, 'migrationManifestSha256': 'a' * 64})
|
|
before_pid = update.read(root / '.runtime/service.json')['supervisor']['pid']
|
|
print('Applying signed success release with actual dump, file replacement and service restart.', flush=True)
|
|
good = release_fixture(root, work, key, 'host-success', 2)
|
|
result = run_update(root, *good)
|
|
assert result['state'] == 'completed', result.get('message')
|
|
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
|
|
assert b'isolated-update-host-success' in web(root)
|
|
assert not (root / 'app/web/obsolete.js').exists()
|
|
assert update.read(root / '.runtime/service.json')['supervisor']['pid'] != before_pid
|
|
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == sentinel
|
|
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == account_hash
|
|
login(root, config['adminUsername'], config['adminPassword'])
|
|
first_journal = update.read(root / '.runtime/update/journal.json')
|
|
first_dump = root / '.runtime/update' / first_journal['backup'] / 'database.sql'
|
|
assert first_dump.is_file() and first_dump.stat().st_size > 10000
|
|
report['checks'].update(realMysqldump=True, successProgramSwitch=True, realServiceRestart=True,
|
|
obsoleteProgramRemoved=True, successSentinelPreserved=True,
|
|
successAccountHashPreserved=True, successLogin=True,
|
|
successDumpBytes=first_dump.stat().st_size)
|
|
print('Applying signed bad Java release: real Flyway mutates sentinel/hash then intentionally fails.', flush=True)
|
|
bad = release_fixture(root, work, key, 'host-bad', 3, bad=True)
|
|
result = run_update(root, *bad)
|
|
assert result['state'] == 'failed' and '已恢复原版本' in result['message'], result
|
|
restored = update.read(root / '.runtime/deployment.json')
|
|
restored_db = restored['mysqlDatabase']
|
|
assert restored_db != database and restored_db.startswith('quality_restore_')
|
|
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
|
|
assert b'isolated-update-host-success' in web(root)
|
|
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", restored_db) == sentinel
|
|
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", restored_db) == account_hash
|
|
assert int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{restored_db}';")) == count
|
|
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == 'FAILED-UPDATE-DATA', 'Failed original sentinel did not retain the migration write.'
|
|
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == 'FAILED-UPDATE-HASH', 'Failed original account hash did not retain the migration write.'
|
|
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999' AND success=0;", database) == '1'
|
|
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999';", restored_db) == '0'
|
|
assert update.read(root / '.runtime/analysis-api-settings.json')['testPrivateMarker'] == private_marker
|
|
assert restored['archiveDirectory'] == config['archiveDirectory']
|
|
assert restored['rootPassword'] == config['rootPassword'] and restored['mysqlPassword'] == config['mysqlPassword']
|
|
assert update.read(root / '.runtime/migration-receipt.json')['database'] == restored_db
|
|
login(root, config['adminUsername'], config['adminPassword'])
|
|
failed_journal = update.read(root / '.runtime/update/journal.json')
|
|
assert failed_journal['phase'] == 'rolledBack'
|
|
failed_dump = root / '.runtime/update' / failed_journal['backup'] / 'database.sql'
|
|
report['checks'].update(badReleaseActuallyMutatedDatabase=True, failedOriginalDatabasePreserved=True,
|
|
restoredIntoNewDatabase=True, restoredTableCount=count, rollbackSentinelRestored=True,
|
|
rollbackAccountHashRestored=True, rollbackLogin=True, oldProgramRestarted=True,
|
|
privateConfigurationPreserved=True, archiveDirectoryPreserved=True,
|
|
migrationReceiptDatabaseAligned=True,
|
|
failedFlywayHistoryPreserved=True, restoredFlywayHistoryClean=True,
|
|
rollbackDumpBytes=failed_dump.stat().st_size,
|
|
originalDatabase=database, rollbackDatabase=restored_db)
|
|
report['passed'] = True
|
|
print('Real MySQL success and failed-migration rollback acceptance passed.', flush=True)
|
|
except Exception as error:
|
|
report['error'] = str(error)
|
|
(work / 'test-failure-trace.txt').write_text(traceback.format_exc())
|
|
print('Isolated acceptance failed: ' + str(error), flush=True)
|
|
finally:
|
|
if root is not None:
|
|
try:
|
|
update.controller(root, 'stop', timeout=180)
|
|
state = update.read(root / '.runtime/service.json', {})
|
|
assert not update.live(state.get('supervisor', {}))
|
|
report['isolatedServicesStopped'] = True
|
|
except Exception as error:
|
|
report['isolatedServicesStopped'] = False
|
|
report['cleanupError'] = str(error)
|
|
report['passed'] = False
|
|
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
|
|
(work / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
|
print(json.dumps(report, ensure_ascii=False), flush=True)
|
|
return 0 if report['passed'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|