完善:同月多文件导入与当前用户离线迁移更新

This commit is contained in:
2026-11-04 11:35:24 +08:00
parent 147b56d0fd
commit f04d4e96e3
89 changed files with 6634 additions and 201 deletions
+328
View File
@@ -0,0 +1,328 @@
#!/usr/bin/env python3
"""Real, isolated MySQL application-update acceptance on the macOS build host.
Never reads the source deployment configuration or connects to its MySQL port.
The test copies the application and creates its own database server and ports.
Only HTTP transport and the target-runtime verify action are adapted for the
host; signature verification, backup, file replacement, startup and rollback
are the production updater implementation.
"""
from __future__ import annotations
import argparse
import base64
from datetime import datetime, timezone
import hashlib
import http.cookiejar
import json
import os
from pathlib import Path
import secrets
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import traceback
from unittest.mock import patch
import urllib.request
import zipfile
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
import update_manager as update
SOURCE = Path(__file__).resolve().parents[2]
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def free_port():
with socket.socket() as test:
test.bind(('127.0.0.1', 0))
return test.getsockname()[1]
def backend_port():
for port in (5285, 5284, 5283):
try:
with socket.socket() as test:
test.bind(('127.0.0.1', port))
return port
except OSError:
continue
raise RuntimeError('No unused standalone test backend port is available; no existing instance was touched.')
def query(root, sql, database=None):
config = update.read(root / '.runtime/deployment.json')
command = [str(root / 'runtimes/mysql/bin/mysql'), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(config['mysqlPort']), '--user=root', '--connect-timeout=5', '--batch', '--skip-column-names',
'--default-character-set=utf8mb4']
if database:
command.append(database)
env = os.environ.copy()
env['MYSQL_PWD'] = config['rootPassword']
result = subprocess.run(command, input=sql, encoding='utf-8', capture_output=True, env=env, timeout=60)
if result.returncode:
(root / '.runtime/update/test-sql-error.log').write_text(result.stderr)
raise RuntimeError('SQL failed in the isolated test server; see its private test log.')
return result.stdout.strip()
def web(root, path='/'):
config = update.read(root / '.runtime/deployment.json')
with OPENER.open(f'http://127.0.0.1:{config["webPort"]}' + path, timeout=10) as response:
return response.read()
def login(root, username, password):
config = update.read(root / '.runtime/deployment.json')
base = f'http://127.0.0.1:{config["webPort"]}'
cookies = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies))
with opener.open(base + '/api/auth/csrf', timeout=10) as response:
token = json.load(response)['token']
request = urllib.request.Request(base + '/api/auth/login', data=json.dumps({'username': username, 'password': password}).encode(),
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
with opener.open(request, timeout=15) as response:
assert response.status == 200
with opener.open(base + '/api/auth/me', timeout=10) as response:
assert response.status == 200
def prepare(work):
# MySQL's Unix socket has a 103-byte path limit on this host.
root = Path(tempfile.mkdtemp(prefix='qc-upd-', dir='/tmp')).resolve()
(work / 'isolated-root.txt').write_text(str(root) + '\n')
for name in ('app/web', 'scripts', 'deploy/offline', 'runtimes/python/bin', '.runtime/update',
'runtimedata/logs', 'runtimedata/sources', 'runtimedata/archive', 'backend/src/main/resources'):
(root / name).mkdir(parents=True, exist_ok=True)
for name in ('control.py', 'gateway.py', 'update_manager.py'):
shutil.copy2(SOURCE / 'deploy/offline' / name, root / 'deploy/offline' / name)
shutil.copy2(SOURCE / 'deploy/offline/runtime_platform.py', root / 'scripts/runtime_platform.py')
# No AI processes run in this acceptance test; preserve supervisor cleanup's
# normal subprocess path while keeping it confined to the disposable root.
(root / 'scripts/nao_quality_runtime.py').write_text('"""Isolated test: no AI processes have been started."""\n')
shutil.copy2(SOURCE / 'backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar', root / 'app/backend.jar')
shutil.copy2(SOURCE / 'backend/src/main/resources/analysis-runtime-policy.json', root / 'backend/src/main/resources/analysis-runtime-policy.json')
(root / 'app/web/index.html').write_text('<!doctype html><html><body><div id="root">isolated-update-v1</div></body></html>')
(root / 'app/web/obsolete.js').write_text('/* obsolete test application asset */')
(root / 'runtimes/java').symlink_to(SOURCE / '.runtime/jdk/Contents/Home', target_is_directory=True)
(root / 'runtimes/mysql').symlink_to(SOURCE / '.runtime/mysql', target_is_directory=True)
(root / 'runtimes/python/bin/python3').symlink_to(sys.executable)
key = Ed25519PrivateKey.generate()
(root / update.KEY).write_bytes(key.public_key().public_bytes(Encoding.PEM, PublicFormat.SubjectPublicKeyInfo))
config = {'mysqlPort': free_port(), 'backendPort': backend_port(), 'webPort': free_port(), 'webBind': '127.0.0.1',
'mysqlDatabase': 'quality_platform', 'archiveDirectory': 'runtimedata/archive',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'update-test-admin', 'adminPassword': secrets.token_urlsafe(24)}
assert config['mysqlPort'] != 53306
update.atomic(root / '.runtime/deployment.json', config)
(root / '.runtime/deployment.json').chmod(0o600)
private_marker = secrets.token_hex(20)
update.atomic(root / '.runtime/analysis-api-settings.json', {'testPrivateMarker': private_marker})
manifest = {'platform': 'darwin', 'architecture': 'arm64', 'updateVersion': 'host-initial', 'updateSequence': 1,
'runtimes': [{'name': 'java', 'hostTestRuntime': True}, {'name': 'mysql', 'hostTestRuntime': True},
{'name': 'python', 'version': sys.version.split()[0], 'hostTestRuntime': True}],
'pythonWheels': [], 'files': []}
for path in sorted(root.rglob('*')):
if path.is_file() and not path.is_symlink() and not path.relative_to(root).as_posix().startswith(('.runtime/', 'runtimedata/', 'runtimes/')):
manifest['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': update.digest(path)})
update.atomic(root / 'manifest.json', manifest)
return root, key, config, private_marker
def release_fixture(root, work, key, version, sequence, bad=False):
folder = work / ('fixture-' + version)
folder.mkdir()
program = {}
for entry in update.read(root / 'manifest.json')['files']:
name = entry['path']
if update.program_path(name) and name != 'app/web/obsolete.js':
program[name] = root / name
index = folder / 'index.html'
index.write_text(f'<!doctype html><html><body><div id="root">isolated-update-{version}</div></body></html>')
program['app/web/index.html'] = index
if bad:
bad_jar = folder / 'bad-backend.jar'
# The real new Java process runs Flyway, changes data, then fails. This
# proves rollback restores the dump, rather than merely restarting Java.
migration = ("UPDATE platform_metadata SET metadata_value='FAILED-UPDATE-DATA' WHERE metadata_key='update_integration_sentinel';\n"
"UPDATE app_user SET password_hash='FAILED-UPDATE-HASH' WHERE username='update-test-admin';\n"
"CREATE TABLE isolated_update_failure_marker(id INT PRIMARY KEY);\n"
"SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Intentional isolated updater integration failure';\n")
with zipfile.ZipFile(root / 'app/backend.jar') as source, zipfile.ZipFile(bad_jar, 'w') as target:
for member in source.infolist():
target.writestr(member, source.read(member))
target.writestr('BOOT-INF/classes/db/migration/V999__isolated_update_failure.sql', migration)
program['app/backend.jar'] = bad_jar
archive = folder / ('quality-update-' + version + '.zip')
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as target:
for name, path in sorted(program.items()):
target.write(path, name)
release = {'schema': 1, 'platform': 'windows-x64', 'version': version, 'sequence': sequence,
'runtimeFingerprint': update.runtime_fingerprint(update.read(root / 'manifest.json')),
'archive': {'file': archive.name, 'size': archive.stat().st_size, 'sha256': update.digest(archive)},
'files': [{'path': name, 'size': path.stat().st_size, 'sha256': update.digest(path)} for name, path in sorted(program.items())]}
payload = update.canonical(release)
envelope = {'payload': base64.b64encode(payload).decode(), 'signature': base64.b64encode(key.sign(payload)).decode()}
(folder / 'latest.json').write_bytes(update.canonical(envelope))
return release, envelope, archive
def run_update(root, release, envelope, archive):
real_controller = update.controller
verify_checks = []
def host_controller(target, action, database_only=False, timeout=1100):
assert target == root
if action != 'verify':
return real_controller(target, action, database_only, timeout)
# Host adaptation: this ARM Mac cannot satisfy Windows/x64 runtime probes.
# All installed program hashes and the native Java/MySQL executables are
# still checked. Real full application startup is checked below by serve.
for entry in update.read(root / 'manifest.json')['files']:
assert update.digest(root / entry['path']) == entry['sha256'], entry['path']
for command in ([root / 'runtimes/java/bin/java', '-version'],
[root / 'runtimes/mysql/bin/mysqld', '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, timeout=30)
assert result.returncode == 0
verify_checks.append(True)
def transport(url, target=None, expected_size=None, expected_hash=None):
if url == update.LATEST and target is None:
return update.canonical(envelope)
assert url == update.BASE + archive.name and target is not None
assert expected_size == archive.stat().st_size and expected_hash == update.digest(archive)
with archive.open('rb') as reader, target.open('xb') as writer:
shutil.copyfileobj(reader, writer)
return b''
launcher = subprocess.Popen([sys.executable, '-c', 'pass'])
launcher.wait(timeout=10)
token = secrets.token_hex(16)
update.atomic(root / '.runtime/update/request.json', {'token': token, 'created': time.time()})
# Patch only this test process. Production Windows/source enablement remains
# unchanged, including in the real backend processes started by control.py.
with patch.object(update, 'enabled', return_value=True), patch.object(update, 'download', side_effect=transport), \
patch.object(update, 'controller', side_effect=host_controller):
update.worker(launcher.pid, token, root)
assert verify_checks, update.read(root / '.runtime/update/status.json').get('message')
return update.read(root / '.runtime/update/status.json')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--work', type=Path, required=True)
parser.add_argument('--expected-tables', type=int, default=83)
args = parser.parse_args()
work = args.work.resolve()
work.mkdir(parents=True, exist_ok=False)
root = None
report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'passed': False,
'environment': 'macOS ARM host, disposable native MySQL + Java + Python runtimes',
'productionServicesTouched': False, 'productionDatabasePortUsed': False,
'networkTransport': 'local signed fixtures substituted only in test process',
'targetRuntimeVerify': 'host adapter: all manifest hashes plus native Java/MySQL probes; actual backend restarts',
'windowsNativeProcessDetachmentValidated': False, 'windowsNativeRuntimeCompatibilityValidated': False,
'checks': {}}
try:
print('Preparing isolated update acceptance root.', flush=True)
root, key, config, private_marker = prepare(work)
report['isolatedRoot'] = str(root)
report['ports'] = {k: config[k] for k in ('mysqlPort', 'backendPort', 'webPort')}
print('Starting isolated MySQL and Java; applying real Flyway schema.', flush=True)
process = update.controller(root, 'serve')
update.wait_ready(root, process)
database = config['mysqlDatabase']
count = int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{database}';"))
assert count == args.expected_tables, f'Expected {args.expected_tables} tables; actual isolated schema has {count}.'
report['checks']['tableCount'] = count
sentinel = secrets.token_hex(24)
query(root, f"INSERT INTO platform_metadata(metadata_key,metadata_value) VALUES ('update_integration_sentinel','{sentinel}');", database)
account_hash = query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database)
assert account_hash.startswith('$2')
login(root, config['adminUsername'], config['adminPassword'])
report['checks']['initialLogin'] = True
config['restoredFromMigration'] = 'isolated-update-migration-fixture'
update.atomic(root / '.runtime/deployment.json', config)
update.atomic(root / '.runtime/migration-receipt.json', {'migrationId': config['restoredFromMigration'],
'database': database, 'migrationManifestSha256': 'a' * 64})
before_pid = update.read(root / '.runtime/service.json')['supervisor']['pid']
print('Applying signed success release with actual dump, file replacement and service restart.', flush=True)
good = release_fixture(root, work, key, 'host-success', 2)
result = run_update(root, *good)
assert result['state'] == 'completed', result.get('message')
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
assert b'isolated-update-host-success' in web(root)
assert not (root / 'app/web/obsolete.js').exists()
assert update.read(root / '.runtime/service.json')['supervisor']['pid'] != before_pid
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == sentinel
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == account_hash
login(root, config['adminUsername'], config['adminPassword'])
first_journal = update.read(root / '.runtime/update/journal.json')
first_dump = root / '.runtime/update' / first_journal['backup'] / 'database.sql'
assert first_dump.is_file() and first_dump.stat().st_size > 10000
report['checks'].update(realMysqldump=True, successProgramSwitch=True, realServiceRestart=True,
obsoleteProgramRemoved=True, successSentinelPreserved=True,
successAccountHashPreserved=True, successLogin=True,
successDumpBytes=first_dump.stat().st_size)
print('Applying signed bad Java release: real Flyway mutates sentinel/hash then intentionally fails.', flush=True)
bad = release_fixture(root, work, key, 'host-bad', 3, bad=True)
result = run_update(root, *bad)
assert result['state'] == 'failed' and '已恢复原版本' in result['message'], result
restored = update.read(root / '.runtime/deployment.json')
restored_db = restored['mysqlDatabase']
assert restored_db != database and restored_db.startswith('quality_restore_')
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
assert b'isolated-update-host-success' in web(root)
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", restored_db) == sentinel
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", restored_db) == account_hash
assert int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{restored_db}';")) == count
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == 'FAILED-UPDATE-DATA', 'Failed original sentinel did not retain the migration write.'
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == 'FAILED-UPDATE-HASH', 'Failed original account hash did not retain the migration write.'
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999' AND success=0;", database) == '1'
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999';", restored_db) == '0'
assert update.read(root / '.runtime/analysis-api-settings.json')['testPrivateMarker'] == private_marker
assert restored['archiveDirectory'] == config['archiveDirectory']
assert restored['rootPassword'] == config['rootPassword'] and restored['mysqlPassword'] == config['mysqlPassword']
assert update.read(root / '.runtime/migration-receipt.json')['database'] == restored_db
login(root, config['adminUsername'], config['adminPassword'])
failed_journal = update.read(root / '.runtime/update/journal.json')
assert failed_journal['phase'] == 'rolledBack'
failed_dump = root / '.runtime/update' / failed_journal['backup'] / 'database.sql'
report['checks'].update(badReleaseActuallyMutatedDatabase=True, failedOriginalDatabasePreserved=True,
restoredIntoNewDatabase=True, restoredTableCount=count, rollbackSentinelRestored=True,
rollbackAccountHashRestored=True, rollbackLogin=True, oldProgramRestarted=True,
privateConfigurationPreserved=True, archiveDirectoryPreserved=True,
migrationReceiptDatabaseAligned=True,
failedFlywayHistoryPreserved=True, restoredFlywayHistoryClean=True,
rollbackDumpBytes=failed_dump.stat().st_size,
originalDatabase=database, rollbackDatabase=restored_db)
report['passed'] = True
print('Real MySQL success and failed-migration rollback acceptance passed.', flush=True)
except Exception as error:
report['error'] = str(error)
(work / 'test-failure-trace.txt').write_text(traceback.format_exc())
print('Isolated acceptance failed: ' + str(error), flush=True)
finally:
if root is not None:
try:
update.controller(root, 'stop', timeout=180)
state = update.read(root / '.runtime/service.json', {})
assert not update.live(state.get('supervisor', {}))
report['isolatedServicesStopped'] = True
except Exception as error:
report['isolatedServicesStopped'] = False
report['cleanupError'] = str(error)
report['passed'] = False
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
(work / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(json.dumps(report, ensure_ascii=False), flush=True)
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())