Files
quality_analyse/deploy/offline/restore_migration.py
T

422 lines
23 KiB
Python

"""Restore into a new database, validate, then switch the Windows deployment."""
from __future__ import annotations
from datetime import datetime, timezone
import gzip
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import sqlite3
import subprocess
import sys
import time
import uuid
import control
from migration_common import extract_checked, query_evidence, relative_name, sha256, sql_text, table_counts
ROOT = control.ROOT
PAYLOAD = ROOT / 'migration'
JOURNAL = control.PRIVATE / 'migration-progress.json'
RECEIPT = control.PRIVATE / 'migration-receipt.json'
def checked_local(relative):
rel = relative_name(relative)
path = ROOT.joinpath(*rel.parts)
if not path.resolve().is_relative_to(ROOT.resolve()):
raise RuntimeError('恢复暂存路径越界。')
return path
def run_query(value, statement, database=None):
command = control.mysql_command(value)
command += ['--batch', '--skip-column-names', '--default-character-set=utf8mb4', '--max-allowed-packet=1G']
if database:
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', database):
raise RuntimeError('恢复数据库名称无效。')
command += ['--database=' + database]
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
result = subprocess.run(command, input=statement, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=1800)
if result.returncode:
raise RuntimeError('恢复数据库校验未完成;已有数据库保留。')
return result.stdout
def stop_owned_service():
state = control.read_state()
if control.matches(state.get('supervisor', {})):
control.atomic(control.STOP, {'token': state['token']})
for _ in range(300):
if not control.matches(state['supervisor']):
break
time.sleep(.5)
else:
raise RuntimeError('原服务仍在关闭,请稍后重新运行。')
if any(control.matches(info) for info in state.get('children', [])):
raise RuntimeError('发现本目录尚未关闭的后台进程,请检查服务日志。')
def start_database(value):
with (control.DATA / 'logs/migration-database-supervisor.log').open('ab') as log:
options = ({'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
if control.IS_WINDOWS else {'start_new_session': True})
process = subprocess.Popen([str(control.PYTHON), str(Path(control.__file__).resolve()),
'serve', '--database-only'], cwd=ROOT, env=control.environment(value),
stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
for _ in range(360):
state = control.read_state()
if (state.get('ready') and state.get('databaseOnly') and
state.get('supervisor', {}).get('pid') == process.pid and control.matches(state['supervisor'])):
return
if process.poll() is not None:
raise RuntimeError('恢复数据库无法启动,请查看 runtimedata/logs。')
time.sleep(.5)
raise RuntimeError('恢复数据库仍未就绪,请查看日志后重试。')
def restore_dump(value, database, dump):
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
command = control.mysql_command(value) + ['--database=' + database,
'--default-character-set=utf8mb4', '--max-allowed-packet=1G',
'--init-command=SET SESSION autocommit=0']
with (control.DATA / 'logs/migration-import.log').open('ab') as errors:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors, env=env)
try:
total, next_progress = dump.stat().st_size, 0
with dump.open('rb') as raw, gzip.GzipFile(fileobj=raw) as source:
while chunk := source.read(1024 * 1024):
process.stdin.write(chunk)
percent = min(99, raw.tell() * 100 // total)
if percent >= next_progress:
print(f'数据库导入进度约 {percent}%……', flush=True)
next_progress = percent + 10
process.stdin.write(b'\nCOMMIT;\n')
process.stdin.close()
if process.wait(timeout=1800):
raise RuntimeError('业务数据导入失败,请查看私有恢复日志;原数据库未覆盖。')
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
def rebind_sources(query, manifest, archive):
counts = {'source_revision': 0, 'source_candidate': 0, 'source_upload_draft': 0}
for entry in manifest['pathBindings']:
pair = (entry['table'], entry['column'])
if pair not in {('source_revision', 'archive_path'), ('source_candidate', 'source_path'),
('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}:
raise RuntimeError('来源路径绑定类型无效。')
rel = relative_name(entry['relative'])
target = archive.joinpath(*rel.parts)
if not entry['expiredMissing']:
if not target.is_file() or target.stat().st_size != entry['bytes'] or sha256(target) != entry['sha256']:
raise RuntimeError('来源文件校验失败,恢复未切换。')
query('UPDATE `' + entry['table'] + '` SET `' + entry['column'] + '`=' + sql_text(str(target))
+ ' WHERE id=' + sql_text(entry['id']) + ' AND `' + entry['column'] + '`=' + sql_text(entry['oldPath']) + ';')
actual = query('SELECT HEX(`' + entry['column'] + '`) FROM `' + entry['table'] + '` WHERE id=' + sql_text(entry['id']) + ';').strip().lower()
# 按 UTF-8 十六进制核对,避免 mysql --batch 转义 Windows 反斜杠。
if actual != str(target).encode('utf-8').hex():
raise RuntimeError('来源文件路径重绑定失败。')
counts[entry['table']] += 1
return counts
def restored_source_paths(query, manifest):
"""批量读取原始路径;HEX 避免 Windows 路径受到客户端转义影响。"""
allowed = {('source_revision', 'archive_path'), ('source_candidate', 'source_path'),
('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}
result = {}
pairs = {(entry['table'], entry['column']) for entry in manifest['pathBindings']}
if not pairs.issubset(allowed):
raise RuntimeError('来源路径绑定类型无效。')
for table, column in sorted(pairs):
values = {}
for line in query('SELECT id,HEX(`' + column + '`) FROM `' + table + '` WHERE `' + column + '` IS NOT NULL;').splitlines():
identity, encoded = line.split('\t', 1)
values[identity] = bytes.fromhex(encoded).decode('utf-8')
for entry in manifest['pathBindings']:
if (entry['table'], entry['column']) == (table, column):
if entry['id'] not in values:
raise RuntimeError('恢复库缺少已登记的来源引用。')
result[(table, column, entry['id'])] = values[entry['id']]
return result
def reusable_archive(manifest, paths):
"""只复用本部署目录此前恢复生成、且全部文件通过校验的归档。"""
parent = (control.DATA / 'migrations').resolve()
attempts = set()
for entry in manifest['pathBindings']:
actual = paths[(entry['table'], entry['column'], entry['id'])]
if actual == entry['oldPath']:
continue
relative = relative_name(entry['relative'])
path = Path(actual)
if not path.is_absolute() or not path.resolve().is_relative_to(parent):
raise RuntimeError('恢复库来源路径不属于本目录。')
parts = path.relative_to(parent).parts
if (len(parts) != len(relative.parts) + 2 or not re.fullmatch(r'[a-f0-9]{16}', parts[0])
or parts[1] != 'archive' or tuple(parts[2:]) != relative.parts):
raise RuntimeError('恢复库来源路径与原迁移包不一致。')
attempts.add(parts[0])
if len(attempts) != 1:
raise RuntimeError('无法确定上次恢复的来源归档,未重新导入。')
archive = parent / next(iter(attempts)) / 'archive'
observed = {path.relative_to(archive).as_posix() for path in archive.rglob('*') if path.is_file()}
if observed != set(manifest['archiveFiles']):
raise RuntimeError('上次恢复的来源文件数量不一致。')
for name, expected in manifest['archiveFiles'].items():
target = archive.joinpath(*relative_name(name).parts)
if (target.is_symlink() or not target.resolve().is_relative_to(archive.resolve())
or target.stat().st_size != expected['bytes'] or sha256(target) != expected['sha256']):
raise RuntimeError('上次恢复的来源文件未通过校验。')
return archive
def select_reusable_import(value, manifest):
"""复核完整导入及来源文件后复用暂存库;绝不回退为重新导入。"""
names = run_query(value, 'SHOW DATABASES;').splitlines()
candidates = [name for name in names if re.fullmatch(r'quality_restore_[a-f0-9]{12,32}', name)
and name != value.get('mysqlDatabase')]
verified = []
for name in candidates:
query = lambda statement: run_query(value, statement, name)
try:
paths = restored_source_paths(query, manifest)
archive = reusable_archive(manifest, paths)
counts = table_counts(query)
if counts != manifest['tableCounts']:
continue
users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()]
if sorted(users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']):
continue
if query_evidence(query) != manifest['queryEvidence']:
continue
# 已确认原路径或同一受控归档路径,后续仍以当前值作为条件更新。
bound = {**manifest, 'pathBindings': [{**entry, 'oldPath': paths[(entry['table'], entry['column'], entry['id'])]}
for entry in manifest['pathBindings']]}
verified.append((archive.stat().st_mtime_ns, name, archive, counts, bound))
except (RuntimeError, ValueError, OSError):
continue
if not verified:
raise RuntimeError('未找到通过完整校验的已导入恢复库。已有数据库保留,本次未重新导入;请保留 runtimedata/logs 供核查。')
_, database, archive, counts, bound = max(verified, key=lambda item: item[0])
print('已核验并复用上次导入完成的数据库,跳过数据库重新导入。', flush=True)
return database, archive, counts, bound
def relocate_native(stage, source_root):
source_root = PurePosixPath(source_root)
def relocated(value):
if isinstance(value, str) and value.startswith(str(source_root) + '/'):
relative = PurePosixPath(value).relative_to(source_root)
if '..' in relative.parts:
raise RuntimeError('原生历史路径无效。')
return str(ROOT.joinpath(*relative.parts))
if isinstance(value, dict):
return {key: relocated(child) for key, child in value.items()}
if isinstance(value, list):
return [relocated(child) for child in value]
return value
for path in stage.rglob('*.json'):
control.atomic(path, relocated(json.loads(path.read_text(encoding='utf-8'))))
databases = 0
for path in stage.rglob('nao.sqlite'):
connection = sqlite3.connect(path)
try:
connection.execute('PRAGMA secure_delete=ON')
for table, column in (('project', 'path'), ('message_part', 'storage_path')):
exists = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
if not exists:
continue
for rowid, original in connection.execute('SELECT rowid,"' + column + '" FROM "' + table + '" WHERE "' + column + '" IS NOT NULL').fetchall():
replacement = relocated(original)
if replacement != original:
connection.execute('UPDATE "' + table + '" SET "' + column + '"=? WHERE rowid=?', (replacement, rowid))
connection.commit()
connection.execute('VACUUM')
if connection.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
raise RuntimeError('原生分析历史数据库校验失败。')
databases += 1
finally:
connection.close()
return databases
def rollback_plan(plan):
# Only paths generated by this migration are eligible. Databases and source
# archive trees are retained even after an interrupted restore.
attempt = plan['attempt']
if not re.fullmatch(r'[a-f0-9]{16}', attempt):
raise RuntimeError('恢复日志无效。')
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
failed = checked_local(item['failed'])
if previous.exists():
if target.exists():
target.rename(failed)
previous.rename(target)
elif not item['existed'] and not staged.exists() and target.exists():
target.rename(failed)
control.atomic(control.CONFIG, plan['previousConfig'])
JOURNAL.rename(control.PRIVATE / ('migration-rolled-back-' + attempt + '.json'))
def commit_plan(plan, receipt):
control.atomic(JOURNAL, plan)
try:
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
if target.is_symlink() or staged.is_symlink() or previous.exists():
raise RuntimeError('私有配置切换路径无效。')
if target.exists():
target.rename(previous)
staged.rename(target)
control.atomic(control.CONFIG, plan['newConfig'])
control.atomic(RECEIPT, receipt)
except BaseException:
rollback_plan(plan)
raise
JOURNAL.unlink(missing_ok=True)
def restore(check_runtime=True):
control.private_directory(control.PRIVATE)
control.private_directory(control.DATA)
(control.DATA / 'logs').mkdir(exist_ok=True)
(control.DATA / 'sources').mkdir(exist_ok=True)
(control.DATA / 'archive').mkdir(exist_ok=True)
lock = os.open(control.PRIVATE / 'migration.lock', os.O_CREAT | os.O_RDWR, 0o600)
try:
control.host.fcntl.flock(lock, control.host.fcntl.LOCK_EX | control.host.fcntl.LOCK_NB)
except BlockingIOError:
os.close(lock)
raise RuntimeError('另一项迁移正在进行,请等待原窗口完成。') from None
try:
return restore_locked(check_runtime)
finally:
os.close(lock)
def restore_locked(check_runtime):
manifest_path = PAYLOAD / 'manifest.json'
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
if manifest.get('formatVersion') != 1 or not re.fullmatch(r'[a-f0-9]{32}', manifest.get('migrationId', '')):
raise RuntimeError('迁移包格式无效。')
migration_id = manifest['migrationId']
if RECEIPT.exists():
receipt = json.loads(RECEIPT.read_text(encoding='utf-8'))
current = control.configuration()
if receipt.get('migrationId') == migration_id and current.get('restoredFromMigration') == migration_id and current.get('mysqlDatabase') == receipt.get('database'):
print('此迁移已完成,保留 Windows 上的后续数据,直接启动。', flush=True)
JOURNAL.unlink(missing_ok=True)
return receipt
raise RuntimeError('此目录已有其他迁移记录,请使用新的部署目录。')
stop_owned_service()
if JOURNAL.exists():
rollback_plan(json.loads(JOURNAL.read_text(encoding='utf-8')))
print('上次未完成的切换已撤回,原配置已保留。', flush=True)
value = control.verify() if check_runtime else control.configuration()
print('正在校验迁移包……', flush=True)
for name, key in (('database.sql.gz', 'databaseDumpSha256'), ('archive.tar.gz', 'archiveSha256'),
('native-history.tar.gz', 'nativeHistorySha256')):
if sha256(PAYLOAD / name) != manifest[key]:
raise RuntimeError('迁移包文件不完整,已停止恢复。')
if manifest.get('modelConfigSha256') and sha256(PAYLOAD / 'model-config.json') != manifest['modelConfigSha256']:
raise RuntimeError('模型配置校验失败。')
reuse_only = os.environ.get('QUALITY_MIGRATION_REUSE_ONLY') == '1'
required_free = (sum(item['bytes'] for item in manifest['nativeHistoryFiles'].values()) + 128 * 1024**2
if reuse_only else manifest['minimumFreeBytes'])
if shutil.disk_usage(ROOT).free < required_free:
raise RuntimeError('可用磁盘空间不足,请至少预留 ' + str((required_free + 1024**3 - 1) // 1024**3) + ' GB。')
attempt = uuid.uuid4().hex[:16]
stage = control.PRIVATE / ('migration-stage-' + attempt)
stage.mkdir(mode=0o700)
archive = control.DATA / 'migrations' / attempt / 'archive'
archive.parent.mkdir(parents=True, mode=0o700)
native = stage / 'nao-users'
print('正在恢复来源文件和分析历史……', flush=True)
if not reuse_only:
extract_checked(PAYLOAD / 'archive.tar.gz', archive, manifest['archiveFiles'])
extract_checked(PAYLOAD / 'native-history.tar.gz', native, manifest['nativeHistoryFiles'])
native_count = relocate_native(native, manifest['sourceProjectRoot'])
if native_count != manifest['nativeHistoryDatabases']:
raise RuntimeError('分析历史文件数量不一致。')
model = stage / 'model-config.json'
if manifest.get('modelConfigSha256'):
shutil.copy2(PAYLOAD / 'model-config.json', model)
model.chmod(0o600)
settings = json.loads(model.read_text(encoding='utf-8'))
if any('mimo' in str(profile.get(key, '')).lower() for profile in settings['profiles'] for key in ('model', 'name', 'baseUrl')):
raise RuntimeError('迁移模型配置中仍含 MiMo。')
database = 'quality_restore_' + uuid.uuid4().hex
try:
start_database(value)
if reuse_only:
print('正在核验上次已导入的数据库和来源文件,不会重新导入……', flush=True)
database, archive, actual_counts, manifest = select_reusable_import(value, manifest)
query = lambda statement: run_query(value, statement, database)
else:
run_query(value, 'CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
restore_dump(value, database, PAYLOAD / 'database.sql.gz')
query = lambda statement: run_query(value, statement, database)
print('正在核对全部数据表、原账号和正式统计结果……', flush=True)
actual_counts = table_counts(query)
if actual_counts != manifest['tableCounts']:
raise RuntimeError('恢复后的数据表行数不一致,未切换正式使用库。')
actual_users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()]
if sorted(actual_users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']):
raise RuntimeError('原用户账号校验失败,未切换使用库。')
if query_evidence(query) != manifest['queryEvidence']:
raise RuntimeError('正式版本和月度统计结果不一致,未切换使用库。')
rebound = rebind_sources(query, manifest, archive)
run_query(value, 'GRANT ALL PRIVILEGES ON `' + database + "`.* TO 'quality_app'@'127.0.0.1';")
run_query(value, 'FLUSH TABLES;')
finally:
stop_owned_service()
new_config = {**value, 'mysqlDatabase': database, 'archiveDirectory': archive.relative_to(ROOT).as_posix(),
'restoredFromMigration': migration_id, 'webBind': '0.0.0.0'}
for name in ('dataWorkflowTestMode', 'analysisAutonomousEnabled', 'analysisProtocolV1Enabled'):
if name in manifest.get('applicationSettings', {}):
new_config[name] = bool(manifest['applicationSettings'][name])
switches = []
for staged, target in [(native, control.PRIVATE / 'nao-users')] + ([(model, control.PRIVATE / 'nao-model-config.json')] if model.exists() else []):
switches.append({'staged': staged.relative_to(ROOT).as_posix(), 'target': target.relative_to(ROOT).as_posix(),
'previous': (control.PRIVATE / ('before-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'failed': (control.PRIVATE / ('failed-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'existed': target.exists()})
receipt = {'migrationId': migration_id, 'migrationManifestSha256': sha256(manifest_path), 'database': database,
'restoredAtUtc': datetime.now(timezone.utc).isoformat(), 'tables': len(actual_counts),
'totalRows': sum(actual_counts.values()), 'sourceReferencesRebound': rebound,
'modelProfilesMigrated': manifest['modelProfilesMigrated'], 'modelProfilesExcluded': manifest['modelProfilesExcluded'],
'nativeHistoryDatabases': native_count, 'originalDatabasesPreserved': True,
'archiveDirectory': new_config['archiveDirectory'], 'reusedCompletedImport': reuse_only}
plan = {'attempt': attempt, 'previousConfig': value, 'newConfig': new_config, 'privateSwitches': switches}
commit_plan(plan, receipt)
print('迁移恢复完成。请使用原来的账号和密码登录;MiMo 配置未迁移。', flush=True)
return receipt
if __name__ == '__main__':
try:
restore()
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error:
import traceback
log = control.DATA / 'logs/migration-validation.log'
log.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as output:
traceback.print_exc(file=output)
if isinstance(error, RuntimeError):
print(str(error), file=sys.stderr)
print('迁移未完成。已有数据库保留;原配置在切换失败时自动恢复。请查看私有恢复日志后重试。', file=sys.stderr)
raise SystemExit(1)