328 lines
17 KiB
Python
328 lines
17 KiB
Python
"""Restore into a new database, validate, then switch the Windows deployment."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
import gzip
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
import control
|
|
from migration_common import extract_checked, query_evidence, relative_name, sha256, sql_text, table_counts
|
|
|
|
ROOT = control.ROOT
|
|
PAYLOAD = ROOT / 'migration'
|
|
JOURNAL = control.PRIVATE / 'migration-progress.json'
|
|
RECEIPT = control.PRIVATE / 'migration-receipt.json'
|
|
|
|
|
|
def checked_local(relative):
|
|
rel = relative_name(relative)
|
|
path = ROOT.joinpath(*rel.parts)
|
|
if not path.resolve().is_relative_to(ROOT.resolve()):
|
|
raise RuntimeError('恢复暂存路径越界。')
|
|
return path
|
|
|
|
|
|
def run_query(value, statement, database=None):
|
|
command = control.mysql_command(value)
|
|
command += ['--batch', '--skip-column-names', '--default-character-set=utf8mb4', '--max-allowed-packet=1G']
|
|
if database:
|
|
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', database):
|
|
raise RuntimeError('恢复数据库名称无效。')
|
|
command += ['--database=' + database]
|
|
env = control.environment(value)
|
|
env['MYSQL_PWD'] = value['rootPassword']
|
|
result = subprocess.run(command, input=statement, text=True, encoding='utf-8',
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=1800)
|
|
if result.returncode:
|
|
raise RuntimeError('恢复数据库校验未完成;已有数据库保留。')
|
|
return result.stdout
|
|
|
|
|
|
def stop_owned_service():
|
|
state = control.read_state()
|
|
if control.matches(state.get('supervisor', {})):
|
|
control.atomic(control.STOP, {'token': state['token']})
|
|
for _ in range(300):
|
|
if not control.matches(state['supervisor']):
|
|
break
|
|
time.sleep(.5)
|
|
else:
|
|
raise RuntimeError('原服务仍在关闭,请稍后重新运行。')
|
|
if any(control.matches(info) for info in state.get('children', [])):
|
|
raise RuntimeError('发现本目录尚未关闭的后台进程,请检查服务日志。')
|
|
|
|
|
|
def start_database(value):
|
|
with (control.DATA / 'logs/migration-database-supervisor.log').open('ab') as log:
|
|
options = ({'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
|
|
if control.IS_WINDOWS else {'start_new_session': True})
|
|
process = subprocess.Popen([str(control.PYTHON), str(Path(control.__file__).resolve()),
|
|
'serve', '--database-only'], cwd=ROOT, env=control.environment(value),
|
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
|
|
for _ in range(360):
|
|
state = control.read_state()
|
|
if (state.get('ready') and state.get('databaseOnly') and
|
|
state.get('supervisor', {}).get('pid') == process.pid and control.matches(state['supervisor'])):
|
|
return
|
|
if process.poll() is not None:
|
|
raise RuntimeError('恢复数据库无法启动,请查看 runtimedata/logs。')
|
|
time.sleep(.5)
|
|
raise RuntimeError('恢复数据库仍未就绪,请查看日志后重试。')
|
|
|
|
|
|
def restore_dump(value, database, dump):
|
|
env = control.environment(value)
|
|
env['MYSQL_PWD'] = value['rootPassword']
|
|
command = control.mysql_command(value) + ['--database=' + database,
|
|
'--default-character-set=utf8mb4', '--max-allowed-packet=1G',
|
|
'--init-command=SET SESSION autocommit=0']
|
|
with (control.DATA / 'logs/migration-import.log').open('ab') as errors:
|
|
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors, env=env)
|
|
try:
|
|
total, next_progress = dump.stat().st_size, 0
|
|
with dump.open('rb') as raw, gzip.GzipFile(fileobj=raw) as source:
|
|
while chunk := source.read(1024 * 1024):
|
|
process.stdin.write(chunk)
|
|
percent = min(99, raw.tell() * 100 // total)
|
|
if percent >= next_progress:
|
|
print(f'数据库导入进度约 {percent}%……', flush=True)
|
|
next_progress = percent + 10
|
|
process.stdin.write(b'\nCOMMIT;\n')
|
|
process.stdin.close()
|
|
if process.wait(timeout=1800):
|
|
raise RuntimeError('业务数据导入失败,请查看私有恢复日志;原数据库未覆盖。')
|
|
except BaseException:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
|
|
|
|
def rebind_sources(query, manifest, archive):
|
|
counts = {'source_revision': 0, 'source_candidate': 0, 'source_upload_draft': 0}
|
|
for entry in manifest['pathBindings']:
|
|
pair = (entry['table'], entry['column'])
|
|
if pair not in {('source_revision', 'archive_path'), ('source_candidate', 'source_path'),
|
|
('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}:
|
|
raise RuntimeError('来源路径绑定类型无效。')
|
|
rel = relative_name(entry['relative'])
|
|
target = archive.joinpath(*rel.parts)
|
|
if not entry['expiredMissing']:
|
|
if not target.is_file() or target.stat().st_size != entry['bytes'] or sha256(target) != entry['sha256']:
|
|
raise RuntimeError('来源文件校验失败,恢复未切换。')
|
|
query('UPDATE `' + entry['table'] + '` SET `' + entry['column'] + '`=' + sql_text(str(target))
|
|
+ ' WHERE id=' + sql_text(entry['id']) + ' AND `' + entry['column'] + '`=' + sql_text(entry['oldPath']) + ';')
|
|
actual = query('SELECT `' + entry['column'] + '` FROM `' + entry['table'] + '` WHERE id=' + sql_text(entry['id']) + ';').strip()
|
|
if actual != str(target):
|
|
raise RuntimeError('来源文件路径重绑定失败。')
|
|
counts[entry['table']] += 1
|
|
return counts
|
|
|
|
|
|
def relocate_native(stage, source_root):
|
|
source_root = PurePosixPath(source_root)
|
|
def relocated(value):
|
|
if isinstance(value, str) and value.startswith(str(source_root) + '/'):
|
|
relative = PurePosixPath(value).relative_to(source_root)
|
|
if '..' in relative.parts:
|
|
raise RuntimeError('原生历史路径无效。')
|
|
return str(ROOT.joinpath(*relative.parts))
|
|
if isinstance(value, dict):
|
|
return {key: relocated(child) for key, child in value.items()}
|
|
if isinstance(value, list):
|
|
return [relocated(child) for child in value]
|
|
return value
|
|
for path in stage.rglob('*.json'):
|
|
control.atomic(path, relocated(json.loads(path.read_text(encoding='utf-8'))))
|
|
databases = 0
|
|
for path in stage.rglob('nao.sqlite'):
|
|
connection = sqlite3.connect(path)
|
|
try:
|
|
connection.execute('PRAGMA secure_delete=ON')
|
|
for table, column in (('project', 'path'), ('message_part', 'storage_path')):
|
|
exists = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
|
|
if not exists:
|
|
continue
|
|
for rowid, original in connection.execute('SELECT rowid,"' + column + '" FROM "' + table + '" WHERE "' + column + '" IS NOT NULL').fetchall():
|
|
replacement = relocated(original)
|
|
if replacement != original:
|
|
connection.execute('UPDATE "' + table + '" SET "' + column + '"=? WHERE rowid=?', (replacement, rowid))
|
|
connection.commit()
|
|
connection.execute('VACUUM')
|
|
if connection.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
|
|
raise RuntimeError('原生分析历史数据库校验失败。')
|
|
databases += 1
|
|
finally:
|
|
connection.close()
|
|
return databases
|
|
|
|
|
|
def rollback_plan(plan):
|
|
# Only paths generated by this migration are eligible. Databases and source
|
|
# archive trees are retained even after an interrupted restore.
|
|
attempt = plan['attempt']
|
|
if not re.fullmatch(r'[a-f0-9]{16}', attempt):
|
|
raise RuntimeError('恢复日志无效。')
|
|
for item in plan['privateSwitches']:
|
|
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
|
|
failed = checked_local(item['failed'])
|
|
if previous.exists():
|
|
if target.exists():
|
|
target.rename(failed)
|
|
previous.rename(target)
|
|
elif not item['existed'] and not staged.exists() and target.exists():
|
|
target.rename(failed)
|
|
control.atomic(control.CONFIG, plan['previousConfig'])
|
|
JOURNAL.rename(control.PRIVATE / ('migration-rolled-back-' + attempt + '.json'))
|
|
|
|
|
|
def commit_plan(plan, receipt):
|
|
control.atomic(JOURNAL, plan)
|
|
try:
|
|
for item in plan['privateSwitches']:
|
|
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
|
|
if target.is_symlink() or staged.is_symlink() or previous.exists():
|
|
raise RuntimeError('私有配置切换路径无效。')
|
|
if target.exists():
|
|
target.rename(previous)
|
|
staged.rename(target)
|
|
control.atomic(control.CONFIG, plan['newConfig'])
|
|
control.atomic(RECEIPT, receipt)
|
|
except BaseException:
|
|
rollback_plan(plan)
|
|
raise
|
|
JOURNAL.unlink(missing_ok=True)
|
|
|
|
|
|
def restore(check_runtime=True):
|
|
control.private_directory(control.PRIVATE)
|
|
control.private_directory(control.DATA)
|
|
(control.DATA / 'logs').mkdir(exist_ok=True)
|
|
(control.DATA / 'sources').mkdir(exist_ok=True)
|
|
(control.DATA / 'archive').mkdir(exist_ok=True)
|
|
lock = os.open(control.PRIVATE / 'migration.lock', os.O_CREAT | os.O_RDWR, 0o600)
|
|
try:
|
|
control.host.fcntl.flock(lock, control.host.fcntl.LOCK_EX | control.host.fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
os.close(lock)
|
|
raise RuntimeError('另一项迁移正在进行,请等待原窗口完成。') from None
|
|
try:
|
|
return restore_locked(check_runtime)
|
|
finally:
|
|
os.close(lock)
|
|
|
|
|
|
def restore_locked(check_runtime):
|
|
manifest_path = PAYLOAD / 'manifest.json'
|
|
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
|
|
if manifest.get('formatVersion') != 1 or not re.fullmatch(r'[a-f0-9]{32}', manifest.get('migrationId', '')):
|
|
raise RuntimeError('迁移包格式无效。')
|
|
migration_id = manifest['migrationId']
|
|
if RECEIPT.exists():
|
|
receipt = json.loads(RECEIPT.read_text(encoding='utf-8'))
|
|
current = control.configuration()
|
|
if receipt.get('migrationId') == migration_id and current.get('restoredFromMigration') == migration_id and current.get('mysqlDatabase') == receipt.get('database'):
|
|
print('此迁移已完成,保留 Windows 上的后续数据,直接启动。', flush=True)
|
|
JOURNAL.unlink(missing_ok=True)
|
|
return receipt
|
|
raise RuntimeError('此目录已有其他迁移记录,请使用新的部署目录。')
|
|
stop_owned_service()
|
|
if JOURNAL.exists():
|
|
rollback_plan(json.loads(JOURNAL.read_text(encoding='utf-8')))
|
|
print('上次未完成的切换已撤回,原配置已保留。', flush=True)
|
|
value = control.verify() if check_runtime else control.configuration()
|
|
print('正在校验迁移包……', flush=True)
|
|
for name, key in (('database.sql.gz', 'databaseDumpSha256'), ('archive.tar.gz', 'archiveSha256'),
|
|
('native-history.tar.gz', 'nativeHistorySha256')):
|
|
if sha256(PAYLOAD / name) != manifest[key]:
|
|
raise RuntimeError('迁移包文件不完整,已停止恢复。')
|
|
if manifest.get('modelConfigSha256') and sha256(PAYLOAD / 'model-config.json') != manifest['modelConfigSha256']:
|
|
raise RuntimeError('模型配置校验失败。')
|
|
if shutil.disk_usage(ROOT).free < manifest['minimumFreeBytes']:
|
|
raise RuntimeError('可用磁盘空间不足,请至少预留 ' + str((manifest['minimumFreeBytes'] + 1024**3 - 1) // 1024**3) + ' GB。')
|
|
attempt = uuid.uuid4().hex[:16]
|
|
stage = control.PRIVATE / ('migration-stage-' + attempt)
|
|
stage.mkdir(mode=0o700)
|
|
archive = control.DATA / 'migrations' / attempt / 'archive'
|
|
archive.parent.mkdir(parents=True, mode=0o700)
|
|
native = stage / 'nao-users'
|
|
print('正在恢复来源文件和分析历史……', flush=True)
|
|
extract_checked(PAYLOAD / 'archive.tar.gz', archive, manifest['archiveFiles'])
|
|
extract_checked(PAYLOAD / 'native-history.tar.gz', native, manifest['nativeHistoryFiles'])
|
|
native_count = relocate_native(native, manifest['sourceProjectRoot'])
|
|
if native_count != manifest['nativeHistoryDatabases']:
|
|
raise RuntimeError('分析历史文件数量不一致。')
|
|
model = stage / 'model-config.json'
|
|
if manifest.get('modelConfigSha256'):
|
|
shutil.copy2(PAYLOAD / 'model-config.json', model)
|
|
model.chmod(0o600)
|
|
settings = json.loads(model.read_text(encoding='utf-8'))
|
|
if any('mimo' in str(profile.get(key, '')).lower() for profile in settings['profiles'] for key in ('model', 'name', 'baseUrl')):
|
|
raise RuntimeError('迁移模型配置中仍含 MiMo。')
|
|
database = 'quality_restore_' + uuid.uuid4().hex
|
|
try:
|
|
start_database(value)
|
|
run_query(value, 'CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
restore_dump(value, database, PAYLOAD / 'database.sql.gz')
|
|
query = lambda statement: run_query(value, statement, database)
|
|
print('正在核对全部数据表、原账号和正式统计结果……', flush=True)
|
|
actual_counts = table_counts(query)
|
|
if actual_counts != manifest['tableCounts']:
|
|
raise RuntimeError('恢复后的数据表行数不一致,未切换正式使用库。')
|
|
actual_users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()]
|
|
if sorted(actual_users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']):
|
|
raise RuntimeError('原用户账号校验失败,未切换使用库。')
|
|
if query_evidence(query) != manifest['queryEvidence']:
|
|
raise RuntimeError('正式版本和月度统计结果不一致,未切换使用库。')
|
|
rebound = rebind_sources(query, manifest, archive)
|
|
run_query(value, 'GRANT ALL PRIVILEGES ON `' + database + "`.* TO 'quality_app'@'127.0.0.1';")
|
|
run_query(value, 'FLUSH TABLES;')
|
|
finally:
|
|
stop_owned_service()
|
|
new_config = {**value, 'mysqlDatabase': database, 'archiveDirectory': archive.relative_to(ROOT).as_posix(),
|
|
'restoredFromMigration': migration_id, 'webBind': '0.0.0.0'}
|
|
for name in ('dataWorkflowTestMode', 'analysisAutonomousEnabled', 'analysisProtocolV1Enabled'):
|
|
if name in manifest.get('applicationSettings', {}):
|
|
new_config[name] = bool(manifest['applicationSettings'][name])
|
|
switches = []
|
|
for staged, target in [(native, control.PRIVATE / 'nao-users')] + ([(model, control.PRIVATE / 'nao-model-config.json')] if model.exists() else []):
|
|
switches.append({'staged': staged.relative_to(ROOT).as_posix(), 'target': target.relative_to(ROOT).as_posix(),
|
|
'previous': (control.PRIVATE / ('before-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
|
|
'failed': (control.PRIVATE / ('failed-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
|
|
'existed': target.exists()})
|
|
receipt = {'migrationId': migration_id, 'migrationManifestSha256': sha256(manifest_path), 'database': database,
|
|
'restoredAtUtc': datetime.now(timezone.utc).isoformat(), 'tables': len(actual_counts),
|
|
'totalRows': sum(actual_counts.values()), 'sourceReferencesRebound': rebound,
|
|
'modelProfilesMigrated': manifest['modelProfilesMigrated'], 'modelProfilesExcluded': manifest['modelProfilesExcluded'],
|
|
'nativeHistoryDatabases': native_count, 'originalDatabasesPreserved': True,
|
|
'archiveDirectory': new_config['archiveDirectory']}
|
|
plan = {'attempt': attempt, 'previousConfig': value, 'newConfig': new_config, 'privateSwitches': switches}
|
|
commit_plan(plan, receipt)
|
|
print('迁移恢复完成。请使用原来的账号和密码登录;MiMo 配置未迁移。', flush=True)
|
|
return receipt
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
restore()
|
|
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error:
|
|
import traceback
|
|
log = control.DATA / 'logs/migration-validation.log'
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as output:
|
|
traceback.print_exc(file=output)
|
|
if isinstance(error, RuntimeError):
|
|
print(str(error), file=sys.stderr)
|
|
print('迁移未完成。已有数据库保留;原配置在切换失败时自动恢复。请查看私有恢复日志后重试。', file=sys.stderr)
|
|
raise SystemExit(1)
|