完善:同月多文件导入与当前用户离线迁移更新

This commit is contained in:
2026-11-04 11:35:24 +08:00
parent 147b56d0fd
commit f04d4e96e3
89 changed files with 6634 additions and 201 deletions
+327
View File
@@ -0,0 +1,327 @@
"""Restore into a new database, validate, then switch the Windows deployment."""
from __future__ import annotations
from datetime import datetime, timezone
import gzip
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import sqlite3
import subprocess
import sys
import time
import uuid
import control
from migration_common import extract_checked, query_evidence, relative_name, sha256, sql_text, table_counts
ROOT = control.ROOT
PAYLOAD = ROOT / 'migration'
JOURNAL = control.PRIVATE / 'migration-progress.json'
RECEIPT = control.PRIVATE / 'migration-receipt.json'
def checked_local(relative):
rel = relative_name(relative)
path = ROOT.joinpath(*rel.parts)
if not path.resolve().is_relative_to(ROOT.resolve()):
raise RuntimeError('恢复暂存路径越界。')
return path
def run_query(value, statement, database=None):
command = control.mysql_command(value)
command += ['--batch', '--skip-column-names', '--default-character-set=utf8mb4', '--max-allowed-packet=1G']
if database:
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', database):
raise RuntimeError('恢复数据库名称无效。')
command += ['--database=' + database]
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
result = subprocess.run(command, input=statement, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=1800)
if result.returncode:
raise RuntimeError('恢复数据库校验未完成;已有数据库保留。')
return result.stdout
def stop_owned_service():
state = control.read_state()
if control.matches(state.get('supervisor', {})):
control.atomic(control.STOP, {'token': state['token']})
for _ in range(300):
if not control.matches(state['supervisor']):
break
time.sleep(.5)
else:
raise RuntimeError('原服务仍在关闭,请稍后重新运行。')
if any(control.matches(info) for info in state.get('children', [])):
raise RuntimeError('发现本目录尚未关闭的后台进程,请检查服务日志。')
def start_database(value):
with (control.DATA / 'logs/migration-database-supervisor.log').open('ab') as log:
options = ({'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
if control.IS_WINDOWS else {'start_new_session': True})
process = subprocess.Popen([str(control.PYTHON), str(Path(control.__file__).resolve()),
'serve', '--database-only'], cwd=ROOT, env=control.environment(value),
stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
for _ in range(360):
state = control.read_state()
if (state.get('ready') and state.get('databaseOnly') and
state.get('supervisor', {}).get('pid') == process.pid and control.matches(state['supervisor'])):
return
if process.poll() is not None:
raise RuntimeError('恢复数据库无法启动,请查看 runtimedata/logs。')
time.sleep(.5)
raise RuntimeError('恢复数据库仍未就绪,请查看日志后重试。')
def restore_dump(value, database, dump):
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
command = control.mysql_command(value) + ['--database=' + database,
'--default-character-set=utf8mb4', '--max-allowed-packet=1G',
'--init-command=SET SESSION autocommit=0']
with (control.DATA / 'logs/migration-import.log').open('ab') as errors:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors, env=env)
try:
total, next_progress = dump.stat().st_size, 0
with dump.open('rb') as raw, gzip.GzipFile(fileobj=raw) as source:
while chunk := source.read(1024 * 1024):
process.stdin.write(chunk)
percent = min(99, raw.tell() * 100 // total)
if percent >= next_progress:
print(f'数据库导入进度约 {percent}%……', flush=True)
next_progress = percent + 10
process.stdin.write(b'\nCOMMIT;\n')
process.stdin.close()
if process.wait(timeout=1800):
raise RuntimeError('业务数据导入失败,请查看私有恢复日志;原数据库未覆盖。')
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
def rebind_sources(query, manifest, archive):
counts = {'source_revision': 0, 'source_candidate': 0, 'source_upload_draft': 0}
for entry in manifest['pathBindings']:
pair = (entry['table'], entry['column'])
if pair not in {('source_revision', 'archive_path'), ('source_candidate', 'source_path'),
('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}:
raise RuntimeError('来源路径绑定类型无效。')
rel = relative_name(entry['relative'])
target = archive.joinpath(*rel.parts)
if not entry['expiredMissing']:
if not target.is_file() or target.stat().st_size != entry['bytes'] or sha256(target) != entry['sha256']:
raise RuntimeError('来源文件校验失败,恢复未切换。')
query('UPDATE `' + entry['table'] + '` SET `' + entry['column'] + '`=' + sql_text(str(target))
+ ' WHERE id=' + sql_text(entry['id']) + ' AND `' + entry['column'] + '`=' + sql_text(entry['oldPath']) + ';')
actual = query('SELECT `' + entry['column'] + '` FROM `' + entry['table'] + '` WHERE id=' + sql_text(entry['id']) + ';').strip()
if actual != str(target):
raise RuntimeError('来源文件路径重绑定失败。')
counts[entry['table']] += 1
return counts
def relocate_native(stage, source_root):
source_root = PurePosixPath(source_root)
def relocated(value):
if isinstance(value, str) and value.startswith(str(source_root) + '/'):
relative = PurePosixPath(value).relative_to(source_root)
if '..' in relative.parts:
raise RuntimeError('原生历史路径无效。')
return str(ROOT.joinpath(*relative.parts))
if isinstance(value, dict):
return {key: relocated(child) for key, child in value.items()}
if isinstance(value, list):
return [relocated(child) for child in value]
return value
for path in stage.rglob('*.json'):
control.atomic(path, relocated(json.loads(path.read_text(encoding='utf-8'))))
databases = 0
for path in stage.rglob('nao.sqlite'):
connection = sqlite3.connect(path)
try:
connection.execute('PRAGMA secure_delete=ON')
for table, column in (('project', 'path'), ('message_part', 'storage_path')):
exists = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
if not exists:
continue
for rowid, original in connection.execute('SELECT rowid,"' + column + '" FROM "' + table + '" WHERE "' + column + '" IS NOT NULL').fetchall():
replacement = relocated(original)
if replacement != original:
connection.execute('UPDATE "' + table + '" SET "' + column + '"=? WHERE rowid=?', (replacement, rowid))
connection.commit()
connection.execute('VACUUM')
if connection.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
raise RuntimeError('原生分析历史数据库校验失败。')
databases += 1
finally:
connection.close()
return databases
def rollback_plan(plan):
# Only paths generated by this migration are eligible. Databases and source
# archive trees are retained even after an interrupted restore.
attempt = plan['attempt']
if not re.fullmatch(r'[a-f0-9]{16}', attempt):
raise RuntimeError('恢复日志无效。')
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
failed = checked_local(item['failed'])
if previous.exists():
if target.exists():
target.rename(failed)
previous.rename(target)
elif not item['existed'] and not staged.exists() and target.exists():
target.rename(failed)
control.atomic(control.CONFIG, plan['previousConfig'])
JOURNAL.rename(control.PRIVATE / ('migration-rolled-back-' + attempt + '.json'))
def commit_plan(plan, receipt):
control.atomic(JOURNAL, plan)
try:
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
if target.is_symlink() or staged.is_symlink() or previous.exists():
raise RuntimeError('私有配置切换路径无效。')
if target.exists():
target.rename(previous)
staged.rename(target)
control.atomic(control.CONFIG, plan['newConfig'])
control.atomic(RECEIPT, receipt)
except BaseException:
rollback_plan(plan)
raise
JOURNAL.unlink(missing_ok=True)
def restore(check_runtime=True):
control.private_directory(control.PRIVATE)
control.private_directory(control.DATA)
(control.DATA / 'logs').mkdir(exist_ok=True)
(control.DATA / 'sources').mkdir(exist_ok=True)
(control.DATA / 'archive').mkdir(exist_ok=True)
lock = os.open(control.PRIVATE / 'migration.lock', os.O_CREAT | os.O_RDWR, 0o600)
try:
control.host.fcntl.flock(lock, control.host.fcntl.LOCK_EX | control.host.fcntl.LOCK_NB)
except BlockingIOError:
os.close(lock)
raise RuntimeError('另一项迁移正在进行,请等待原窗口完成。') from None
try:
return restore_locked(check_runtime)
finally:
os.close(lock)
def restore_locked(check_runtime):
manifest_path = PAYLOAD / 'manifest.json'
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
if manifest.get('formatVersion') != 1 or not re.fullmatch(r'[a-f0-9]{32}', manifest.get('migrationId', '')):
raise RuntimeError('迁移包格式无效。')
migration_id = manifest['migrationId']
if RECEIPT.exists():
receipt = json.loads(RECEIPT.read_text(encoding='utf-8'))
current = control.configuration()
if receipt.get('migrationId') == migration_id and current.get('restoredFromMigration') == migration_id and current.get('mysqlDatabase') == receipt.get('database'):
print('此迁移已完成,保留 Windows 上的后续数据,直接启动。', flush=True)
JOURNAL.unlink(missing_ok=True)
return receipt
raise RuntimeError('此目录已有其他迁移记录,请使用新的部署目录。')
stop_owned_service()
if JOURNAL.exists():
rollback_plan(json.loads(JOURNAL.read_text(encoding='utf-8')))
print('上次未完成的切换已撤回,原配置已保留。', flush=True)
value = control.verify() if check_runtime else control.configuration()
print('正在校验迁移包……', flush=True)
for name, key in (('database.sql.gz', 'databaseDumpSha256'), ('archive.tar.gz', 'archiveSha256'),
('native-history.tar.gz', 'nativeHistorySha256')):
if sha256(PAYLOAD / name) != manifest[key]:
raise RuntimeError('迁移包文件不完整,已停止恢复。')
if manifest.get('modelConfigSha256') and sha256(PAYLOAD / 'model-config.json') != manifest['modelConfigSha256']:
raise RuntimeError('模型配置校验失败。')
if shutil.disk_usage(ROOT).free < manifest['minimumFreeBytes']:
raise RuntimeError('可用磁盘空间不足,请至少预留 ' + str((manifest['minimumFreeBytes'] + 1024**3 - 1) // 1024**3) + ' GB。')
attempt = uuid.uuid4().hex[:16]
stage = control.PRIVATE / ('migration-stage-' + attempt)
stage.mkdir(mode=0o700)
archive = control.DATA / 'migrations' / attempt / 'archive'
archive.parent.mkdir(parents=True, mode=0o700)
native = stage / 'nao-users'
print('正在恢复来源文件和分析历史……', flush=True)
extract_checked(PAYLOAD / 'archive.tar.gz', archive, manifest['archiveFiles'])
extract_checked(PAYLOAD / 'native-history.tar.gz', native, manifest['nativeHistoryFiles'])
native_count = relocate_native(native, manifest['sourceProjectRoot'])
if native_count != manifest['nativeHistoryDatabases']:
raise RuntimeError('分析历史文件数量不一致。')
model = stage / 'model-config.json'
if manifest.get('modelConfigSha256'):
shutil.copy2(PAYLOAD / 'model-config.json', model)
model.chmod(0o600)
settings = json.loads(model.read_text(encoding='utf-8'))
if any('mimo' in str(profile.get(key, '')).lower() for profile in settings['profiles'] for key in ('model', 'name', 'baseUrl')):
raise RuntimeError('迁移模型配置中仍含 MiMo。')
database = 'quality_restore_' + uuid.uuid4().hex
try:
start_database(value)
run_query(value, 'CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
restore_dump(value, database, PAYLOAD / 'database.sql.gz')
query = lambda statement: run_query(value, statement, database)
print('正在核对全部数据表、原账号和正式统计结果……', flush=True)
actual_counts = table_counts(query)
if actual_counts != manifest['tableCounts']:
raise RuntimeError('恢复后的数据表行数不一致,未切换正式使用库。')
actual_users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()]
if sorted(actual_users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']):
raise RuntimeError('原用户账号校验失败,未切换使用库。')
if query_evidence(query) != manifest['queryEvidence']:
raise RuntimeError('正式版本和月度统计结果不一致,未切换使用库。')
rebound = rebind_sources(query, manifest, archive)
run_query(value, 'GRANT ALL PRIVILEGES ON `' + database + "`.* TO 'quality_app'@'127.0.0.1';")
run_query(value, 'FLUSH TABLES;')
finally:
stop_owned_service()
new_config = {**value, 'mysqlDatabase': database, 'archiveDirectory': archive.relative_to(ROOT).as_posix(),
'restoredFromMigration': migration_id, 'webBind': '0.0.0.0'}
for name in ('dataWorkflowTestMode', 'analysisAutonomousEnabled', 'analysisProtocolV1Enabled'):
if name in manifest.get('applicationSettings', {}):
new_config[name] = bool(manifest['applicationSettings'][name])
switches = []
for staged, target in [(native, control.PRIVATE / 'nao-users')] + ([(model, control.PRIVATE / 'nao-model-config.json')] if model.exists() else []):
switches.append({'staged': staged.relative_to(ROOT).as_posix(), 'target': target.relative_to(ROOT).as_posix(),
'previous': (control.PRIVATE / ('before-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'failed': (control.PRIVATE / ('failed-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'existed': target.exists()})
receipt = {'migrationId': migration_id, 'migrationManifestSha256': sha256(manifest_path), 'database': database,
'restoredAtUtc': datetime.now(timezone.utc).isoformat(), 'tables': len(actual_counts),
'totalRows': sum(actual_counts.values()), 'sourceReferencesRebound': rebound,
'modelProfilesMigrated': manifest['modelProfilesMigrated'], 'modelProfilesExcluded': manifest['modelProfilesExcluded'],
'nativeHistoryDatabases': native_count, 'originalDatabasesPreserved': True,
'archiveDirectory': new_config['archiveDirectory']}
plan = {'attempt': attempt, 'previousConfig': value, 'newConfig': new_config, 'privateSwitches': switches}
commit_plan(plan, receipt)
print('迁移恢复完成。请使用原来的账号和密码登录;MiMo 配置未迁移。', flush=True)
return receipt
if __name__ == '__main__':
try:
restore()
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error:
import traceback
log = control.DATA / 'logs/migration-validation.log'
log.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as output:
traceback.print_exc(file=output)
if isinstance(error, RuntimeError):
print(str(error), file=sys.stderr)
print('迁移未完成。已有数据库保留;原配置在切换失败时自动恢复。请查看私有恢复日志后重试。', file=sys.stderr)
raise SystemExit(1)