Files

201 lines
10 KiB
Python

"""Capture verified source files and private model settings for a Windows migration."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import sys
import tarfile
import uuid
from migration_common import filter_models, relative_name, sha256
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup
def private_json(path, data):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as output:
json.dump(data, output, ensure_ascii=False, indent=2)
output.write('\n')
def native_copy(source, target):
source_db = sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)
destination = sqlite3.connect(target)
try:
source_db.backup(destination)
destination.execute('PRAGMA foreign_keys=OFF')
destination.execute('PRAGMA secure_delete=ON')
tables = {row[0] for row in destination.execute("SELECT name FROM sqlite_master WHERE type='table'")}
# Access sessions, external integrations and cached provider keys are
# recreated on Windows. Historical chats and their ownership stay intact.
for name in ('session', 'verification', 'api_key', 'jwks', 'oauth_access_token',
'oauth_refresh_token', 'oauth_consent', 'oauth_client', 'mcp_user_token',
'mcp_oauth_client', 'sandbox_secret', 'log', 'mcp_call_log'):
if name in tables:
destination.execute('DELETE FROM "' + name + '"')
if 'project_llm_config' in tables:
destination.execute('DELETE FROM project_llm_config')
for name in ('user', 'account', 'organization'):
if name not in tables:
continue
columns = list(destination.execute('PRAGMA table_info("' + name + '")'))
for column in columns:
key = column[1]
if key in ('github_access_token', 'gitlab_access_token', 'access_token', 'refresh_token',
'id_token', 'google_client_secret'):
replacement = "''" if column[3] else 'NULL'
destination.execute('UPDATE "' + name + '" SET "' + key + '"=' + replacement)
destination.commit()
destination.execute('VACUUM')
assert destination.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
finally:
destination.close()
source_db.close()
target.chmod(0o600)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--work', type=Path, required=True)
args = parser.parse_args()
work = args.work.resolve()
payload = work / 'migration'
manifest = json.loads((payload / 'source-snapshot.json').read_text())
database = manifest['sourceDatabase']
query = lambda sql: backup.client(sql, database)
archive_root = ROOT / 'runtimedata/archive'
files, sources, digest_index = {}, {}, {}
for path in sorted(archive_root.rglob('*')):
if path.is_symlink():
raise RuntimeError('来源归档包含符号链接。')
if not path.is_file():
continue
rel = relative_name(path.relative_to(archive_root).as_posix()).as_posix()
digest = sha256(path)
files[rel] = {'sha256': digest, 'bytes': path.stat().st_size}
sources[rel] = path
digest_index.setdefault(digest, rel)
bindings = []
missing_optional = 0
for table, column, state in (
('source_revision', 'archive_path', "'REQUIRED'"),
('source_candidate', 'source_path', 'status'),
('source_candidate', 'archive_path', 'status'),
('source_upload_draft', 'source_path', "IF(expires_at<CURRENT_TIMESTAMP(6),'EXPIRED','ACTIVE')"),
):
rows = query("SELECT JSON_OBJECT('id',id,'path'," + column + ",'sha256',sha256,'size',file_size,'state'," + state + ') FROM ' + table + ' WHERE ' + column + ' IS NOT NULL;')
for line in rows.splitlines():
row = json.loads(line)
old = Path(row['path'])
digest = row['sha256']
relative = None
if old.is_relative_to(archive_root) and old.is_file():
relative = old.relative_to(archive_root).as_posix()
elif digest in digest_index:
relative = digest_index[digest]
elif old.is_file() and not old.is_symlink():
if sha256(old) != digest or old.stat().st_size != int(row['size']):
raise RuntimeError('需要迁移的来源文件与登记内容不一致。')
relative = 'migration-inputs/' + digest + '.xlsx'
files[relative] = {'sha256': digest, 'bytes': old.stat().st_size}
sources[relative] = old
digest_index[digest] = relative
optional = False
if relative is None:
if table != 'source_upload_draft' or row['state'] != 'EXPIRED':
raise RuntimeError('需要迁移的来源文件缺失。')
relative = 'expired-upload-drafts/' + row['id'] + '.xlsx'
optional = True
missing_optional += 1
else:
info = files[relative]
if info['sha256'] != digest or info['bytes'] != int(row['size']):
raise RuntimeError('来源引用与归档校验不一致。')
bindings.append({'table': table, 'column': column, 'id': row['id'], 'oldPath': row['path'],
'relative': relative, 'sha256': digest, 'bytes': int(row['size']), 'expiredMissing': optional})
archive = payload / 'archive.tar.gz'
with tarfile.open(archive, 'w:gz', compresslevel=1) as output:
for rel, path in sorted(sources.items()):
output.add(path, arcname=rel, recursive=False)
if sha256(path) != files[rel]['sha256']:
raise RuntimeError('打包期间来源文件发生变化。')
archive.chmod(0o600)
print('Source files captured:', len(files), 'references:', len(bindings), flush=True)
config = json.loads((ROOT / '.runtime/nao-model-config.json').read_text())
filtered, excluded = filter_models(config)
if filtered:
private_json(payload / 'model-config.json', filtered)
kept_keys = {p.get('apiKey') for p in (filtered or {}).get('profiles', [])}
forbidden_keys = {p.get('apiKey') for p in excluded if p.get('apiKey') and p.get('apiKey') not in kept_keys}
# Used only by the local privacy verifier; never shipped or printed.
private_json(work / 'excluded-model-audit-private.json', {'keys': sorted(forbidden_keys)})
session_rows = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'userId',user_id) FROM analysis_session;").splitlines()]
users = {row['id'] for row in manifest['sourceUsers']}
native_source = ROOT / '.runtime/nao-users'
native_stage = work / 'native-snapshot'
native_stage.mkdir(mode=0o700, exist_ok=False)
runtime_roots = []
for actor in users:
if not re.fullmatch(r'[a-fA-F0-9-]{36}', actor):
raise RuntimeError('用户标识无效。')
if (native_source / actor / 'private/nao.sqlite').is_file():
runtime_roots.append(native_source / actor)
for row in session_rows:
source = native_source / row['userId'] / 'sessions' / row['id']
if (source / 'private/nao.sqlite').is_file():
runtime_roots.append(source)
selected = set()
for runtime in runtime_roots:
for name in ('credentials.json', 'nao.sqlite', 'native-chat.json', 'current-session.json', 'sessions.json'):
source = runtime / 'private' / name
if source.is_file() and not source.is_symlink():
selected.add(source)
evidence = runtime / 'evidence'
if evidence.is_dir():
selected.update(p for p in evidence.rglob('*') if p.is_file() and not p.is_symlink())
native_files = {}
for source in sorted(selected):
rel = source.relative_to(native_source).as_posix()
relative_name(rel)
target = native_stage / rel
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
if source.name == 'nao.sqlite':
native_copy(source, target)
else:
shutil.copy2(source, target)
target.chmod(0o600)
content = target.read_bytes()
if any(key.encode() in content for key in forbidden_keys):
raise RuntimeError('原生会话中检测到被排除模型的密钥,已停止打包。')
native_files[rel] = {'sha256': sha256(target), 'bytes': target.stat().st_size}
with tarfile.open(payload / 'native-history.tar.gz', 'w:gz', compresslevel=1) as output:
for rel in sorted(native_files):
output.add(native_stage / rel, arcname=rel, recursive=False)
(payload / 'native-history.tar.gz').chmod(0o600)
manifest.update({'migrationId': uuid.uuid4().hex, 'archiveFiles': files, 'pathBindings': bindings,
'archiveSha256': sha256(archive), 'nativeHistoryFiles': native_files,
'nativeHistorySha256': sha256(payload / 'native-history.tar.gz'),
'sourceProjectRoot': str(ROOT), 'modelProfilesMigrated': len((filtered or {}).get('profiles', [])),
'modelProfilesExcluded': len(excluded), 'modelConfigSha256': sha256(payload / 'model-config.json') if filtered else None,
'expiredMissingDrafts': missing_optional,
'sourceDatabaseBytes': int(backup.client("SELECT SUM(data_length+index_length) FROM information_schema.tables WHERE table_schema='" + database + "';").strip()),
'nativeHistoryDatabases': len(runtime_roots)})
private_json(work / 'migration-file-snapshot.json', manifest)
print('Model profiles migrated:', manifest['modelProfilesMigrated'], '; MiMo excluded:', len(excluded), flush=True)
print('Native business-history databases:', len(runtime_roots), '; files:', len(native_files), flush=True)
if __name__ == '__main__':
main()