201 lines
10 KiB
Python
201 lines
10 KiB
Python
"""Capture verified source files and private model settings for a Windows migration."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
import uuid
|
|
|
|
from migration_common import filter_models, relative_name, sha256
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import backup
|
|
|
|
|
|
def private_json(path, data):
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as output:
|
|
json.dump(data, output, ensure_ascii=False, indent=2)
|
|
output.write('\n')
|
|
|
|
|
|
def native_copy(source, target):
|
|
source_db = sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)
|
|
destination = sqlite3.connect(target)
|
|
try:
|
|
source_db.backup(destination)
|
|
destination.execute('PRAGMA foreign_keys=OFF')
|
|
destination.execute('PRAGMA secure_delete=ON')
|
|
tables = {row[0] for row in destination.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
# Access sessions, external integrations and cached provider keys are
|
|
# recreated on Windows. Historical chats and their ownership stay intact.
|
|
for name in ('session', 'verification', 'api_key', 'jwks', 'oauth_access_token',
|
|
'oauth_refresh_token', 'oauth_consent', 'oauth_client', 'mcp_user_token',
|
|
'mcp_oauth_client', 'sandbox_secret', 'log', 'mcp_call_log'):
|
|
if name in tables:
|
|
destination.execute('DELETE FROM "' + name + '"')
|
|
if 'project_llm_config' in tables:
|
|
destination.execute('DELETE FROM project_llm_config')
|
|
for name in ('user', 'account', 'organization'):
|
|
if name not in tables:
|
|
continue
|
|
columns = list(destination.execute('PRAGMA table_info("' + name + '")'))
|
|
for column in columns:
|
|
key = column[1]
|
|
if key in ('github_access_token', 'gitlab_access_token', 'access_token', 'refresh_token',
|
|
'id_token', 'google_client_secret'):
|
|
replacement = "''" if column[3] else 'NULL'
|
|
destination.execute('UPDATE "' + name + '" SET "' + key + '"=' + replacement)
|
|
destination.commit()
|
|
destination.execute('VACUUM')
|
|
assert destination.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
|
|
finally:
|
|
destination.close()
|
|
source_db.close()
|
|
target.chmod(0o600)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--work', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
work = args.work.resolve()
|
|
payload = work / 'migration'
|
|
manifest = json.loads((payload / 'source-snapshot.json').read_text())
|
|
database = manifest['sourceDatabase']
|
|
query = lambda sql: backup.client(sql, database)
|
|
archive_root = ROOT / 'runtimedata/archive'
|
|
files, sources, digest_index = {}, {}, {}
|
|
for path in sorted(archive_root.rglob('*')):
|
|
if path.is_symlink():
|
|
raise RuntimeError('来源归档包含符号链接。')
|
|
if not path.is_file():
|
|
continue
|
|
rel = relative_name(path.relative_to(archive_root).as_posix()).as_posix()
|
|
digest = sha256(path)
|
|
files[rel] = {'sha256': digest, 'bytes': path.stat().st_size}
|
|
sources[rel] = path
|
|
digest_index.setdefault(digest, rel)
|
|
bindings = []
|
|
missing_optional = 0
|
|
for table, column, state in (
|
|
('source_revision', 'archive_path', "'REQUIRED'"),
|
|
('source_candidate', 'source_path', 'status'),
|
|
('source_candidate', 'archive_path', 'status'),
|
|
('source_upload_draft', 'source_path', "IF(expires_at<CURRENT_TIMESTAMP(6),'EXPIRED','ACTIVE')"),
|
|
):
|
|
rows = query("SELECT JSON_OBJECT('id',id,'path'," + column + ",'sha256',sha256,'size',file_size,'state'," + state + ') FROM ' + table + ' WHERE ' + column + ' IS NOT NULL;')
|
|
for line in rows.splitlines():
|
|
row = json.loads(line)
|
|
old = Path(row['path'])
|
|
digest = row['sha256']
|
|
relative = None
|
|
if old.is_relative_to(archive_root) and old.is_file():
|
|
relative = old.relative_to(archive_root).as_posix()
|
|
elif digest in digest_index:
|
|
relative = digest_index[digest]
|
|
elif old.is_file() and not old.is_symlink():
|
|
if sha256(old) != digest or old.stat().st_size != int(row['size']):
|
|
raise RuntimeError('需要迁移的来源文件与登记内容不一致。')
|
|
relative = 'migration-inputs/' + digest + '.xlsx'
|
|
files[relative] = {'sha256': digest, 'bytes': old.stat().st_size}
|
|
sources[relative] = old
|
|
digest_index[digest] = relative
|
|
optional = False
|
|
if relative is None:
|
|
if table != 'source_upload_draft' or row['state'] != 'EXPIRED':
|
|
raise RuntimeError('需要迁移的来源文件缺失。')
|
|
relative = 'expired-upload-drafts/' + row['id'] + '.xlsx'
|
|
optional = True
|
|
missing_optional += 1
|
|
else:
|
|
info = files[relative]
|
|
if info['sha256'] != digest or info['bytes'] != int(row['size']):
|
|
raise RuntimeError('来源引用与归档校验不一致。')
|
|
bindings.append({'table': table, 'column': column, 'id': row['id'], 'oldPath': row['path'],
|
|
'relative': relative, 'sha256': digest, 'bytes': int(row['size']), 'expiredMissing': optional})
|
|
archive = payload / 'archive.tar.gz'
|
|
with tarfile.open(archive, 'w:gz', compresslevel=1) as output:
|
|
for rel, path in sorted(sources.items()):
|
|
output.add(path, arcname=rel, recursive=False)
|
|
if sha256(path) != files[rel]['sha256']:
|
|
raise RuntimeError('打包期间来源文件发生变化。')
|
|
archive.chmod(0o600)
|
|
print('Source files captured:', len(files), 'references:', len(bindings), flush=True)
|
|
|
|
config = json.loads((ROOT / '.runtime/nao-model-config.json').read_text())
|
|
filtered, excluded = filter_models(config)
|
|
if filtered:
|
|
private_json(payload / 'model-config.json', filtered)
|
|
kept_keys = {p.get('apiKey') for p in (filtered or {}).get('profiles', [])}
|
|
forbidden_keys = {p.get('apiKey') for p in excluded if p.get('apiKey') and p.get('apiKey') not in kept_keys}
|
|
# Used only by the local privacy verifier; never shipped or printed.
|
|
private_json(work / 'excluded-model-audit-private.json', {'keys': sorted(forbidden_keys)})
|
|
|
|
session_rows = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'userId',user_id) FROM analysis_session;").splitlines()]
|
|
users = {row['id'] for row in manifest['sourceUsers']}
|
|
native_source = ROOT / '.runtime/nao-users'
|
|
native_stage = work / 'native-snapshot'
|
|
native_stage.mkdir(mode=0o700, exist_ok=False)
|
|
runtime_roots = []
|
|
for actor in users:
|
|
if not re.fullmatch(r'[a-fA-F0-9-]{36}', actor):
|
|
raise RuntimeError('用户标识无效。')
|
|
if (native_source / actor / 'private/nao.sqlite').is_file():
|
|
runtime_roots.append(native_source / actor)
|
|
for row in session_rows:
|
|
source = native_source / row['userId'] / 'sessions' / row['id']
|
|
if (source / 'private/nao.sqlite').is_file():
|
|
runtime_roots.append(source)
|
|
selected = set()
|
|
for runtime in runtime_roots:
|
|
for name in ('credentials.json', 'nao.sqlite', 'native-chat.json', 'current-session.json', 'sessions.json'):
|
|
source = runtime / 'private' / name
|
|
if source.is_file() and not source.is_symlink():
|
|
selected.add(source)
|
|
evidence = runtime / 'evidence'
|
|
if evidence.is_dir():
|
|
selected.update(p for p in evidence.rglob('*') if p.is_file() and not p.is_symlink())
|
|
native_files = {}
|
|
for source in sorted(selected):
|
|
rel = source.relative_to(native_source).as_posix()
|
|
relative_name(rel)
|
|
target = native_stage / rel
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
if source.name == 'nao.sqlite':
|
|
native_copy(source, target)
|
|
else:
|
|
shutil.copy2(source, target)
|
|
target.chmod(0o600)
|
|
content = target.read_bytes()
|
|
if any(key.encode() in content for key in forbidden_keys):
|
|
raise RuntimeError('原生会话中检测到被排除模型的密钥,已停止打包。')
|
|
native_files[rel] = {'sha256': sha256(target), 'bytes': target.stat().st_size}
|
|
with tarfile.open(payload / 'native-history.tar.gz', 'w:gz', compresslevel=1) as output:
|
|
for rel in sorted(native_files):
|
|
output.add(native_stage / rel, arcname=rel, recursive=False)
|
|
(payload / 'native-history.tar.gz').chmod(0o600)
|
|
manifest.update({'migrationId': uuid.uuid4().hex, 'archiveFiles': files, 'pathBindings': bindings,
|
|
'archiveSha256': sha256(archive), 'nativeHistoryFiles': native_files,
|
|
'nativeHistorySha256': sha256(payload / 'native-history.tar.gz'),
|
|
'sourceProjectRoot': str(ROOT), 'modelProfilesMigrated': len((filtered or {}).get('profiles', [])),
|
|
'modelProfilesExcluded': len(excluded), 'modelConfigSha256': sha256(payload / 'model-config.json') if filtered else None,
|
|
'expiredMissingDrafts': missing_optional,
|
|
'sourceDatabaseBytes': int(backup.client("SELECT SUM(data_length+index_length) FROM information_schema.tables WHERE table_schema='" + database + "';").strip()),
|
|
'nativeHistoryDatabases': len(runtime_roots)})
|
|
private_json(work / 'migration-file-snapshot.json', manifest)
|
|
print('Model profiles migrated:', manifest['modelProfilesMigrated'], '; MiMo excluded:', len(excluded), flush=True)
|
|
print('Native business-history databases:', len(runtime_roots), '; files:', len(native_files), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|