完善:同月多文件导入与当前用户离线迁移更新
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
"""Capture verified source files and private model settings for a Windows migration."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tarfile
|
||||
import uuid
|
||||
|
||||
from migration_common import filter_models, relative_name, sha256
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import backup
|
||||
|
||||
|
||||
def private_json(path, data):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as output:
|
||||
json.dump(data, output, ensure_ascii=False, indent=2)
|
||||
output.write('\n')
|
||||
|
||||
|
||||
def native_copy(source, target):
|
||||
source_db = sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)
|
||||
destination = sqlite3.connect(target)
|
||||
try:
|
||||
source_db.backup(destination)
|
||||
destination.execute('PRAGMA foreign_keys=OFF')
|
||||
destination.execute('PRAGMA secure_delete=ON')
|
||||
tables = {row[0] for row in destination.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
# Access sessions, external integrations and cached provider keys are
|
||||
# recreated on Windows. Historical chats and their ownership stay intact.
|
||||
for name in ('session', 'verification', 'api_key', 'jwks', 'oauth_access_token',
|
||||
'oauth_refresh_token', 'oauth_consent', 'oauth_client', 'mcp_user_token',
|
||||
'mcp_oauth_client', 'sandbox_secret', 'log', 'mcp_call_log'):
|
||||
if name in tables:
|
||||
destination.execute('DELETE FROM "' + name + '"')
|
||||
if 'project_llm_config' in tables:
|
||||
destination.execute('DELETE FROM project_llm_config')
|
||||
for name in ('user', 'account', 'organization'):
|
||||
if name not in tables:
|
||||
continue
|
||||
columns = list(destination.execute('PRAGMA table_info("' + name + '")'))
|
||||
for column in columns:
|
||||
key = column[1]
|
||||
if key in ('github_access_token', 'gitlab_access_token', 'access_token', 'refresh_token',
|
||||
'id_token', 'google_client_secret'):
|
||||
replacement = "''" if column[3] else 'NULL'
|
||||
destination.execute('UPDATE "' + name + '" SET "' + key + '"=' + replacement)
|
||||
destination.commit()
|
||||
destination.execute('VACUUM')
|
||||
assert destination.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
|
||||
finally:
|
||||
destination.close()
|
||||
source_db.close()
|
||||
target.chmod(0o600)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--work', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
work = args.work.resolve()
|
||||
payload = work / 'migration'
|
||||
manifest = json.loads((payload / 'source-snapshot.json').read_text())
|
||||
database = manifest['sourceDatabase']
|
||||
query = lambda sql: backup.client(sql, database)
|
||||
archive_root = ROOT / 'runtimedata/archive'
|
||||
files, sources, digest_index = {}, {}, {}
|
||||
for path in sorted(archive_root.rglob('*')):
|
||||
if path.is_symlink():
|
||||
raise RuntimeError('来源归档包含符号链接。')
|
||||
if not path.is_file():
|
||||
continue
|
||||
rel = relative_name(path.relative_to(archive_root).as_posix()).as_posix()
|
||||
digest = sha256(path)
|
||||
files[rel] = {'sha256': digest, 'bytes': path.stat().st_size}
|
||||
sources[rel] = path
|
||||
digest_index.setdefault(digest, rel)
|
||||
bindings = []
|
||||
missing_optional = 0
|
||||
for table, column, state in (
|
||||
('source_revision', 'archive_path', "'REQUIRED'"),
|
||||
('source_candidate', 'source_path', 'status'),
|
||||
('source_candidate', 'archive_path', 'status'),
|
||||
('source_upload_draft', 'source_path', "IF(expires_at<CURRENT_TIMESTAMP(6),'EXPIRED','ACTIVE')"),
|
||||
):
|
||||
rows = query("SELECT JSON_OBJECT('id',id,'path'," + column + ",'sha256',sha256,'size',file_size,'state'," + state + ') FROM ' + table + ' WHERE ' + column + ' IS NOT NULL;')
|
||||
for line in rows.splitlines():
|
||||
row = json.loads(line)
|
||||
old = Path(row['path'])
|
||||
digest = row['sha256']
|
||||
relative = None
|
||||
if old.is_relative_to(archive_root) and old.is_file():
|
||||
relative = old.relative_to(archive_root).as_posix()
|
||||
elif digest in digest_index:
|
||||
relative = digest_index[digest]
|
||||
elif old.is_file() and not old.is_symlink():
|
||||
if sha256(old) != digest or old.stat().st_size != int(row['size']):
|
||||
raise RuntimeError('需要迁移的来源文件与登记内容不一致。')
|
||||
relative = 'migration-inputs/' + digest + '.xlsx'
|
||||
files[relative] = {'sha256': digest, 'bytes': old.stat().st_size}
|
||||
sources[relative] = old
|
||||
digest_index[digest] = relative
|
||||
optional = False
|
||||
if relative is None:
|
||||
if table != 'source_upload_draft' or row['state'] != 'EXPIRED':
|
||||
raise RuntimeError('需要迁移的来源文件缺失。')
|
||||
relative = 'expired-upload-drafts/' + row['id'] + '.xlsx'
|
||||
optional = True
|
||||
missing_optional += 1
|
||||
else:
|
||||
info = files[relative]
|
||||
if info['sha256'] != digest or info['bytes'] != int(row['size']):
|
||||
raise RuntimeError('来源引用与归档校验不一致。')
|
||||
bindings.append({'table': table, 'column': column, 'id': row['id'], 'oldPath': row['path'],
|
||||
'relative': relative, 'sha256': digest, 'bytes': int(row['size']), 'expiredMissing': optional})
|
||||
archive = payload / 'archive.tar.gz'
|
||||
with tarfile.open(archive, 'w:gz', compresslevel=1) as output:
|
||||
for rel, path in sorted(sources.items()):
|
||||
output.add(path, arcname=rel, recursive=False)
|
||||
if sha256(path) != files[rel]['sha256']:
|
||||
raise RuntimeError('打包期间来源文件发生变化。')
|
||||
archive.chmod(0o600)
|
||||
print('Source files captured:', len(files), 'references:', len(bindings), flush=True)
|
||||
|
||||
config = json.loads((ROOT / '.runtime/nao-model-config.json').read_text())
|
||||
filtered, excluded = filter_models(config)
|
||||
if filtered:
|
||||
private_json(payload / 'model-config.json', filtered)
|
||||
kept_keys = {p.get('apiKey') for p in (filtered or {}).get('profiles', [])}
|
||||
forbidden_keys = {p.get('apiKey') for p in excluded if p.get('apiKey') and p.get('apiKey') not in kept_keys}
|
||||
# Used only by the local privacy verifier; never shipped or printed.
|
||||
private_json(work / 'excluded-model-audit-private.json', {'keys': sorted(forbidden_keys)})
|
||||
|
||||
session_rows = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'userId',user_id) FROM analysis_session;").splitlines()]
|
||||
users = {row['id'] for row in manifest['sourceUsers']}
|
||||
native_source = ROOT / '.runtime/nao-users'
|
||||
native_stage = work / 'native-snapshot'
|
||||
native_stage.mkdir(mode=0o700, exist_ok=False)
|
||||
runtime_roots = []
|
||||
for actor in users:
|
||||
if not re.fullmatch(r'[a-fA-F0-9-]{36}', actor):
|
||||
raise RuntimeError('用户标识无效。')
|
||||
if (native_source / actor / 'private/nao.sqlite').is_file():
|
||||
runtime_roots.append(native_source / actor)
|
||||
for row in session_rows:
|
||||
source = native_source / row['userId'] / 'sessions' / row['id']
|
||||
if (source / 'private/nao.sqlite').is_file():
|
||||
runtime_roots.append(source)
|
||||
selected = set()
|
||||
for runtime in runtime_roots:
|
||||
for name in ('credentials.json', 'nao.sqlite', 'native-chat.json', 'current-session.json', 'sessions.json'):
|
||||
source = runtime / 'private' / name
|
||||
if source.is_file() and not source.is_symlink():
|
||||
selected.add(source)
|
||||
evidence = runtime / 'evidence'
|
||||
if evidence.is_dir():
|
||||
selected.update(p for p in evidence.rglob('*') if p.is_file() and not p.is_symlink())
|
||||
native_files = {}
|
||||
for source in sorted(selected):
|
||||
rel = source.relative_to(native_source).as_posix()
|
||||
relative_name(rel)
|
||||
target = native_stage / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if source.name == 'nao.sqlite':
|
||||
native_copy(source, target)
|
||||
else:
|
||||
shutil.copy2(source, target)
|
||||
target.chmod(0o600)
|
||||
content = target.read_bytes()
|
||||
if any(key.encode() in content for key in forbidden_keys):
|
||||
raise RuntimeError('原生会话中检测到被排除模型的密钥,已停止打包。')
|
||||
native_files[rel] = {'sha256': sha256(target), 'bytes': target.stat().st_size}
|
||||
with tarfile.open(payload / 'native-history.tar.gz', 'w:gz', compresslevel=1) as output:
|
||||
for rel in sorted(native_files):
|
||||
output.add(native_stage / rel, arcname=rel, recursive=False)
|
||||
(payload / 'native-history.tar.gz').chmod(0o600)
|
||||
manifest.update({'migrationId': uuid.uuid4().hex, 'archiveFiles': files, 'pathBindings': bindings,
|
||||
'archiveSha256': sha256(archive), 'nativeHistoryFiles': native_files,
|
||||
'nativeHistorySha256': sha256(payload / 'native-history.tar.gz'),
|
||||
'sourceProjectRoot': str(ROOT), 'modelProfilesMigrated': len((filtered or {}).get('profiles', [])),
|
||||
'modelProfilesExcluded': len(excluded), 'modelConfigSha256': sha256(payload / 'model-config.json') if filtered else None,
|
||||
'expiredMissingDrafts': missing_optional,
|
||||
'sourceDatabaseBytes': int(backup.client("SELECT SUM(data_length+index_length) FROM information_schema.tables WHERE table_schema='" + database + "';").strip()),
|
||||
'nativeHistoryDatabases': len(runtime_roots)})
|
||||
private_json(work / 'migration-file-snapshot.json', manifest)
|
||||
print('Model profiles migrated:', manifest['modelProfilesMigrated'], '; MiMo excluded:', len(excluded), flush=True)
|
||||
print('Native business-history databases:', len(runtime_roots), '; files:', len(native_files), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user