820 lines
42 KiB
Python
820 lines
42 KiB
Python
#!/usr/bin/env python3
|
|
"""Create and verify a local backup using an isolated database and archive tree."""
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from contextvars import ContextVar
|
|
from datetime import datetime, timezone
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
import project
|
|
import backup_retention
|
|
|
|
ROOT = project.ROOT
|
|
BACKUP_ROOT = ROOT / 'output/backups'
|
|
REQUIRED_APPLICATION_DIRECTORIES = ('backend/src', 'web/src', 'web/public', 'scripts')
|
|
REQUIRED_APPLICATION_FILES = (
|
|
'runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
|
|
'web/index.html', 'web/tsconfig.json', 'web/vite.config.ts', 'web/pnpm-workspace.yaml',
|
|
'ai/nao-quality/RULES.md', 'ai/nao-quality/agent/prompts/system.md',
|
|
'requirements-nao.in', 'requirements-nao.lock', '.runtime/local-config.json',
|
|
)
|
|
OPTIONAL_PRIVATE_APPLICATION_FILES = ('.runtime/nao-model-config.json', 'apikey.txt')
|
|
SAFE_FAILURES = {
|
|
('INPUTS', 'APPLICATION_INPUT_MISSING'): '备份缺少必要的应用输入文件。',
|
|
('INPUTS', 'APPLICATION_INPUT_UNSAFE'): '备份应用输入包含无效链接或路径。',
|
|
('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED'): '数据库导出失败,请检查本机数据库运维状态。',
|
|
('DATABASE_EXPORT', 'DATABASE_CHANGED'): '备份期间正式数据发生变化,请重新执行备份。',
|
|
('DATABASE_EXPORT', 'DATABASE_READ_FAILED'): '数据库读取失败,请检查本机数据库运维状态。',
|
|
('ARCHIVE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。',
|
|
('ARCHIVE', 'ARCHIVE_PACK_FAILED'): '来源归档打包失败。',
|
|
('APPLICATION', 'APPLICATION_PACK_FAILED'): '应用文件打包或校验失败。',
|
|
('MANIFEST', 'MANIFEST_WRITE_FAILED'): '备份清单写入失败。',
|
|
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'): '备份校验和不匹配,恢复已停止。',
|
|
('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID'): '应用归档不完整或包含不安全路径。',
|
|
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'): '隔离数据库恢复失败,正式数据库未修改。',
|
|
('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH'): '隔离数据库内容与备份清单不一致。',
|
|
('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH'): '来源归档内容与备份清单不一致。',
|
|
('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。',
|
|
('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN'): '旧备份缺少来源归档根目录,无法验证迁移引用。',
|
|
('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID'): '数据库中的来源归档引用无法安全重绑定。',
|
|
('RESTORE', 'RESTORE_TARGET_INVALID'): '恢复目标必须是新的独立目录。',
|
|
('RESTORE', 'RESTORE_DATABASE_INVALID'): '恢复数据库必须是本项目新建的专用恢复库。',
|
|
('RESTORE', 'RESTORE_DATABASE_EXISTS'): '恢复目标数据库已存在,恢复已停止。',
|
|
('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED'): '此备份缺少安全迁移恢复所需的来源根目录或应用输入。',
|
|
('RESTORE', 'RESTORE_FAILED'): '持久恢复未完成,已清理本次创建的隔离资源。',
|
|
('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'): '恢复失败后的专用数据库清理未完成。',
|
|
('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED'): '恢复失败后的私有暂存目录清理未完成。',
|
|
('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'): '恢复失败后的专用数据库与私有暂存目录清理均未完成。',
|
|
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
|
|
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
|
|
}
|
|
CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None)
|
|
|
|
|
|
def client_config():
|
|
return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf'
|
|
|
|
|
|
def verification_temp_root():
|
|
# 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。
|
|
configuration = CLIENT_CONFIG_OVERRIDE.get()
|
|
return configuration.parent if configuration is not None else None
|
|
|
|
|
|
class BackupFailure(RuntimeError):
|
|
def __init__(self, stage, code):
|
|
if (stage, code) not in SAFE_FAILURES:
|
|
stage, code = 'PROTOCOL', 'BACKUP_INTERNAL_ERROR'
|
|
self.stage, self.code = stage, code
|
|
self.safe_message = SAFE_FAILURES[(stage, code)]
|
|
super().__init__(self.safe_message)
|
|
|
|
|
|
def fail(stage, code):
|
|
raise BackupFailure(stage, code)
|
|
|
|
|
|
def emit_failure(error):
|
|
print(json.dumps({'event': 'failure', 'stage': error.stage, 'code': error.code,
|
|
'message': error.safe_message}, ensure_ascii=False, separators=(',', ':')), flush=True)
|
|
|
|
|
|
def file_sha256(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def database_config():
|
|
value = project.config().copy()
|
|
override = os.environ.get('QUALITY_BACKUP_DATABASE')
|
|
if override:
|
|
if not re.fullmatch(r'quality_(?:verify|scale|restore)_[A-Za-z0-9_]+', override):
|
|
raise ValueError('Backup database override is restricted to this project test databases')
|
|
value['mysqlDatabase'] = override
|
|
return value
|
|
|
|
|
|
def application_files():
|
|
"""Return the fixed application snapshot, failing when any required input is absent."""
|
|
required_paths = [ROOT / relative for relative in REQUIRED_APPLICATION_FILES]
|
|
missing = [relative for relative, path in zip(REQUIRED_APPLICATION_FILES, required_paths) if not path.is_file()]
|
|
missing.extend(relative for relative in REQUIRED_APPLICATION_DIRECTORIES if not (ROOT / relative).is_dir())
|
|
if missing:
|
|
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
|
paths = set()
|
|
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'):
|
|
folder = ROOT / relative
|
|
for path in folder.rglob('*'):
|
|
if path.is_symlink():
|
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
|
if path.is_file() and '__pycache__' not in path.parts:
|
|
paths.add(path)
|
|
for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'):
|
|
path = ROOT / relative
|
|
if path.exists():
|
|
if path.is_symlink() or not path.is_file():
|
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
|
paths.add(path)
|
|
for relative in OPTIONAL_PRIVATE_APPLICATION_FILES:
|
|
path = ROOT / relative
|
|
if path.exists() or path.is_symlink():
|
|
if path.is_symlink() or not path.is_file():
|
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
|
paths.add(path)
|
|
for path in (ROOT / 'backend/target').glob('*.jar'):
|
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
|
paths.add(path)
|
|
accepted = backup_retention.accepted_backend(ROOT)
|
|
if accepted:
|
|
paths.add(accepted[0])
|
|
paths.update(backup_retention.native_files(ROOT))
|
|
if not set(required_paths).issubset(paths):
|
|
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
|
return sorted(paths)
|
|
|
|
|
|
def archive_application(folder):
|
|
archive = folder / 'application.tar.gz'
|
|
files = {}
|
|
try:
|
|
selected = application_files()
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
for path in selected:
|
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
|
relative = path.relative_to(ROOT).as_posix()
|
|
if path.name == 'nao.sqlite':
|
|
with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary:
|
|
copied = Path(temporary) / 'nao.sqlite'
|
|
backup_retention.snapshot_sqlite(path, copied)
|
|
files[relative] = file_sha256(copied)
|
|
tar.add(copied, arcname=relative, recursive=False)
|
|
continue
|
|
before = file_sha256(path)
|
|
tar.add(path, arcname=relative, recursive=False)
|
|
if before != file_sha256(path):
|
|
fail('APPLICATION', 'APPLICATION_PACK_FAILED')
|
|
files[relative] = before
|
|
archive.chmod(0o600)
|
|
return files, file_sha256(archive)
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('APPLICATION', 'APPLICATION_PACK_FAILED')
|
|
|
|
|
|
def client(sql, database=None):
|
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()),
|
|
'--batch', '--skip-column-names']
|
|
if database:
|
|
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
|
|
raise ValueError('Invalid database identifier')
|
|
args.append('--database=' + database)
|
|
try:
|
|
result = subprocess.run(args, input=sql, text=True, capture_output=True, check=False)
|
|
except Exception:
|
|
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
|
if result.returncode:
|
|
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
|
return result.stdout
|
|
|
|
|
|
def counts(database):
|
|
tables = client('SHOW TABLES;', database).splitlines()
|
|
result = {}
|
|
for table in tables:
|
|
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', table):
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
|
|
return result
|
|
|
|
|
|
def query_evidence(database):
|
|
"""Reproduce release membership and monthly formal input components in SQL."""
|
|
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
|
if not current or current == 'NULL':
|
|
return {'releaseId': None, 'months': []}
|
|
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
|
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
|
result = {'releaseId': current, 'months': []}
|
|
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
|
|
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
|
|
'bright_processing_scrap', 'bright_diffusion_scrap']
|
|
for stage in ('black', 'bright'):
|
|
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
|
|
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
|
|
for key in keys:
|
|
pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
|
|
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
|
|
result['months'].extend(json.loads(line) for line in rows.splitlines())
|
|
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
|
|
return result
|
|
|
|
|
|
def _safe_archive_member(name):
|
|
if not isinstance(name, str) or not name or '\\' in name or name.startswith('/') or re.match(r'^[A-Za-z]:', name):
|
|
return None
|
|
pure = PurePosixPath(name)
|
|
if pure.is_absolute() or any(part in ('', '.', '..') for part in pure.parts) or pure.as_posix() != name:
|
|
return None
|
|
return pure
|
|
|
|
|
|
def extract_verified_tar(archive, destination, expected, private=False, precreated=False):
|
|
"""Extract only regular, unique, manifest-listed files and verify every SHA256."""
|
|
destination = Path(destination)
|
|
if precreated:
|
|
if (destination.is_symlink() or not destination.is_dir() or any(destination.iterdir())):
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
else:
|
|
destination.mkdir(parents=True, exist_ok=False)
|
|
root = destination.resolve()
|
|
destination.chmod(0o700)
|
|
actual = {}
|
|
try:
|
|
with tarfile.open(archive, 'r:gz') as tar:
|
|
for member in tar:
|
|
pure = _safe_archive_member(member.name)
|
|
if pure is None or not member.isfile() or member.name in actual or member.name not in expected:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
target = destination.joinpath(*pure.parts)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if not target.resolve().is_relative_to(root) or target.exists() or target.is_symlink():
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
source = tar.extractfile(member)
|
|
if source is None:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
digest = hashlib.sha256()
|
|
with source, target.open('xb') as output:
|
|
for chunk in iter(lambda: source.read(1024 * 1024), b''):
|
|
output.write(chunk)
|
|
digest.update(chunk)
|
|
actual[member.name] = digest.hexdigest()
|
|
if private:
|
|
parent = target.parent
|
|
while parent != destination:
|
|
parent.chmod(0o700)
|
|
parent = parent.parent
|
|
if private or member.name in ('.runtime/local-config.json', '.runtime/nao-model-config.json', 'apikey.txt'):
|
|
target.chmod(0o600)
|
|
else:
|
|
target.chmod(member.mode & 0o755)
|
|
if member.name.startswith('.runtime/'):
|
|
target.parent.chmod(0o700)
|
|
if actual != expected:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
return actual
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
|
|
|
|
def _sql_text(value):
|
|
return "CONVERT(0x" + value.encode('utf-8').hex() + " USING utf8mb4)"
|
|
|
|
|
|
def verify_references(database, archive_root, source_root=None, rebound_root=None):
|
|
try:
|
|
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
|
|
except Exception:
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
if source_root is None:
|
|
return {'verified': False, 'rebound': False, 'count': 0}
|
|
source_root = Path(source_root)
|
|
if not source_root.is_absolute():
|
|
fail('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN')
|
|
source_root = source_root.resolve(strict=False)
|
|
archive_root = Path(archive_root).resolve()
|
|
rebound_root = Path(rebound_root).resolve() if rebound_root is not None else None
|
|
references = []
|
|
for line in rows.splitlines():
|
|
row = json.loads(line)
|
|
stored = Path(row['path'])
|
|
if not stored.is_absolute() or '..' in stored.parts or not stored.is_relative_to(source_root):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
relative = stored.relative_to(source_root)
|
|
if not relative.parts or any(part in ('', '.', '..') for part in relative.parts):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
file = archive_root.joinpath(*relative.parts)
|
|
if file.is_symlink() or not file.is_file() or not file.resolve().is_relative_to(archive_root) or file.stat().st_size != int(row['size']) or file_sha256(file) != row['sha256']:
|
|
fail('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID')
|
|
rebound = rebound_root.joinpath(*relative.parts) if rebound_root is not None else None
|
|
if rebound is not None and not rebound.resolve(strict=False).is_relative_to(rebound_root):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
references.append((row['path'], rebound, relative.as_posix()))
|
|
if rebound_root is not None:
|
|
try:
|
|
for old, rebound, _ in references:
|
|
client('UPDATE source_revision SET archive_path=' + _sql_text(str(rebound)) +
|
|
' WHERE archive_path=' + _sql_text(old) + ';', database)
|
|
rebound_rows = client("SELECT JSON_OBJECT('path',archive_path) FROM source_revision;", database)
|
|
except Exception:
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
actual_paths = {json.loads(line)['path'] for line in rebound_rows.splitlines()}
|
|
expected_paths = {str(rebound) for _, rebound, _ in references}
|
|
if actual_paths != expected_paths:
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
return {'verified': True, 'rebound': rebound_root is not None, 'count': len(references)}
|
|
|
|
|
|
def _safe_extract_tree(tar_path, directory, expected):
|
|
try:
|
|
return extract_verified_tar(tar_path, directory, expected, private=True)
|
|
except BackupFailure as error:
|
|
if error.stage == 'APPLICATION_RESTORE':
|
|
fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH')
|
|
raise
|
|
|
|
|
|
def _snapshot_counts(sql):
|
|
snapshot_counts = {}
|
|
try:
|
|
with gzip.open(sql, 'rt') as dump:
|
|
for line in dump:
|
|
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
|
|
if table:
|
|
snapshot_counts[table.group(1)] = 0
|
|
row = re.match(r'^INSERT INTO `([^`]+)`', line)
|
|
if row:
|
|
snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
|
|
except Exception:
|
|
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
|
|
return snapshot_counts
|
|
|
|
|
|
@backup_retention.serialized
|
|
def create():
|
|
# Validate the full, fixed application input set before creating any backup files.
|
|
selected_application = application_files()
|
|
config = database_config()
|
|
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
|
|
before_queries = query_evidence(config['mysqlDatabase'])
|
|
before_counts = counts(config['mysqlDatabase'])
|
|
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
|
folder.mkdir(parents=True, exist_ok=False)
|
|
folder.chmod(0o700)
|
|
backup_retention.register_candidate(folder, ROOT)
|
|
sql = folder / 'database.sql.gz'
|
|
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
|
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
|
'--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']]
|
|
process = None
|
|
try:
|
|
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
|
backup_retention.candidate_child(folder, pending=True)
|
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
|
backup_retention.candidate_child(folder, process.pid)
|
|
try:
|
|
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
|
|
shutil.copyfileobj(process.stdout, compressed)
|
|
process.stdout.close()
|
|
if process.wait() != 0:
|
|
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
|
|
except BaseException:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
|
|
finally:
|
|
if process is not None and process.stdout is not None and not process.stdout.closed:
|
|
process.stdout.close()
|
|
if process is not None and process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
sql.chmod(0o600)
|
|
if query_evidence(config['mysqlDatabase']) != before_queries:
|
|
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
|
# The dump is a single consistent transaction. Restore verification below
|
|
# independently checks these counts, including a net-zero concurrent race.
|
|
if counts(config['mysqlDatabase']) != before_counts:
|
|
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
|
table_counts = before_counts
|
|
archive = folder / 'archive.tar.gz'
|
|
source = ROOT / 'runtimedata/archive'
|
|
files = {}
|
|
process = None
|
|
try:
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
if source.exists():
|
|
for file in sorted(source.rglob('*')):
|
|
if file.is_symlink():
|
|
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
|
|
if file.is_file():
|
|
relative = file.relative_to(source).as_posix()
|
|
before = file_sha256(file)
|
|
tar.add(file, arcname=relative, recursive=False)
|
|
if before != file_sha256(file):
|
|
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
|
|
files[relative] = before
|
|
archive.chmod(0o600)
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
|
|
app_files, app_hash = archive_application(folder)
|
|
archive_source_root = source.resolve()
|
|
manifest = {'formatVersion': 3, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
|
|
'databaseDumpSha256': file_sha256(sql), 'archiveSha256': file_sha256(archive), 'files': files,
|
|
'archiveSourceRoot': str(archive_source_root),
|
|
'applicationSha256': app_hash, 'applicationFiles': app_files,
|
|
'requiredApplicationFiles': list(REQUIRED_APPLICATION_FILES),
|
|
'privateConfigurationFiles': [name for name in OPTIONAL_PRIVATE_APPLICATION_FILES if name in app_files],
|
|
'applicationRuntimePrerequisites': ['匹配的JDK/Maven/MySQL运行时', 'Node.js与pnpm',
|
|
'requirements-nao.lock对应的Python/NAO运行时'],
|
|
'containsPrivateConfiguration': True,
|
|
'recoveryMode': 'BACKUP_NOT_YET_RESTORE_VERIFIED',
|
|
'verification': {'databaseRestoreVerified': False, 'applicationFilesVerified': False,
|
|
'requiredApplicationInputsVerified': False, 'archiveFilesVerified': False,
|
|
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
|
|
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
|
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
|
accepted = backup_retention.accepted_backend(ROOT)
|
|
if accepted:
|
|
manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]}
|
|
try:
|
|
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
|
except Exception:
|
|
fail('MANIFEST', 'MANIFEST_WRITE_FAILED')
|
|
return folder
|
|
|
|
|
|
def _verify(folder):
|
|
started = time.monotonic()
|
|
folder = Path(folder)
|
|
if folder.is_symlink():
|
|
raise ValueError('Backup directory cannot be a symbolic link')
|
|
folder = folder.resolve()
|
|
if not folder.is_relative_to(BACKUP_ROOT.resolve()):
|
|
raise ValueError('Only this project backup directory may be verified')
|
|
for artifact in (folder / 'manifest.json', folder / 'database.sql.gz', folder / 'archive.tar.gz', folder / 'application.tar.gz'):
|
|
if artifact.is_symlink():
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
try:
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
except Exception:
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
sql, archive = folder / 'database.sql.gz', folder / 'archive.tar.gz'
|
|
try:
|
|
if file_sha256(sql) != manifest['databaseDumpSha256'] or file_sha256(archive) != manifest['archiveSha256']:
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
app_archive = folder / 'application.tar.gz'
|
|
expected_app = manifest.get('applicationFiles')
|
|
application_verified = False
|
|
required_inputs_verified = False
|
|
if manifest.get('applicationSha256') and isinstance(expected_app, dict):
|
|
try:
|
|
if file_sha256(app_archive) != manifest['applicationSha256']:
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp:
|
|
Path(temp).chmod(0o700)
|
|
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
|
|
application_verified = actual_app == expected_app
|
|
required_inputs_verified = (set(REQUIRED_APPLICATION_FILES).issubset(expected_app) and
|
|
set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', [])))
|
|
except BackupFailure as error:
|
|
if error.stage == 'APPLICATION_RESTORE':
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
raise
|
|
except Exception:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
|
created_scratch = False
|
|
scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip())
|
|
try:
|
|
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
created_scratch = True
|
|
backup_retention.scratch_created(scratch_registration)
|
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
|
|
process = None
|
|
try:
|
|
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
|
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
|
backup_retention.scratch_child(scratch_registration, process.pid)
|
|
try:
|
|
shutil.copyfileobj(data, process.stdin)
|
|
process.stdin.write(b'\nCOMMIT;\n')
|
|
process.stdin.close()
|
|
code = process.wait()
|
|
except BaseException:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
if code != 0:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
restored = counts(scratch)
|
|
if restored != manifest.get('tableCounts'):
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp:
|
|
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
|
|
relocated_root.parent.mkdir(parents=True, exist_ok=True)
|
|
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
|
|
archive_files_verified = actual_archives == manifest.get('files', {})
|
|
source_root = manifest.get('archiveSourceRoot')
|
|
references = verify_references(scratch, relocated_root, source_root,
|
|
relocated_root if source_root else None)
|
|
if references['verified']:
|
|
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", scratch)
|
|
for line in rows.splitlines():
|
|
row = json.loads(line)
|
|
path = Path(row['path'])
|
|
if not path.is_relative_to(relocated_root.resolve()) or not path.is_file() or file_sha256(path) != row['sha256'] or path.stat().st_size != int(row['size']):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
|
|
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
|
|
manifest['restoredTables'] = len(restored)
|
|
manifest['restoredApplicationFiles'] = len(expected_app or {})
|
|
manifest['restoredArchiveFiles'] = len(manifest.get('files', {}))
|
|
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
|
|
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
|
|
manifest['recoveryMode'] = 'ISOLATED_DATA_AND_FILE_CHECKS_NO_APPLICATION_STARTUP'
|
|
manifest['verification'] = {'databaseRestoreVerified': True, 'applicationFilesVerified': application_verified,
|
|
'requiredApplicationInputsVerified': required_inputs_verified,
|
|
'archiveFilesVerified': archive_files_verified,
|
|
'archiveReferencesVerified': references['verified'] and references['rebound'],
|
|
'applicationStartupVerified': False}
|
|
manifest['archiveReferencesRebound'] = bool(references['verified'] and references['rebound'])
|
|
manifest['migrationReady'] = False
|
|
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
|
return {'status': manifest['status'], 'restoredTables': len(restored),
|
|
'restoredApplicationFiles': len(expected_app or {}), 'restoredArchiveFiles': len(manifest.get('files', {})),
|
|
'verification': manifest['verification'], 'archiveReferencesRebound': manifest['archiveReferencesRebound'],
|
|
'migrationReady': manifest['migrationReady'],
|
|
'elapsedSeconds': manifest['restoreElapsedSeconds']}
|
|
finally:
|
|
if created_scratch:
|
|
try:
|
|
client('DROP DATABASE `' + scratch + '`;')
|
|
scratch_registration.unlink(missing_ok=True)
|
|
except Exception:
|
|
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
|
|
else:
|
|
scratch_registration.unlink(missing_ok=True)
|
|
|
|
|
|
@backup_retention.serialized
|
|
def verify(folder):
|
|
if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1':
|
|
import backup_verifier
|
|
with backup_verifier.instance(ROOT) as configuration:
|
|
token = CLIENT_CONFIG_OVERRIDE.set(configuration)
|
|
try:
|
|
return _verify(folder)
|
|
finally:
|
|
CLIENT_CONFIG_OVERRIDE.reset(token)
|
|
return _verify(folder)
|
|
|
|
|
|
@backup_retention.serialized
|
|
def restore(folder, target, database):
|
|
"""Restore a verified format-v3 backup to a new project directory and new database.
|
|
|
|
The destination is assembled beside its final path and atomically renamed only after
|
|
the SQL, application files, and relocated source references pass their checks.
|
|
Existing destinations or databases are never overwritten or removed.
|
|
"""
|
|
folder = Path(folder)
|
|
target = Path(target).expanduser()
|
|
if folder.is_symlink():
|
|
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
|
|
folder = folder.resolve()
|
|
if not folder.is_relative_to(BACKUP_ROOT.resolve()):
|
|
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
|
|
if (not target.is_absolute() or target.exists() or target.is_symlink() or
|
|
not target.parent.is_dir() or target.name in ('', '.', '..')):
|
|
fail('RESTORE', 'RESTORE_TARGET_INVALID')
|
|
target = target.parent.resolve() / target.name
|
|
if target == ROOT.resolve() or ROOT.resolve().is_relative_to(target):
|
|
fail('RESTORE', 'RESTORE_TARGET_INVALID')
|
|
if target.is_relative_to(BACKUP_ROOT.resolve()) or folder.is_relative_to(target):
|
|
fail('RESTORE', 'RESTORE_TARGET_INVALID')
|
|
if not re.fullmatch(r'quality_restore_[A-Za-z0-9_]{1,48}', database):
|
|
fail('RESTORE', 'RESTORE_DATABASE_INVALID')
|
|
|
|
try:
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
sql, archive, app_archive = (folder / 'database.sql.gz', folder / 'archive.tar.gz',
|
|
folder / 'application.tar.gz')
|
|
if (manifest.get('formatVersion') != 3 or not Path(manifest.get('archiveSourceRoot', '')).is_absolute() or
|
|
not isinstance(manifest.get('applicationFiles'), dict) or
|
|
not set(REQUIRED_APPLICATION_FILES).issubset(manifest['applicationFiles']) or
|
|
not set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', []))):
|
|
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
|
|
if any(path.is_symlink() for path in (sql, archive, app_archive)):
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
if (file_sha256(sql) != manifest['databaseDumpSha256'] or
|
|
file_sha256(archive) != manifest['archiveSha256'] or
|
|
file_sha256(app_archive) != manifest['applicationSha256']):
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
app_files = manifest['applicationFiles']
|
|
if (not isinstance(manifest.get('files'), dict) or
|
|
not isinstance(manifest.get('tableCounts'), dict)):
|
|
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
|
|
|
try:
|
|
existing = set(client('SHOW DATABASES;').splitlines())
|
|
except BackupFailure:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
if database in existing:
|
|
fail('RESTORE', 'RESTORE_DATABASE_EXISTS')
|
|
|
|
stage = target.parent / ('.' + target.name + '.restore-' + uuid.uuid4().hex[:12])
|
|
created_database = False
|
|
stage_owned = False
|
|
completed = False
|
|
try:
|
|
try:
|
|
client('CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
created_database = True
|
|
except BackupFailure:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
|
|
stage.mkdir(mode=0o700, exist_ok=False)
|
|
stage_owned = True
|
|
app_verified = extract_verified_tar(app_archive, stage, app_files, private=True, precreated=True) == app_files
|
|
if not set(REQUIRED_APPLICATION_FILES).issubset(app_verified and app_files):
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
try:
|
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
|
'--database=' + database]
|
|
process = None
|
|
with gzip.open(sql, 'rb') as dump, tempfile.TemporaryFile() as errors:
|
|
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
|
try:
|
|
shutil.copyfileobj(dump, process.stdin)
|
|
process.stdin.close()
|
|
code = process.wait()
|
|
except BaseException:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
if code != 0:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
except BackupFailure:
|
|
raise
|
|
except Exception:
|
|
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
|
|
|
|
restored = counts(database)
|
|
if restored != manifest.get('tableCounts'):
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
if manifest.get('queryEvidence') is not None and query_evidence(database) != manifest['queryEvidence']:
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
|
|
archive_root = stage / 'runtimedata/archive'
|
|
archive_root.parent.mkdir(parents=True, exist_ok=True)
|
|
actual_archives = _safe_extract_tree(archive, archive_root, manifest['files'])
|
|
if actual_archives != manifest['files']:
|
|
fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH')
|
|
final_archive_root = target / 'runtimedata/archive'
|
|
references = verify_references(database, archive_root, manifest['archiveSourceRoot'], final_archive_root)
|
|
if not references['verified'] or not references['rebound']:
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
|
|
for line in rows.splitlines():
|
|
row = json.loads(line)
|
|
path = Path(row['path'])
|
|
if not path.is_relative_to(final_archive_root.resolve(strict=False)):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
staged_file = archive_root / path.relative_to(final_archive_root.resolve(strict=False))
|
|
if (staged_file.is_symlink() or not staged_file.is_file() or
|
|
staged_file.stat().st_size != int(row['size']) or file_sha256(staged_file) != row['sha256']):
|
|
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
|
|
if query_evidence(database) != manifest.get('queryEvidence'):
|
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
|
|
|
config_path = stage / '.runtime/local-config.json'
|
|
try:
|
|
config = json.loads(config_path.read_text())
|
|
config['mysqlDatabase'] = database
|
|
config['databaseReady'] = True
|
|
config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2) + '\n')
|
|
config_path.chmod(0o600)
|
|
except Exception:
|
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
|
|
receipt = {'status': 'ISOLATED_DATA_FILE_RESTORE_VERIFIED',
|
|
'formatVersion': manifest['formatVersion'], 'restoredTables': len(restored),
|
|
'restoredApplicationFiles': len(app_files), 'restoredArchiveFiles': len(actual_archives),
|
|
'verification': {'databaseRestoreVerified': True, 'applicationFilesVerified': True,
|
|
'requiredApplicationInputsVerified': True, 'archiveFilesVerified': True,
|
|
'archiveReferencesVerified': True, 'applicationStartupVerified': False},
|
|
'migrationReady': False,
|
|
'note': 'Build, schema migration, and application startup require separate verification.'}
|
|
receipt_path = stage / 'restore-receipt.json'
|
|
receipt_path.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n')
|
|
receipt_path.chmod(0o600)
|
|
if target.exists() or target.is_symlink():
|
|
fail('RESTORE', 'RESTORE_TARGET_INVALID')
|
|
os.rename(stage, target)
|
|
stage_owned = False
|
|
completed = True
|
|
return receipt
|
|
finally:
|
|
if not completed:
|
|
database_cleanup_failed = False
|
|
stage_cleanup_failed = False
|
|
if created_database:
|
|
try:
|
|
client('DROP DATABASE `' + database + '`;')
|
|
except Exception:
|
|
database_cleanup_failed = True
|
|
if stage_owned and stage.exists() and not stage.is_symlink():
|
|
try:
|
|
shutil.rmtree(stage)
|
|
except Exception:
|
|
stage_cleanup_failed = True
|
|
if database_cleanup_failed and stage_cleanup_failed:
|
|
fail('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED')
|
|
if database_cleanup_failed:
|
|
fail('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED')
|
|
if stage_cleanup_failed:
|
|
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
|
|
|
|
|
|
def _main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
|
|
parser.add_argument('--directory', type=Path)
|
|
parser.add_argument('--target', type=Path)
|
|
parser.add_argument('--database')
|
|
args = parser.parse_args(argv)
|
|
previous_backups = backup_retention.backup_ids(ROOT)
|
|
try:
|
|
if args.action == 'verify':
|
|
if not args.directory:
|
|
parser.error('--directory is required for verify')
|
|
result = verify(args.directory)
|
|
elif args.action == 'restore':
|
|
if not args.directory or not args.target or not args.database:
|
|
parser.error('restore requires --directory, --target, and --database')
|
|
result = restore(args.directory, args.target, args.database)
|
|
else:
|
|
folder = create()
|
|
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
|
if args.action == 'create-and-verify':
|
|
result = verify(folder)
|
|
if args.action in ('verify', 'create-and-verify'):
|
|
selected = args.directory if args.action == 'verify' else folder
|
|
backup_retention.rotate(ROOT, selected)
|
|
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
|
|
separators=(',', ':')), flush=True)
|
|
return 0
|
|
except BackupFailure as error:
|
|
backup_retention.discard_failed_new(ROOT, previous_backups)
|
|
emit_failure(error)
|
|
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
|
|
except Exception:
|
|
backup_retention.discard_failed_new(ROOT, previous_backups)
|
|
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
|
|
emit_failure(error)
|
|
return 1
|
|
|
|
|
|
def main(argv=None):
|
|
try:
|
|
with backup_retention.lock(ROOT):
|
|
return _main(argv)
|
|
except backup_retention.Busy:
|
|
emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR'))
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|