Files
quality_analyse/scripts/backup.py
T

819 lines
42 KiB
Python

#!/usr/bin/env python3
"""Create and verify a local backup using an isolated database and archive tree."""
import argparse
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime, timezone
import gzip
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import uuid
import project
import backup_retention
ROOT = project.ROOT
BACKUP_ROOT = ROOT / 'output/backups'
REQUIRED_APPLICATION_DIRECTORIES = ('backend/src', 'web/src', 'web/public', 'scripts')
REQUIRED_APPLICATION_FILES = (
'runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
'web/index.html', 'web/tsconfig.json', 'web/vite.config.ts', 'web/pnpm-workspace.yaml',
'ai/nao-quality/RULES.md', 'ai/nao-quality/agent/prompts/system.md',
'requirements-nao.in', 'requirements-nao.lock', '.runtime/local-config.json',
)
OPTIONAL_PRIVATE_APPLICATION_FILES = ('.runtime/nao-model-config.json', 'apikey.txt')
SAFE_FAILURES = {
('INPUTS', 'APPLICATION_INPUT_MISSING'): '备份缺少必要的应用输入文件。',
('INPUTS', 'APPLICATION_INPUT_UNSAFE'): '备份应用输入包含无效链接或路径。',
('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED'): '数据库导出失败,请检查本机数据库运维状态。',
('DATABASE_EXPORT', 'DATABASE_CHANGED'): '备份期间正式数据发生变化,请重新执行备份。',
('DATABASE_EXPORT', 'DATABASE_READ_FAILED'): '数据库读取失败,请检查本机数据库运维状态。',
('ARCHIVE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。',
('ARCHIVE', 'ARCHIVE_PACK_FAILED'): '来源归档打包失败。',
('APPLICATION', 'APPLICATION_PACK_FAILED'): '应用文件打包或校验失败。',
('MANIFEST', 'MANIFEST_WRITE_FAILED'): '备份清单写入失败。',
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'): '备份校验和不匹配,恢复已停止。',
('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID'): '应用归档不完整或包含不安全路径。',
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'): '隔离数据库恢复失败,正式数据库未修改。',
('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH'): '隔离数据库内容与备份清单不一致。',
('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH'): '来源归档内容与备份清单不一致。',
('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。',
('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN'): '旧备份缺少来源归档根目录,无法验证迁移引用。',
('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID'): '数据库中的来源归档引用无法安全重绑定。',
('RESTORE', 'RESTORE_TARGET_INVALID'): '恢复目标必须是新的独立目录。',
('RESTORE', 'RESTORE_DATABASE_INVALID'): '恢复数据库必须是本项目新建的专用恢复库。',
('RESTORE', 'RESTORE_DATABASE_EXISTS'): '恢复目标数据库已存在,恢复已停止。',
('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED'): '此备份缺少安全迁移恢复所需的来源根目录或应用输入。',
('RESTORE', 'RESTORE_FAILED'): '持久恢复未完成,已清理本次创建的隔离资源。',
('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'): '恢复失败后的专用数据库清理未完成。',
('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED'): '恢复失败后的私有暂存目录清理未完成。',
('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'): '恢复失败后的专用数据库与私有暂存目录清理均未完成。',
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
}
CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None)
def client_config():
return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf'
def verification_temp_root():
# 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。
configuration = CLIENT_CONFIG_OVERRIDE.get()
return configuration.parent if configuration is not None else None
class BackupFailure(RuntimeError):
def __init__(self, stage, code):
if (stage, code) not in SAFE_FAILURES:
stage, code = 'PROTOCOL', 'BACKUP_INTERNAL_ERROR'
self.stage, self.code = stage, code
self.safe_message = SAFE_FAILURES[(stage, code)]
super().__init__(self.safe_message)
def fail(stage, code):
raise BackupFailure(stage, code)
def emit_failure(error):
print(json.dumps({'event': 'failure', 'stage': error.stage, 'code': error.code,
'message': error.safe_message}, ensure_ascii=False, separators=(',', ':')), flush=True)
def file_sha256(path):
digest = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
digest.update(chunk)
return digest.hexdigest()
def database_config():
value = project.config().copy()
override = os.environ.get('QUALITY_BACKUP_DATABASE')
if override:
if not re.fullmatch(r'quality_(?:verify|scale|restore)_[A-Za-z0-9_]+', override):
raise ValueError('Backup database override is restricted to this project test databases')
value['mysqlDatabase'] = override
return value
def application_files():
"""Return the fixed application snapshot, failing when any required input is absent."""
required_paths = [ROOT / relative for relative in REQUIRED_APPLICATION_FILES]
missing = [relative for relative, path in zip(REQUIRED_APPLICATION_FILES, required_paths) if not path.is_file()]
missing.extend(relative for relative in REQUIRED_APPLICATION_DIRECTORIES if not (ROOT / relative).is_dir())
if missing:
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
paths = set()
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'):
folder = ROOT / relative
for path in folder.rglob('*'):
if path.is_symlink():
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
if path.is_file() and '__pycache__' not in path.parts:
paths.add(path)
for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'):
path = ROOT / relative
if path.exists():
if path.is_symlink() or not path.is_file():
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
paths.add(path)
for relative in OPTIONAL_PRIVATE_APPLICATION_FILES:
path = ROOT / relative
if path.exists() or path.is_symlink():
if path.is_symlink() or not path.is_file():
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
paths.add(path)
for path in (ROOT / 'backend/target').glob('*.jar'):
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
paths.add(path)
accepted = backup_retention.accepted_backend(ROOT)
if accepted:
paths.add(accepted[0])
paths.update(backup_retention.native_files(ROOT))
if not set(required_paths).issubset(paths):
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
return sorted(paths)
def archive_application(folder):
archive = folder / 'application.tar.gz'
files = {}
try:
selected = application_files()
with tarfile.open(archive, 'w:gz') as tar:
for path in selected:
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
relative = path.relative_to(ROOT).as_posix()
if path.name == 'nao.sqlite':
with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary:
copied = Path(temporary) / 'nao.sqlite'
backup_retention.snapshot_sqlite(path, copied)
files[relative] = file_sha256(copied)
tar.add(copied, arcname=relative, recursive=False)
continue
before = file_sha256(path)
tar.add(path, arcname=relative, recursive=False)
if before != file_sha256(path):
fail('APPLICATION', 'APPLICATION_PACK_FAILED')
files[relative] = before
archive.chmod(0o600)
return files, file_sha256(archive)
except BackupFailure:
raise
except Exception:
fail('APPLICATION', 'APPLICATION_PACK_FAILED')
def client(sql, database=None):
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()),
'--batch', '--skip-column-names']
if database:
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
raise ValueError('Invalid database identifier')
args.append('--database=' + database)
try:
result = subprocess.run(args, input=sql, text=True, capture_output=True, check=False)
except Exception:
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
if result.returncode:
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
return result.stdout
def counts(database):
tables = client('SHOW TABLES;', database).splitlines()
result = {}
for table in tables:
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', table):
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
return result
def query_evidence(database):
"""Reproduce release membership and monthly formal input components in SQL."""
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
if not current or current == 'NULL':
return {'releaseId': None, 'months': []}
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
result = {'releaseId': current, 'months': []}
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
'bright_processing_scrap', 'bright_diffusion_scrap']
for stage in ('black', 'bright'):
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
for key in keys:
pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
result['months'].extend(json.loads(line) for line in rows.splitlines())
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
return result
def _safe_archive_member(name):
if not isinstance(name, str) or not name or '\\' in name or name.startswith('/') or re.match(r'^[A-Za-z]:', name):
return None
pure = PurePosixPath(name)
if pure.is_absolute() or any(part in ('', '.', '..') for part in pure.parts) or pure.as_posix() != name:
return None
return pure
def extract_verified_tar(archive, destination, expected, private=False, precreated=False):
"""Extract only regular, unique, manifest-listed files and verify every SHA256."""
destination = Path(destination)
if precreated:
if (destination.is_symlink() or not destination.is_dir() or any(destination.iterdir())):
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
else:
destination.mkdir(parents=True, exist_ok=False)
root = destination.resolve()
destination.chmod(0o700)
actual = {}
try:
with tarfile.open(archive, 'r:gz') as tar:
for member in tar:
pure = _safe_archive_member(member.name)
if pure is None or not member.isfile() or member.name in actual or member.name not in expected:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
target = destination.joinpath(*pure.parts)
target.parent.mkdir(parents=True, exist_ok=True)
if not target.resolve().is_relative_to(root) or target.exists() or target.is_symlink():
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
source = tar.extractfile(member)
if source is None:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
digest = hashlib.sha256()
with source, target.open('xb') as output:
for chunk in iter(lambda: source.read(1024 * 1024), b''):
output.write(chunk)
digest.update(chunk)
actual[member.name] = digest.hexdigest()
if private:
parent = target.parent
while parent != destination:
parent.chmod(0o700)
parent = parent.parent
if private or member.name in ('.runtime/local-config.json', '.runtime/nao-model-config.json', 'apikey.txt'):
target.chmod(0o600)
else:
target.chmod(member.mode & 0o755)
if member.name.startswith('.runtime/'):
target.parent.chmod(0o700)
if actual != expected:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
return actual
except BackupFailure:
raise
except Exception:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
def _sql_text(value):
return "CONVERT(0x" + value.encode('utf-8').hex() + " USING utf8mb4)"
def verify_references(database, archive_root, source_root=None, rebound_root=None):
try:
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
except Exception:
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
if source_root is None:
return {'verified': False, 'rebound': False, 'count': 0}
source_root = Path(source_root)
if not source_root.is_absolute():
fail('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN')
source_root = source_root.resolve(strict=False)
archive_root = Path(archive_root).resolve()
rebound_root = Path(rebound_root).resolve() if rebound_root is not None else None
references = []
for line in rows.splitlines():
row = json.loads(line)
stored = Path(row['path'])
if not stored.is_absolute() or '..' in stored.parts or not stored.is_relative_to(source_root):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
relative = stored.relative_to(source_root)
if not relative.parts or any(part in ('', '.', '..') for part in relative.parts):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
file = archive_root.joinpath(*relative.parts)
if file.is_symlink() or not file.is_file() or not file.resolve().is_relative_to(archive_root) or file.stat().st_size != int(row['size']) or file_sha256(file) != row['sha256']:
fail('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID')
rebound = rebound_root.joinpath(*relative.parts) if rebound_root is not None else None
if rebound is not None and not rebound.resolve(strict=False).is_relative_to(rebound_root):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
references.append((row['path'], rebound, relative.as_posix()))
if rebound_root is not None:
try:
for old, rebound, _ in references:
client('UPDATE source_revision SET archive_path=' + _sql_text(str(rebound)) +
' WHERE archive_path=' + _sql_text(old) + ';', database)
rebound_rows = client("SELECT JSON_OBJECT('path',archive_path) FROM source_revision;", database)
except Exception:
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
actual_paths = {json.loads(line)['path'] for line in rebound_rows.splitlines()}
expected_paths = {str(rebound) for _, rebound, _ in references}
if actual_paths != expected_paths:
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
return {'verified': True, 'rebound': rebound_root is not None, 'count': len(references)}
def _safe_extract_tree(tar_path, directory, expected):
try:
return extract_verified_tar(tar_path, directory, expected, private=True)
except BackupFailure as error:
if error.stage == 'APPLICATION_RESTORE':
fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH')
raise
def _snapshot_counts(sql):
snapshot_counts = {}
try:
with gzip.open(sql, 'rt') as dump:
for line in dump:
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
if table:
snapshot_counts[table.group(1)] = 0
row = re.match(r'^INSERT INTO `([^`]+)`', line)
if row:
snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
except Exception:
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
return snapshot_counts
@backup_retention.serialized
def create():
# Validate the full, fixed application input set before creating any backup files.
selected_application = application_files()
config = database_config()
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
before_queries = query_evidence(config['mysqlDatabase'])
before_counts = counts(config['mysqlDatabase'])
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
'--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']]
process = None
try:
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
try:
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
shutil.copyfileobj(process.stdout, compressed)
process.stdout.close()
if process.wait() != 0:
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
except BackupFailure:
raise
except Exception:
fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED')
finally:
if process is not None and process.stdout is not None and not process.stdout.closed:
process.stdout.close()
if process is not None and process.poll() is None:
process.kill()
process.wait()
sql.chmod(0o600)
if query_evidence(config['mysqlDatabase']) != before_queries:
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
# The dump is a single consistent transaction. Restore verification below
# independently checks these counts, including a net-zero concurrent race.
if counts(config['mysqlDatabase']) != before_counts:
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
table_counts = before_counts
archive = folder / 'archive.tar.gz'
source = ROOT / 'runtimedata/archive'
files = {}
process = None
try:
with tarfile.open(archive, 'w:gz') as tar:
if source.exists():
for file in sorted(source.rglob('*')):
if file.is_symlink():
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
if file.is_file():
relative = file.relative_to(source).as_posix()
before = file_sha256(file)
tar.add(file, arcname=relative, recursive=False)
if before != file_sha256(file):
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
files[relative] = before
archive.chmod(0o600)
except BackupFailure:
raise
except Exception:
fail('ARCHIVE', 'ARCHIVE_PACK_FAILED')
app_files, app_hash = archive_application(folder)
archive_source_root = source.resolve()
manifest = {'formatVersion': 3, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
'databaseDumpSha256': file_sha256(sql), 'archiveSha256': file_sha256(archive), 'files': files,
'archiveSourceRoot': str(archive_source_root),
'applicationSha256': app_hash, 'applicationFiles': app_files,
'requiredApplicationFiles': list(REQUIRED_APPLICATION_FILES),
'privateConfigurationFiles': [name for name in OPTIONAL_PRIVATE_APPLICATION_FILES if name in app_files],
'applicationRuntimePrerequisites': ['匹配的JDK/Maven/MySQL运行时', 'Node.js与pnpm',
'requirements-nao.lock对应的Python/NAO运行时'],
'containsPrivateConfiguration': True,
'recoveryMode': 'BACKUP_NOT_YET_RESTORE_VERIFIED',
'verification': {'databaseRestoreVerified': False, 'applicationFilesVerified': False,
'requiredApplicationInputsVerified': False, 'archiveFilesVerified': False,
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
accepted = backup_retention.accepted_backend(ROOT)
if accepted:
manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]}
try:
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
except Exception:
fail('MANIFEST', 'MANIFEST_WRITE_FAILED')
return folder
def _verify(folder):
started = time.monotonic()
folder = Path(folder)
if folder.is_symlink():
raise ValueError('Backup directory cannot be a symbolic link')
folder = folder.resolve()
if not folder.is_relative_to(BACKUP_ROOT.resolve()):
raise ValueError('Only this project backup directory may be verified')
for artifact in (folder / 'manifest.json', folder / 'database.sql.gz', folder / 'archive.tar.gz', folder / 'application.tar.gz'):
if artifact.is_symlink():
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
try:
manifest = json.loads((folder / 'manifest.json').read_text())
except Exception:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
sql, archive = folder / 'database.sql.gz', folder / 'archive.tar.gz'
try:
if file_sha256(sql) != manifest['databaseDumpSha256'] or file_sha256(archive) != manifest['archiveSha256']:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
except BackupFailure:
raise
except Exception:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
app_archive = folder / 'application.tar.gz'
expected_app = manifest.get('applicationFiles')
application_verified = False
required_inputs_verified = False
if manifest.get('applicationSha256') and isinstance(expected_app, dict):
try:
if file_sha256(app_archive) != manifest['applicationSha256']:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp:
Path(temp).chmod(0o700)
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
application_verified = actual_app == expected_app
required_inputs_verified = (set(REQUIRED_APPLICATION_FILES).issubset(expected_app) and
set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', [])))
except BackupFailure as error:
if error.stage == 'APPLICATION_RESTORE':
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
raise
except Exception:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
created_scratch = False
scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip())
try:
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
created_scratch = True
backup_retention.scratch_created(scratch_registration)
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
process = None
try:
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
backup_retention.scratch_child(scratch_registration, process.pid)
try:
shutil.copyfileobj(data, process.stdin)
process.stdin.write(b'\nCOMMIT;\n')
process.stdin.close()
code = process.wait()
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
if code != 0:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
except BackupFailure:
raise
except Exception:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
restored = counts(scratch)
if restored != manifest.get('tableCounts'):
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp:
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
relocated_root.parent.mkdir(parents=True, exist_ok=True)
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
archive_files_verified = actual_archives == manifest.get('files', {})
source_root = manifest.get('archiveSourceRoot')
references = verify_references(scratch, relocated_root, source_root,
relocated_root if source_root else None)
if references['verified']:
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", scratch)
for line in rows.splitlines():
row = json.loads(line)
path = Path(row['path'])
if not path.is_relative_to(relocated_root.resolve()) or not path.is_file() or file_sha256(path) != row['sha256'] or path.stat().st_size != int(row['size']):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
manifest['restoredTables'] = len(restored)
manifest['restoredApplicationFiles'] = len(expected_app or {})
manifest['restoredArchiveFiles'] = len(manifest.get('files', {}))
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
manifest['recoveryMode'] = 'ISOLATED_DATA_AND_FILE_CHECKS_NO_APPLICATION_STARTUP'
manifest['verification'] = {'databaseRestoreVerified': True, 'applicationFilesVerified': application_verified,
'requiredApplicationInputsVerified': required_inputs_verified,
'archiveFilesVerified': archive_files_verified,
'archiveReferencesVerified': references['verified'] and references['rebound'],
'applicationStartupVerified': False}
manifest['archiveReferencesRebound'] = bool(references['verified'] and references['rebound'])
manifest['migrationReady'] = False
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
return {'status': manifest['status'], 'restoredTables': len(restored),
'restoredApplicationFiles': len(expected_app or {}), 'restoredArchiveFiles': len(manifest.get('files', {})),
'verification': manifest['verification'], 'archiveReferencesRebound': manifest['archiveReferencesRebound'],
'migrationReady': manifest['migrationReady'],
'elapsedSeconds': manifest['restoreElapsedSeconds']}
finally:
if created_scratch:
try:
client('DROP DATABASE `' + scratch + '`;')
scratch_registration.unlink(missing_ok=True)
except Exception:
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
else:
scratch_registration.unlink(missing_ok=True)
@backup_retention.serialized
def verify(folder):
if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1':
import backup_verifier
with backup_verifier.instance(ROOT) as configuration:
token = CLIENT_CONFIG_OVERRIDE.set(configuration)
try:
return _verify(folder)
finally:
CLIENT_CONFIG_OVERRIDE.reset(token)
return _verify(folder)
@backup_retention.serialized
def restore(folder, target, database):
"""Restore a verified format-v3 backup to a new project directory and new database.
The destination is assembled beside its final path and atomically renamed only after
the SQL, application files, and relocated source references pass their checks.
Existing destinations or databases are never overwritten or removed.
"""
folder = Path(folder)
target = Path(target).expanduser()
if folder.is_symlink():
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
folder = folder.resolve()
if not folder.is_relative_to(BACKUP_ROOT.resolve()):
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
if (not target.is_absolute() or target.exists() or target.is_symlink() or
not target.parent.is_dir() or target.name in ('', '.', '..')):
fail('RESTORE', 'RESTORE_TARGET_INVALID')
target = target.parent.resolve() / target.name
if target == ROOT.resolve() or ROOT.resolve().is_relative_to(target):
fail('RESTORE', 'RESTORE_TARGET_INVALID')
if target.is_relative_to(BACKUP_ROOT.resolve()) or folder.is_relative_to(target):
fail('RESTORE', 'RESTORE_TARGET_INVALID')
if not re.fullmatch(r'quality_restore_[A-Za-z0-9_]{1,48}', database):
fail('RESTORE', 'RESTORE_DATABASE_INVALID')
try:
manifest = json.loads((folder / 'manifest.json').read_text())
sql, archive, app_archive = (folder / 'database.sql.gz', folder / 'archive.tar.gz',
folder / 'application.tar.gz')
if (manifest.get('formatVersion') != 3 or not Path(manifest.get('archiveSourceRoot', '')).is_absolute() or
not isinstance(manifest.get('applicationFiles'), dict) or
not set(REQUIRED_APPLICATION_FILES).issubset(manifest['applicationFiles']) or
not set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', []))):
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
if any(path.is_symlink() for path in (sql, archive, app_archive)):
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
if (file_sha256(sql) != manifest['databaseDumpSha256'] or
file_sha256(archive) != manifest['archiveSha256'] or
file_sha256(app_archive) != manifest['applicationSha256']):
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
app_files = manifest['applicationFiles']
if (not isinstance(manifest.get('files'), dict) or
not isinstance(manifest.get('tableCounts'), dict)):
fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED')
except BackupFailure:
raise
except Exception:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
try:
existing = set(client('SHOW DATABASES;').splitlines())
except BackupFailure:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
if database in existing:
fail('RESTORE', 'RESTORE_DATABASE_EXISTS')
stage = target.parent / ('.' + target.name + '.restore-' + uuid.uuid4().hex[:12])
created_database = False
stage_owned = False
completed = False
try:
try:
client('CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
created_database = True
except BackupFailure:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
stage.mkdir(mode=0o700, exist_ok=False)
stage_owned = True
app_verified = extract_verified_tar(app_archive, stage, app_files, private=True, precreated=True) == app_files
if not set(REQUIRED_APPLICATION_FILES).issubset(app_verified and app_files):
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
try:
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--database=' + database]
process = None
with gzip.open(sql, 'rb') as dump, tempfile.TemporaryFile() as errors:
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
try:
shutil.copyfileobj(dump, process.stdin)
process.stdin.close()
code = process.wait()
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
if code != 0:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
except BackupFailure:
raise
except Exception:
fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')
restored = counts(database)
if restored != manifest.get('tableCounts'):
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
if manifest.get('queryEvidence') is not None and query_evidence(database) != manifest['queryEvidence']:
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
archive_root = stage / 'runtimedata/archive'
archive_root.parent.mkdir(parents=True, exist_ok=True)
actual_archives = _safe_extract_tree(archive, archive_root, manifest['files'])
if actual_archives != manifest['files']:
fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH')
final_archive_root = target / 'runtimedata/archive'
references = verify_references(database, archive_root, manifest['archiveSourceRoot'], final_archive_root)
if not references['verified'] or not references['rebound']:
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
for line in rows.splitlines():
row = json.loads(line)
path = Path(row['path'])
if not path.is_relative_to(final_archive_root.resolve(strict=False)):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
staged_file = archive_root / path.relative_to(final_archive_root.resolve(strict=False))
if (staged_file.is_symlink() or not staged_file.is_file() or
staged_file.stat().st_size != int(row['size']) or file_sha256(staged_file) != row['sha256']):
fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID')
if query_evidence(database) != manifest.get('queryEvidence'):
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
config_path = stage / '.runtime/local-config.json'
try:
config = json.loads(config_path.read_text())
config['mysqlDatabase'] = database
config['databaseReady'] = True
config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2) + '\n')
config_path.chmod(0o600)
except Exception:
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
receipt = {'status': 'ISOLATED_DATA_FILE_RESTORE_VERIFIED',
'formatVersion': manifest['formatVersion'], 'restoredTables': len(restored),
'restoredApplicationFiles': len(app_files), 'restoredArchiveFiles': len(actual_archives),
'verification': {'databaseRestoreVerified': True, 'applicationFilesVerified': True,
'requiredApplicationInputsVerified': True, 'archiveFilesVerified': True,
'archiveReferencesVerified': True, 'applicationStartupVerified': False},
'migrationReady': False,
'note': 'Build, schema migration, and application startup require separate verification.'}
receipt_path = stage / 'restore-receipt.json'
receipt_path.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n')
receipt_path.chmod(0o600)
if target.exists() or target.is_symlink():
fail('RESTORE', 'RESTORE_TARGET_INVALID')
os.rename(stage, target)
stage_owned = False
completed = True
return receipt
finally:
if not completed:
database_cleanup_failed = False
stage_cleanup_failed = False
if created_database:
try:
client('DROP DATABASE `' + database + '`;')
except Exception:
database_cleanup_failed = True
if stage_owned and stage.exists() and not stage.is_symlink():
try:
shutil.rmtree(stage)
except Exception:
stage_cleanup_failed = True
if database_cleanup_failed and stage_cleanup_failed:
fail('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED')
if database_cleanup_failed:
fail('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED')
if stage_cleanup_failed:
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
def _main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
parser.add_argument('--directory', type=Path)
parser.add_argument('--target', type=Path)
parser.add_argument('--database')
args = parser.parse_args(argv)
previous_backups = backup_retention.backup_ids(ROOT)
try:
if args.action == 'verify':
if not args.directory:
parser.error('--directory is required for verify')
result = verify(args.directory)
elif args.action == 'restore':
if not args.directory or not args.target or not args.database:
parser.error('restore requires --directory, --target, and --database')
result = restore(args.directory, args.target, args.database)
else:
folder = create()
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
if args.action == 'create-and-verify':
result = verify(folder)
if args.action in ('verify', 'create-and-verify'):
selected = args.directory if args.action == 'verify' else folder
backup_retention.rotate(ROOT, selected)
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
separators=(',', ':')), flush=True)
return 0
except BackupFailure as error:
backup_retention.discard_failed_new(ROOT, previous_backups)
emit_failure(error)
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
except Exception:
backup_retention.discard_failed_new(ROOT, previous_backups)
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
emit_failure(error)
return 1
def main(argv=None):
try:
with backup_retention.lock(ROOT):
return _main(argv)
except backup_retention.Busy:
emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR'))
return 1
if __name__ == '__main__':
raise SystemExit(main())