#!/usr/bin/env python3 """Create and verify a local backup using an isolated database and archive tree.""" import argparse from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone import gzip import hashlib import json import os from pathlib import Path, PurePosixPath import re import shutil import subprocess import sys import tarfile import tempfile import time import uuid import project import backup_retention ROOT = project.ROOT BACKUP_ROOT = ROOT / 'output/backups' REQUIRED_APPLICATION_DIRECTORIES = ('backend/src', 'web/src', 'web/public', 'scripts') REQUIRED_APPLICATION_FILES = ( 'runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml', 'web/index.html', 'web/tsconfig.json', 'web/vite.config.ts', 'web/pnpm-workspace.yaml', 'ai/nao-quality/RULES.md', 'ai/nao-quality/agent/prompts/system.md', 'requirements-nao.in', 'requirements-nao.lock', '.runtime/local-config.json', ) OPTIONAL_PRIVATE_APPLICATION_FILES = ('.runtime/nao-model-config.json', 'apikey.txt') SAFE_FAILURES = { ('INPUTS', 'APPLICATION_INPUT_MISSING'): '备份缺少必要的应用输入文件。', ('INPUTS', 'APPLICATION_INPUT_UNSAFE'): '备份应用输入包含无效链接或路径。', ('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED'): '数据库导出失败,请检查本机数据库运维状态。', ('DATABASE_EXPORT', 'DATABASE_CHANGED'): '备份期间正式数据发生变化,请重新执行备份。', ('DATABASE_EXPORT', 'DATABASE_READ_FAILED'): '数据库读取失败,请检查本机数据库运维状态。', ('ARCHIVE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。', ('ARCHIVE', 'ARCHIVE_PACK_FAILED'): '来源归档打包失败。', ('APPLICATION', 'APPLICATION_PACK_FAILED'): '应用文件打包或校验失败。', ('MANIFEST', 'MANIFEST_WRITE_FAILED'): '备份清单写入失败。', ('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'): '备份校验和不匹配,恢复已停止。', ('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID'): '应用归档不完整或包含不安全路径。', ('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'): '隔离数据库恢复失败,正式数据库未修改。', ('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH'): '隔离数据库内容与备份清单不一致。', ('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH'): '来源归档内容与备份清单不一致。', ('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'): '来源归档缺失或与数据库指纹不一致。', ('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN'): '旧备份缺少来源归档根目录,无法验证迁移引用。', ('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID'): '数据库中的来源归档引用无法安全重绑定。', ('RESTORE', 'RESTORE_TARGET_INVALID'): '恢复目标必须是新的独立目录。', ('RESTORE', 'RESTORE_DATABASE_INVALID'): '恢复数据库必须是本项目新建的专用恢复库。', ('RESTORE', 'RESTORE_DATABASE_EXISTS'): '恢复目标数据库已存在,恢复已停止。', ('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED'): '此备份缺少安全迁移恢复所需的来源根目录或应用输入。', ('RESTORE', 'RESTORE_FAILED'): '持久恢复未完成,已清理本次创建的隔离资源。', ('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'): '恢复失败后的专用数据库清理未完成。', ('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED'): '恢复失败后的私有暂存目录清理未完成。', ('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'): '恢复失败后的专用数据库与私有暂存目录清理均未完成。', ('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。', ('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。', } CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None) def client_config(): return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf' def verification_temp_root(): # 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。 configuration = CLIENT_CONFIG_OVERRIDE.get() return configuration.parent if configuration is not None else None class BackupFailure(RuntimeError): def __init__(self, stage, code): if (stage, code) not in SAFE_FAILURES: stage, code = 'PROTOCOL', 'BACKUP_INTERNAL_ERROR' self.stage, self.code = stage, code self.safe_message = SAFE_FAILURES[(stage, code)] super().__init__(self.safe_message) def fail(stage, code): raise BackupFailure(stage, code) def emit_failure(error): print(json.dumps({'event': 'failure', 'stage': error.stage, 'code': error.code, 'message': error.safe_message}, ensure_ascii=False, separators=(',', ':')), flush=True) def file_sha256(path): digest = hashlib.sha256() with Path(path).open('rb') as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk) return digest.hexdigest() def database_config(): value = project.config().copy() override = os.environ.get('QUALITY_BACKUP_DATABASE') if override: if not re.fullmatch(r'quality_(?:verify|scale|restore)_[A-Za-z0-9_]+', override): raise ValueError('Backup database override is restricted to this project test databases') value['mysqlDatabase'] = override return value def application_files(): """Return the fixed application snapshot, failing when any required input is absent.""" required_paths = [ROOT / relative for relative in REQUIRED_APPLICATION_FILES] missing = [relative for relative, path in zip(REQUIRED_APPLICATION_FILES, required_paths) if not path.is_file()] missing.extend(relative for relative in REQUIRED_APPLICATION_DIRECTORIES if not (ROOT / relative).is_dir()) if missing: fail('INPUTS', 'APPLICATION_INPUT_MISSING') paths = set() for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'): folder = ROOT / relative for path in folder.rglob('*'): if path.is_symlink(): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') if path.is_file() and '__pycache__' not in path.parts: paths.add(path) for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'): path = ROOT / relative if path.exists(): if path.is_symlink() or not path.is_file(): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') paths.add(path) for relative in OPTIONAL_PRIVATE_APPLICATION_FILES: path = ROOT / relative if path.exists() or path.is_symlink(): if path.is_symlink() or not path.is_file(): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') paths.add(path) for path in (ROOT / 'backend/target').glob('*.jar'): if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') paths.add(path) accepted = backup_retention.accepted_backend(ROOT) if accepted: paths.add(accepted[0]) paths.update(backup_retention.native_files(ROOT)) if not set(required_paths).issubset(paths): fail('INPUTS', 'APPLICATION_INPUT_MISSING') return sorted(paths) def archive_application(folder): archive = folder / 'application.tar.gz' files = {} try: selected = application_files() with tarfile.open(archive, 'w:gz') as tar: for path in selected: if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') relative = path.relative_to(ROOT).as_posix() if path.name == 'nao.sqlite': with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary: copied = Path(temporary) / 'nao.sqlite' backup_retention.snapshot_sqlite(path, copied) files[relative] = file_sha256(copied) tar.add(copied, arcname=relative, recursive=False) continue before = file_sha256(path) tar.add(path, arcname=relative, recursive=False) if before != file_sha256(path): fail('APPLICATION', 'APPLICATION_PACK_FAILED') files[relative] = before archive.chmod(0o600) return files, file_sha256(archive) except BackupFailure: raise except Exception: fail('APPLICATION', 'APPLICATION_PACK_FAILED') def client(sql, database=None): args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--batch', '--skip-column-names'] if database: if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database): raise ValueError('Invalid database identifier') args.append('--database=' + database) try: result = subprocess.run(args, input=sql, text=True, capture_output=True, check=False) except Exception: fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED') if result.returncode: fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED') return result.stdout def counts(database): tables = client('SHOW TABLES;', database).splitlines() result = {} for table in tables: if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', table): fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip()) return result def query_evidence(database): """Reproduce release membership and monthly formal input components in SQL.""" current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip() if not current or current == 'NULL': return {'releaseId': None, 'months': []} if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current): fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED') result = {'releaseId': current, 'months': []} keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap', 'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap', 'bright_processing_scrap', 'bright_diffusion_scrap'] for stage in ('black', 'bright'): pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \ "'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)" for key in keys: pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))" rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database) result['months'].extend(json.loads(line) for line in rows.splitlines()) result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()] return result def _safe_archive_member(name): if not isinstance(name, str) or not name or '\\' in name or name.startswith('/') or re.match(r'^[A-Za-z]:', name): return None pure = PurePosixPath(name) if pure.is_absolute() or any(part in ('', '.', '..') for part in pure.parts) or pure.as_posix() != name: return None return pure def extract_verified_tar(archive, destination, expected, private=False, precreated=False): """Extract only regular, unique, manifest-listed files and verify every SHA256.""" destination = Path(destination) if precreated: if (destination.is_symlink() or not destination.is_dir() or any(destination.iterdir())): fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') else: destination.mkdir(parents=True, exist_ok=False) root = destination.resolve() destination.chmod(0o700) actual = {} try: with tarfile.open(archive, 'r:gz') as tar: for member in tar: pure = _safe_archive_member(member.name) if pure is None or not member.isfile() or member.name in actual or member.name not in expected: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') target = destination.joinpath(*pure.parts) target.parent.mkdir(parents=True, exist_ok=True) if not target.resolve().is_relative_to(root) or target.exists() or target.is_symlink(): fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') source = tar.extractfile(member) if source is None: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') digest = hashlib.sha256() with source, target.open('xb') as output: for chunk in iter(lambda: source.read(1024 * 1024), b''): output.write(chunk) digest.update(chunk) actual[member.name] = digest.hexdigest() if private: parent = target.parent while parent != destination: parent.chmod(0o700) parent = parent.parent if private or member.name in ('.runtime/local-config.json', '.runtime/nao-model-config.json', 'apikey.txt'): target.chmod(0o600) else: target.chmod(member.mode & 0o755) if member.name.startswith('.runtime/'): target.parent.chmod(0o700) if actual != expected: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') return actual except BackupFailure: raise except Exception: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') def _sql_text(value): return "CONVERT(0x" + value.encode('utf-8').hex() + " USING utf8mb4)" def verify_references(database, archive_root, source_root=None, rebound_root=None): try: rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database) except Exception: fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') if source_root is None: return {'verified': False, 'rebound': False, 'count': 0} source_root = Path(source_root) if not source_root.is_absolute(): fail('ARCHIVE_RESTORE', 'ARCHIVE_ROOT_UNKNOWN') source_root = source_root.resolve(strict=False) archive_root = Path(archive_root).resolve() rebound_root = Path(rebound_root).resolve() if rebound_root is not None else None references = [] for line in rows.splitlines(): row = json.loads(line) stored = Path(row['path']) if not stored.is_absolute() or '..' in stored.parts or not stored.is_relative_to(source_root): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') relative = stored.relative_to(source_root) if not relative.parts or any(part in ('', '.', '..') for part in relative.parts): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') file = archive_root.joinpath(*relative.parts) if file.is_symlink() or not file.is_file() or not file.resolve().is_relative_to(archive_root) or file.stat().st_size != int(row['size']) or file_sha256(file) != row['sha256']: fail('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID') rebound = rebound_root.joinpath(*relative.parts) if rebound_root is not None else None if rebound is not None and not rebound.resolve(strict=False).is_relative_to(rebound_root): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') references.append((row['path'], rebound, relative.as_posix())) if rebound_root is not None: try: for old, rebound, _ in references: client('UPDATE source_revision SET archive_path=' + _sql_text(str(rebound)) + ' WHERE archive_path=' + _sql_text(old) + ';', database) rebound_rows = client("SELECT JSON_OBJECT('path',archive_path) FROM source_revision;", database) except Exception: fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') actual_paths = {json.loads(line)['path'] for line in rebound_rows.splitlines()} expected_paths = {str(rebound) for _, rebound, _ in references} if actual_paths != expected_paths: fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') return {'verified': True, 'rebound': rebound_root is not None, 'count': len(references)} def _safe_extract_tree(tar_path, directory, expected): try: return extract_verified_tar(tar_path, directory, expected, private=True) except BackupFailure as error: if error.stage == 'APPLICATION_RESTORE': fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH') raise def _snapshot_counts(sql): snapshot_counts = {} try: with gzip.open(sql, 'rt') as dump: for line in dump: table = re.match(r'^CREATE TABLE `([^`]+)`', line) if table: snapshot_counts[table.group(1)] = 0 row = re.match(r'^INSERT INTO `([^`]+)`', line) if row: snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1 except Exception: fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED') return snapshot_counts @backup_retention.serialized def create(): # Validate the full, fixed application input set before creating any backup files. selected_application = application_files() config = database_config() verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive') before_queries = query_evidence(config['mysqlDatabase']) before_counts = counts(config['mysqlDatabase']) folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8]) folder.mkdir(parents=True, exist_ok=False) folder.chmod(0o700) backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()), 'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()}) sql = folder / 'database.sql.gz' command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks', '--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']] process = None try: with tempfile.TemporaryFile() as errors, sql.open('wb') as output: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors) try: with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed: shutil.copyfileobj(process.stdout, compressed) process.stdout.close() if process.wait() != 0: fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED') except BaseException: if process.poll() is None: process.kill() process.wait() raise except BackupFailure: raise except Exception: fail('DATABASE_EXPORT', 'DATABASE_DUMP_FAILED') finally: if process is not None and process.stdout is not None and not process.stdout.closed: process.stdout.close() if process is not None and process.poll() is None: process.kill() process.wait() sql.chmod(0o600) if query_evidence(config['mysqlDatabase']) != before_queries: fail('DATABASE_EXPORT', 'DATABASE_CHANGED') # The dump is a single consistent transaction. Restore verification below # independently checks these counts, including a net-zero concurrent race. if counts(config['mysqlDatabase']) != before_counts: fail('DATABASE_EXPORT', 'DATABASE_CHANGED') table_counts = before_counts archive = folder / 'archive.tar.gz' source = ROOT / 'runtimedata/archive' files = {} process = None try: with tarfile.open(archive, 'w:gz') as tar: if source.exists(): for file in sorted(source.rglob('*')): if file.is_symlink(): fail('ARCHIVE', 'ARCHIVE_PACK_FAILED') if file.is_file(): relative = file.relative_to(source).as_posix() before = file_sha256(file) tar.add(file, arcname=relative, recursive=False) if before != file_sha256(file): fail('ARCHIVE', 'ARCHIVE_PACK_FAILED') files[relative] = before archive.chmod(0o600) except BackupFailure: raise except Exception: fail('ARCHIVE', 'ARCHIVE_PACK_FAILED') app_files, app_hash = archive_application(folder) archive_source_root = source.resolve() manifest = {'formatVersion': 3, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'], 'databaseDumpSha256': file_sha256(sql), 'archiveSha256': file_sha256(archive), 'files': files, 'archiveSourceRoot': str(archive_source_root), 'applicationSha256': app_hash, 'applicationFiles': app_files, 'requiredApplicationFiles': list(REQUIRED_APPLICATION_FILES), 'privateConfigurationFiles': [name for name in OPTIONAL_PRIVATE_APPLICATION_FILES if name in app_files], 'applicationRuntimePrerequisites': ['匹配的JDK/Maven/MySQL运行时', 'Node.js与pnpm', 'requirements-nao.lock对应的Python/NAO运行时'], 'containsPrivateConfiguration': True, 'recoveryMode': 'BACKUP_NOT_YET_RESTORE_VERIFIED', 'verification': {'databaseRestoreVerified': False, 'applicationFilesVerified': False, 'requiredApplicationInputsVerified': False, 'archiveFilesVerified': False, 'archiveReferencesVerified': False, 'applicationStartupVerified': False}, 'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'}, 'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'} accepted = backup_retention.accepted_backend(ROOT) if accepted: manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]} try: project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') except Exception: fail('MANIFEST', 'MANIFEST_WRITE_FAILED') return folder def _verify(folder): started = time.monotonic() folder = Path(folder) if folder.is_symlink(): raise ValueError('Backup directory cannot be a symbolic link') folder = folder.resolve() if not folder.is_relative_to(BACKUP_ROOT.resolve()): raise ValueError('Only this project backup directory may be verified') for artifact in (folder / 'manifest.json', folder / 'database.sql.gz', folder / 'archive.tar.gz', folder / 'application.tar.gz'): if artifact.is_symlink(): fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') try: manifest = json.loads((folder / 'manifest.json').read_text()) except Exception: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') sql, archive = folder / 'database.sql.gz', folder / 'archive.tar.gz' try: if file_sha256(sql) != manifest['databaseDumpSha256'] or file_sha256(archive) != manifest['archiveSha256']: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') except BackupFailure: raise except Exception: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') app_archive = folder / 'application.tar.gz' expected_app = manifest.get('applicationFiles') application_verified = False required_inputs_verified = False if manifest.get('applicationSha256') and isinstance(expected_app, dict): try: if file_sha256(app_archive) != manifest['applicationSha256']: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp: Path(temp).chmod(0o700) actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app) application_verified = actual_app == expected_app required_inputs_verified = (set(REQUIRED_APPLICATION_FILES).issubset(expected_app) and set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', []))) except BackupFailure as error: if error.stage == 'APPLICATION_RESTORE': fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') raise except Exception: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8] created_scratch = False scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip()) try: client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') created_scratch = True backup_retention.scratch_created(scratch_registration) args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0'] process = None try: with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors: process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors) backup_retention.scratch_child(scratch_registration, process.pid) try: shutil.copyfileobj(data, process.stdin) process.stdin.write(b'\nCOMMIT;\n') process.stdin.close() code = process.wait() except BaseException: if process.poll() is None: process.kill() process.wait() raise if code != 0: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') except BackupFailure: raise except Exception: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') restored = counts(scratch) if restored != manifest.get('tableCounts'): fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']: fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp: relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive' relocated_root.parent.mkdir(parents=True, exist_ok=True) actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {})) archive_files_verified = actual_archives == manifest.get('files', {}) source_root = manifest.get('archiveSourceRoot') references = verify_references(scratch, relocated_root, source_root, relocated_root if source_root else None) if references['verified']: rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", scratch) for line in rows.splitlines(): row = json.loads(line) path = Path(row['path']) if not path.is_relative_to(relocated_root.resolve()) or not path.is_file() or file_sha256(path) != row['sha256'] or path.stat().st_size != int(row['size']): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') manifest['status'] = 'ISOLATED_RESTORE_VERIFIED' manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat() manifest['restoredTables'] = len(restored) manifest['restoredApplicationFiles'] = len(expected_app or {}) manifest['restoredArchiveFiles'] = len(manifest.get('files', {})) manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3) manifest['recoveryMode'] = 'ISOLATED_DATA_AND_FILE_CHECKS_NO_APPLICATION_STARTUP' manifest['verification'] = {'databaseRestoreVerified': True, 'applicationFilesVerified': application_verified, 'requiredApplicationInputsVerified': required_inputs_verified, 'archiveFilesVerified': archive_files_verified, 'archiveReferencesVerified': references['verified'] and references['rebound'], 'applicationStartupVerified': False} manifest['archiveReferencesRebound'] = bool(references['verified'] and references['rebound']) manifest['migrationReady'] = False project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') return {'status': manifest['status'], 'restoredTables': len(restored), 'restoredApplicationFiles': len(expected_app or {}), 'restoredArchiveFiles': len(manifest.get('files', {})), 'verification': manifest['verification'], 'archiveReferencesRebound': manifest['archiveReferencesRebound'], 'migrationReady': manifest['migrationReady'], 'elapsedSeconds': manifest['restoreElapsedSeconds']} finally: if created_scratch: try: client('DROP DATABASE `' + scratch + '`;') scratch_registration.unlink(missing_ok=True) except Exception: fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED') else: scratch_registration.unlink(missing_ok=True) @backup_retention.serialized def verify(folder): if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1': import backup_verifier with backup_verifier.instance(ROOT) as configuration: token = CLIENT_CONFIG_OVERRIDE.set(configuration) try: return _verify(folder) finally: CLIENT_CONFIG_OVERRIDE.reset(token) return _verify(folder) @backup_retention.serialized def restore(folder, target, database): """Restore a verified format-v3 backup to a new project directory and new database. The destination is assembled beside its final path and atomically renamed only after the SQL, application files, and relocated source references pass their checks. Existing destinations or databases are never overwritten or removed. """ folder = Path(folder) target = Path(target).expanduser() if folder.is_symlink(): fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED') folder = folder.resolve() if not folder.is_relative_to(BACKUP_ROOT.resolve()): fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED') if (not target.is_absolute() or target.exists() or target.is_symlink() or not target.parent.is_dir() or target.name in ('', '.', '..')): fail('RESTORE', 'RESTORE_TARGET_INVALID') target = target.parent.resolve() / target.name if target == ROOT.resolve() or ROOT.resolve().is_relative_to(target): fail('RESTORE', 'RESTORE_TARGET_INVALID') if target.is_relative_to(BACKUP_ROOT.resolve()) or folder.is_relative_to(target): fail('RESTORE', 'RESTORE_TARGET_INVALID') if not re.fullmatch(r'quality_restore_[A-Za-z0-9_]{1,48}', database): fail('RESTORE', 'RESTORE_DATABASE_INVALID') try: manifest = json.loads((folder / 'manifest.json').read_text()) sql, archive, app_archive = (folder / 'database.sql.gz', folder / 'archive.tar.gz', folder / 'application.tar.gz') if (manifest.get('formatVersion') != 3 or not Path(manifest.get('archiveSourceRoot', '')).is_absolute() or not isinstance(manifest.get('applicationFiles'), dict) or not set(REQUIRED_APPLICATION_FILES).issubset(manifest['applicationFiles']) or not set(REQUIRED_APPLICATION_FILES).issubset(manifest.get('requiredApplicationFiles', []))): fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED') if any(path.is_symlink() for path in (sql, archive, app_archive)): fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') if (file_sha256(sql) != manifest['databaseDumpSha256'] or file_sha256(archive) != manifest['archiveSha256'] or file_sha256(app_archive) != manifest['applicationSha256']): fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') app_files = manifest['applicationFiles'] if (not isinstance(manifest.get('files'), dict) or not isinstance(manifest.get('tableCounts'), dict)): fail('RESTORE', 'RESTORE_FORMAT_UNSUPPORTED') except BackupFailure: raise except Exception: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') try: existing = set(client('SHOW DATABASES;').splitlines()) except BackupFailure: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') if database in existing: fail('RESTORE', 'RESTORE_DATABASE_EXISTS') stage = target.parent / ('.' + target.name + '.restore-' + uuid.uuid4().hex[:12]) created_database = False stage_owned = False completed = False try: try: client('CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') created_database = True except BackupFailure: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') stage.mkdir(mode=0o700, exist_ok=False) stage_owned = True app_verified = extract_verified_tar(app_archive, stage, app_files, private=True, precreated=True) == app_files if not set(REQUIRED_APPLICATION_FILES).issubset(app_verified and app_files): fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') try: args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + database] process = None with gzip.open(sql, 'rb') as dump, tempfile.TemporaryFile() as errors: process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors) try: shutil.copyfileobj(dump, process.stdin) process.stdin.close() code = process.wait() except BaseException: if process.poll() is None: process.kill() process.wait() raise if code != 0: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') except BackupFailure: raise except Exception: fail('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED') restored = counts(database) if restored != manifest.get('tableCounts'): fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') if manifest.get('queryEvidence') is not None and query_evidence(database) != manifest['queryEvidence']: fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') archive_root = stage / 'runtimedata/archive' archive_root.parent.mkdir(parents=True, exist_ok=True) actual_archives = _safe_extract_tree(archive, archive_root, manifest['files']) if actual_archives != manifest['files']: fail('ARCHIVE_RESTORE', 'ARCHIVE_CONTENT_MISMATCH') final_archive_root = target / 'runtimedata/archive' references = verify_references(database, archive_root, manifest['archiveSourceRoot'], final_archive_root) if not references['verified'] or not references['rebound']: fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database) for line in rows.splitlines(): row = json.loads(line) path = Path(row['path']) if not path.is_relative_to(final_archive_root.resolve(strict=False)): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') staged_file = archive_root / path.relative_to(final_archive_root.resolve(strict=False)) if (staged_file.is_symlink() or not staged_file.is_file() or staged_file.stat().st_size != int(row['size']) or file_sha256(staged_file) != row['sha256']): fail('ARCHIVE_REBIND', 'ARCHIVE_REFERENCE_INVALID') if query_evidence(database) != manifest.get('queryEvidence'): fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') config_path = stage / '.runtime/local-config.json' try: config = json.loads(config_path.read_text()) config['mysqlDatabase'] = database config['databaseReady'] = True config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2) + '\n') config_path.chmod(0o600) except Exception: fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') receipt = {'status': 'ISOLATED_DATA_FILE_RESTORE_VERIFIED', 'formatVersion': manifest['formatVersion'], 'restoredTables': len(restored), 'restoredApplicationFiles': len(app_files), 'restoredArchiveFiles': len(actual_archives), 'verification': {'databaseRestoreVerified': True, 'applicationFilesVerified': True, 'requiredApplicationInputsVerified': True, 'archiveFilesVerified': True, 'archiveReferencesVerified': True, 'applicationStartupVerified': False}, 'migrationReady': False, 'note': 'Build, schema migration, and application startup require separate verification.'} receipt_path = stage / 'restore-receipt.json' receipt_path.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n') receipt_path.chmod(0o600) if target.exists() or target.is_symlink(): fail('RESTORE', 'RESTORE_TARGET_INVALID') os.rename(stage, target) stage_owned = False completed = True return receipt finally: if not completed: database_cleanup_failed = False stage_cleanup_failed = False if created_database: try: client('DROP DATABASE `' + database + '`;') except Exception: database_cleanup_failed = True if stage_owned and stage.exists() and not stage.is_symlink(): try: shutil.rmtree(stage) except Exception: stage_cleanup_failed = True if database_cleanup_failed and stage_cleanup_failed: fail('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED') if database_cleanup_failed: fail('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED') if stage_cleanup_failed: fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED') def _main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore']) parser.add_argument('--directory', type=Path) parser.add_argument('--target', type=Path) parser.add_argument('--database') args = parser.parse_args(argv) previous_backups = backup_retention.backup_ids(ROOT) try: if args.action == 'verify': if not args.directory: parser.error('--directory is required for verify') result = verify(args.directory) elif args.action == 'restore': if not args.directory or not args.target or not args.database: parser.error('restore requires --directory, --target, and --database') result = restore(args.directory, args.target, args.database) else: folder = create() result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'} if args.action == 'create-and-verify': result = verify(folder) if args.action in ('verify', 'create-and-verify'): selected = args.directory if args.action == 'verify' else folder backup_retention.rotate(ROOT, selected) print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False, separators=(',', ':')), flush=True) return 0 except BackupFailure as error: backup_retention.discard_failed_new(ROOT, previous_backups) emit_failure(error) return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code)) except Exception: backup_retention.discard_failed_new(ROOT, previous_backups) error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR') emit_failure(error) return 1 def main(argv=None): try: with backup_retention.lock(ROOT): return _main(argv) except backup_retention.Busy: emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')) return 1 if __name__ == '__main__': raise SystemExit(main())