252 lines
14 KiB
Python
252 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Create a local backup and verify restoration into a new, isolated database."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
import project
|
|
|
|
ROOT = project.ROOT
|
|
|
|
|
|
def database_config():
|
|
value = project.config().copy()
|
|
override = os.environ.get('QUALITY_BACKUP_DATABASE')
|
|
if override:
|
|
if not re.fullmatch(r'quality_(?:verify|scale)_[A-Za-z0-9_]+', override):
|
|
raise ValueError('Backup database override is restricted to this project test databases')
|
|
value['mysqlDatabase'] = override
|
|
return value
|
|
|
|
|
|
def application_files():
|
|
"""Fixed project inputs only; archives stay private and are never served as public assets."""
|
|
paths = set()
|
|
for relative in ('backend/src', 'web/src', 'web/public', 'web/dist', 'scripts'):
|
|
folder = ROOT / relative
|
|
if folder.exists():
|
|
for path in folder.rglob('*'):
|
|
if path.is_file() and '__pycache__' not in path.parts:
|
|
paths.add(path)
|
|
for relative in ('runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
|
|
'.runtime/local-config.json', 'apikey.txt', '启动项目.command', '停止项目.command'):
|
|
path = ROOT / relative
|
|
if path.is_file(): paths.add(path)
|
|
paths.update((ROOT / 'backend/target').glob('*.jar'))
|
|
return sorted(paths)
|
|
|
|
|
|
def archive_application(folder):
|
|
archive = folder / 'application.tar.gz'
|
|
files = {}
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
for path in application_files():
|
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
|
raise RuntimeError('Application backup contains a link outside the project')
|
|
relative = path.relative_to(ROOT).as_posix()
|
|
before = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
tar.add(path, arcname=relative, recursive=False)
|
|
if before != hashlib.sha256(path.read_bytes()).hexdigest():
|
|
raise RuntimeError('Application files changed during backup; retry after the build completes')
|
|
files[relative] = before
|
|
archive.chmod(0o600)
|
|
return files, hashlib.sha256(archive.read_bytes()).hexdigest()
|
|
|
|
|
|
def client(sql, database=None):
|
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
|
'--batch', '--skip-column-names']
|
|
if database:
|
|
if not database.replace('_', '').isalnum(): raise ValueError('Invalid database identifier')
|
|
args.append('--database=' + database)
|
|
result = subprocess.run(args, input=sql, text=True, capture_output=True)
|
|
if result.returncode:
|
|
raise RuntimeError('Local MySQL backup/restore command failed; credentials were not printed.')
|
|
return result.stdout
|
|
|
|
|
|
def counts(database):
|
|
tables = client('SHOW TABLES;', database).splitlines()
|
|
result = {}
|
|
for table in tables:
|
|
if not table.replace('_', '').isalnum(): raise ValueError('Unexpected table identifier')
|
|
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
|
|
return result
|
|
|
|
|
|
def query_evidence(database):
|
|
"""Reproduce release membership and monthly formal input components in SQL."""
|
|
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
|
if not current:
|
|
return {'releaseId': None, 'months': []}
|
|
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
|
raise RuntimeError('Unexpected release identity in backup')
|
|
result = {'releaseId': current, 'months': []}
|
|
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
|
|
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
|
|
'bright_processing_scrap', 'bright_diffusion_scrap']
|
|
for stage in ('black', 'bright'):
|
|
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
|
|
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
|
|
for key in keys:
|
|
pairs += ",'" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
|
|
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
|
|
result['months'].extend(json.loads(line) for line in rows.splitlines())
|
|
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
|
|
return result
|
|
|
|
|
|
def verify_references(database, archive_root):
|
|
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
|
|
source_root = (ROOT / 'runtimedata/archive').resolve()
|
|
for line in rows.splitlines():
|
|
row = json.loads(line)
|
|
stored = Path(row['path']).resolve()
|
|
if not stored.is_relative_to(source_root):
|
|
raise RuntimeError('A source revision references a file outside the project archive')
|
|
file = archive_root / stored.relative_to(source_root)
|
|
if not file.is_file() or file.stat().st_size != row['size'] or hashlib.sha256(file.read_bytes()).hexdigest() != row['sha256']:
|
|
raise RuntimeError('A referenced source archive is missing or does not match its database revision')
|
|
|
|
|
|
def create():
|
|
config = database_config()
|
|
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive')
|
|
before_queries = query_evidence(config['mysqlDatabase'])
|
|
folder = ROOT / 'output/backups' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
|
folder.mkdir(parents=True, exist_ok=False)
|
|
folder.chmod(0o700)
|
|
sql = folder / 'database.sql.gz'
|
|
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
|
'--single-transaction', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
|
'--skip-extended-insert', config['mysqlDatabase']]
|
|
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
|
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
|
|
shutil.copyfileobj(process.stdout, compressed)
|
|
process.stdout.close()
|
|
if process.wait() != 0:
|
|
raise RuntimeError('Database dump failed; no backup was marked complete.')
|
|
sql.chmod(0o600)
|
|
if query_evidence(config['mysqlDatabase']) != before_queries:
|
|
raise RuntimeError('The selected release changed during backup; retry a consistent snapshot')
|
|
# Count the consistent dump itself; live audit/task tables may change during a backup.
|
|
snapshot_counts = {}
|
|
with gzip.open(sql, 'rt') as dump:
|
|
for line in dump:
|
|
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
|
|
if table: snapshot_counts[table.group(1)] = 0
|
|
row = re.match(r'^INSERT INTO `([^`]+)`', line)
|
|
if row: snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
|
|
archive = folder / 'archive.tar.gz'
|
|
files = {}
|
|
source = ROOT / 'runtimedata/archive'
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
if source.exists():
|
|
for file in sorted(source.rglob('*')):
|
|
if file.is_symlink(): raise RuntimeError('Archive contains a symbolic link; inspect before backing up.')
|
|
if file.is_file():
|
|
relative = file.relative_to(source).as_posix()
|
|
files[relative] = hashlib.sha256(file.read_bytes()).hexdigest()
|
|
tar.add(file, arcname=relative, recursive=False)
|
|
archive.chmod(0o600)
|
|
app_files, app_hash = archive_application(folder)
|
|
manifest = {'formatVersion': 2, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
|
|
'databaseDumpSha256': hashlib.sha256(sql.read_bytes()).hexdigest(),
|
|
'archiveSha256': hashlib.sha256(archive.read_bytes()).hexdigest(), 'files': files,
|
|
'applicationSha256': app_hash, 'applicationFiles': app_files,
|
|
'containsPrivateConfiguration': True,
|
|
'recoveryMode': 'VERIFIED_FULL_SNAPSHOT',
|
|
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
|
'tableCounts': snapshot_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
|
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
|
print('Backup created: ' + str(folder))
|
|
return folder
|
|
|
|
|
|
def verify(folder):
|
|
started = time.monotonic()
|
|
folder = folder.resolve()
|
|
if not folder.is_relative_to((ROOT / 'output/backups').resolve()):
|
|
raise ValueError('Only this project backup directory may be verified')
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
sql = folder / 'database.sql.gz'
|
|
archive = folder / 'archive.tar.gz'
|
|
if hashlib.sha256(sql.read_bytes()).hexdigest() != manifest['databaseDumpSha256'] or hashlib.sha256(archive.read_bytes()).hexdigest() != manifest['archiveSha256']:
|
|
raise RuntimeError('Backup checksum mismatch; restoration stopped')
|
|
if manifest.get('applicationSha256'):
|
|
app_archive = folder / 'application.tar.gz'
|
|
if hashlib.sha256(app_archive.read_bytes()).hexdigest() != manifest['applicationSha256']:
|
|
raise RuntimeError('Application backup checksum mismatch; restoration stopped')
|
|
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
|
|
Path(temp).chmod(0o700)
|
|
with tarfile.open(app_archive) as tar:
|
|
tar.extractall(temp, filter='data')
|
|
actual_app = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in Path(temp).rglob('*') if p.is_file()}
|
|
if actual_app != manifest['applicationFiles']:
|
|
raise RuntimeError('Restored application and configuration checksums do not match')
|
|
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
|
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
try:
|
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch]
|
|
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
|
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
|
try:
|
|
shutil.copyfileobj(data, process.stdin)
|
|
finally:
|
|
process.stdin.close()
|
|
code = process.wait()
|
|
if code != 0: raise RuntimeError('Isolated restore failed; no live database was changed')
|
|
restored = counts(scratch)
|
|
if restored != manifest['tableCounts']:
|
|
raise RuntimeError('Restored table counts do not match the backup manifest')
|
|
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
|
raise RuntimeError('Restored release query components or source provenance do not match')
|
|
with tempfile.TemporaryDirectory(prefix='quality-restore-files-') as temp:
|
|
with tarfile.open(archive) as tar:
|
|
tar.extractall(temp, filter='data')
|
|
actual = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in Path(temp).rglob('*') if p.is_file()}
|
|
if actual != manifest['files']: raise RuntimeError('Restored archive checksums do not match')
|
|
verify_references(scratch, Path(temp))
|
|
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
|
|
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
|
|
manifest['restoredTables'] = len(restored)
|
|
manifest['restoredApplicationFiles'] = len(manifest.get('applicationFiles', {}))
|
|
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
|
|
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
|
|
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
|
print('Isolated restore verified: ' + str(len(restored)) + ' tables and ' + str(len(actual)) + ' archived files; live database unchanged.')
|
|
finally:
|
|
# Only the new scratch database created above is eligible for cleanup.
|
|
if scratch.startswith('quality_restore_'):
|
|
client('DROP DATABASE `' + scratch + '`;')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify'])
|
|
parser.add_argument('--directory', type=Path)
|
|
args = parser.parse_args()
|
|
if args.action == 'verify':
|
|
if not args.directory: parser.error('--directory is required for verify')
|
|
verify(args.directory)
|
|
else:
|
|
folder = create()
|
|
if args.action == 'create-and-verify': verify(folder)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|