初始化:质量数据分析平台源码

This commit is contained in:
2026-10-22 15:58:57 +08:00
commit 1e8a655adc
280 changed files with 43922 additions and 0 deletions
+251
View File
@@ -0,0 +1,251 @@
#!/usr/bin/env python3
"""Create a local backup and verify restoration into a new, isolated database."""
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import tarfile
import tempfile
import time
import uuid
import project
ROOT = project.ROOT
def database_config():
value = project.config().copy()
override = os.environ.get('QUALITY_BACKUP_DATABASE')
if override:
if not re.fullmatch(r'quality_(?:verify|scale)_[A-Za-z0-9_]+', override):
raise ValueError('Backup database override is restricted to this project test databases')
value['mysqlDatabase'] = override
return value
def application_files():
"""Fixed project inputs only; archives stay private and are never served as public assets."""
paths = set()
for relative in ('backend/src', 'web/src', 'web/public', 'web/dist', 'scripts'):
folder = ROOT / relative
if folder.exists():
for path in folder.rglob('*'):
if path.is_file() and '__pycache__' not in path.parts:
paths.add(path)
for relative in ('runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
'.runtime/local-config.json', 'apikey.txt', '启动项目.command', '停止项目.command'):
path = ROOT / relative
if path.is_file(): paths.add(path)
paths.update((ROOT / 'backend/target').glob('*.jar'))
return sorted(paths)
def archive_application(folder):
archive = folder / 'application.tar.gz'
files = {}
with tarfile.open(archive, 'w:gz') as tar:
for path in application_files():
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
raise RuntimeError('Application backup contains a link outside the project')
relative = path.relative_to(ROOT).as_posix()
before = hashlib.sha256(path.read_bytes()).hexdigest()
tar.add(path, arcname=relative, recursive=False)
if before != hashlib.sha256(path.read_bytes()).hexdigest():
raise RuntimeError('Application files changed during backup; retry after the build completes')
files[relative] = before
archive.chmod(0o600)
return files, hashlib.sha256(archive.read_bytes()).hexdigest()
def client(sql, database=None):
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--batch', '--skip-column-names']
if database:
if not database.replace('_', '').isalnum(): raise ValueError('Invalid database identifier')
args.append('--database=' + database)
result = subprocess.run(args, input=sql, text=True, capture_output=True)
if result.returncode:
raise RuntimeError('Local MySQL backup/restore command failed; credentials were not printed.')
return result.stdout
def counts(database):
tables = client('SHOW TABLES;', database).splitlines()
result = {}
for table in tables:
if not table.replace('_', '').isalnum(): raise ValueError('Unexpected table identifier')
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
return result
def query_evidence(database):
"""Reproduce release membership and monthly formal input components in SQL."""
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
if not current:
return {'releaseId': None, 'months': []}
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
raise RuntimeError('Unexpected release identity in backup')
result = {'releaseId': current, 'months': []}
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
'bright_processing_scrap', 'bright_diffusion_scrap']
for stage in ('black', 'bright'):
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
for key in keys:
pairs += ",'" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
result['months'].extend(json.loads(line) for line in rows.splitlines())
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
return result
def verify_references(database, archive_root):
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
source_root = (ROOT / 'runtimedata/archive').resolve()
for line in rows.splitlines():
row = json.loads(line)
stored = Path(row['path']).resolve()
if not stored.is_relative_to(source_root):
raise RuntimeError('A source revision references a file outside the project archive')
file = archive_root / stored.relative_to(source_root)
if not file.is_file() or file.stat().st_size != row['size'] or hashlib.sha256(file.read_bytes()).hexdigest() != row['sha256']:
raise RuntimeError('A referenced source archive is missing or does not match its database revision')
def create():
config = database_config()
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive')
before_queries = query_evidence(config['mysqlDatabase'])
folder = ROOT / 'output/backups' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
'--skip-extended-insert', config['mysqlDatabase']]
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
shutil.copyfileobj(process.stdout, compressed)
process.stdout.close()
if process.wait() != 0:
raise RuntimeError('Database dump failed; no backup was marked complete.')
sql.chmod(0o600)
if query_evidence(config['mysqlDatabase']) != before_queries:
raise RuntimeError('The selected release changed during backup; retry a consistent snapshot')
# Count the consistent dump itself; live audit/task tables may change during a backup.
snapshot_counts = {}
with gzip.open(sql, 'rt') as dump:
for line in dump:
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
if table: snapshot_counts[table.group(1)] = 0
row = re.match(r'^INSERT INTO `([^`]+)`', line)
if row: snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
archive = folder / 'archive.tar.gz'
files = {}
source = ROOT / 'runtimedata/archive'
with tarfile.open(archive, 'w:gz') as tar:
if source.exists():
for file in sorted(source.rglob('*')):
if file.is_symlink(): raise RuntimeError('Archive contains a symbolic link; inspect before backing up.')
if file.is_file():
relative = file.relative_to(source).as_posix()
files[relative] = hashlib.sha256(file.read_bytes()).hexdigest()
tar.add(file, arcname=relative, recursive=False)
archive.chmod(0o600)
app_files, app_hash = archive_application(folder)
manifest = {'formatVersion': 2, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
'databaseDumpSha256': hashlib.sha256(sql.read_bytes()).hexdigest(),
'archiveSha256': hashlib.sha256(archive.read_bytes()).hexdigest(), 'files': files,
'applicationSha256': app_hash, 'applicationFiles': app_files,
'containsPrivateConfiguration': True,
'recoveryMode': 'VERIFIED_FULL_SNAPSHOT',
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
'tableCounts': snapshot_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
print('Backup created: ' + str(folder))
return folder
def verify(folder):
started = time.monotonic()
folder = folder.resolve()
if not folder.is_relative_to((ROOT / 'output/backups').resolve()):
raise ValueError('Only this project backup directory may be verified')
manifest = json.loads((folder / 'manifest.json').read_text())
sql = folder / 'database.sql.gz'
archive = folder / 'archive.tar.gz'
if hashlib.sha256(sql.read_bytes()).hexdigest() != manifest['databaseDumpSha256'] or hashlib.sha256(archive.read_bytes()).hexdigest() != manifest['archiveSha256']:
raise RuntimeError('Backup checksum mismatch; restoration stopped')
if manifest.get('applicationSha256'):
app_archive = folder / 'application.tar.gz'
if hashlib.sha256(app_archive.read_bytes()).hexdigest() != manifest['applicationSha256']:
raise RuntimeError('Application backup checksum mismatch; restoration stopped')
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
Path(temp).chmod(0o700)
with tarfile.open(app_archive) as tar:
tar.extractall(temp, filter='data')
actual_app = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in Path(temp).rglob('*') if p.is_file()}
if actual_app != manifest['applicationFiles']:
raise RuntimeError('Restored application and configuration checksums do not match')
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
try:
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch]
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
try:
shutil.copyfileobj(data, process.stdin)
finally:
process.stdin.close()
code = process.wait()
if code != 0: raise RuntimeError('Isolated restore failed; no live database was changed')
restored = counts(scratch)
if restored != manifest['tableCounts']:
raise RuntimeError('Restored table counts do not match the backup manifest')
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
raise RuntimeError('Restored release query components or source provenance do not match')
with tempfile.TemporaryDirectory(prefix='quality-restore-files-') as temp:
with tarfile.open(archive) as tar:
tar.extractall(temp, filter='data')
actual = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in Path(temp).rglob('*') if p.is_file()}
if actual != manifest['files']: raise RuntimeError('Restored archive checksums do not match')
verify_references(scratch, Path(temp))
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
manifest['restoredTables'] = len(restored)
manifest['restoredApplicationFiles'] = len(manifest.get('applicationFiles', {}))
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
print('Isolated restore verified: ' + str(len(restored)) + ' tables and ' + str(len(actual)) + ' archived files; live database unchanged.')
finally:
# Only the new scratch database created above is eligible for cleanup.
if scratch.startswith('quality_restore_'):
client('DROP DATABASE `' + scratch + '`;')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify'])
parser.add_argument('--directory', type=Path)
args = parser.parse_args()
if args.action == 'verify':
if not args.directory: parser.error('--directory is required for verify')
verify(args.directory)
else:
folder = create()
if args.action == 'create-and-verify': verify(folder)
if __name__ == '__main__':
main()