新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
+84
-12
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify a local backup using an isolated database and archive tree."""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from datetime import datetime, timezone
|
||||
import gzip
|
||||
import hashlib
|
||||
@@ -10,11 +12,13 @@ from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
import project
|
||||
import backup_retention
|
||||
|
||||
ROOT = project.ROOT
|
||||
BACKUP_ROOT = ROOT / 'output/backups'
|
||||
@@ -55,6 +59,17 @@ SAFE_FAILURES = {
|
||||
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
|
||||
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
|
||||
}
|
||||
CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None)
|
||||
|
||||
|
||||
def client_config():
|
||||
return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf'
|
||||
|
||||
|
||||
def verification_temp_root():
|
||||
# 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。
|
||||
configuration = CLIENT_CONFIG_OVERRIDE.get()
|
||||
return configuration.parent if configuration is not None else None
|
||||
|
||||
|
||||
class BackupFailure(RuntimeError):
|
||||
@@ -101,14 +116,14 @@ def application_files():
|
||||
if missing:
|
||||
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
||||
paths = set()
|
||||
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist'):
|
||||
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'):
|
||||
folder = ROOT / relative
|
||||
for path in folder.rglob('*'):
|
||||
if path.is_symlink():
|
||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||
if path.is_file() and '__pycache__' not in path.parts:
|
||||
paths.add(path)
|
||||
for relative in (*REQUIRED_APPLICATION_FILES, '启动项目.command', '停止项目.command'):
|
||||
for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'):
|
||||
path = ROOT / relative
|
||||
if path.exists():
|
||||
if path.is_symlink() or not path.is_file():
|
||||
@@ -124,6 +139,10 @@ def application_files():
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||
paths.add(path)
|
||||
accepted = backup_retention.accepted_backend(ROOT)
|
||||
if accepted:
|
||||
paths.add(accepted[0])
|
||||
paths.update(backup_retention.native_files(ROOT))
|
||||
if not set(required_paths).issubset(paths):
|
||||
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
||||
return sorted(paths)
|
||||
@@ -139,6 +158,13 @@ def archive_application(folder):
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||
relative = path.relative_to(ROOT).as_posix()
|
||||
if path.name == 'nao.sqlite':
|
||||
with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary:
|
||||
copied = Path(temporary) / 'nao.sqlite'
|
||||
backup_retention.snapshot_sqlite(path, copied)
|
||||
files[relative] = file_sha256(copied)
|
||||
tar.add(copied, arcname=relative, recursive=False)
|
||||
continue
|
||||
before = file_sha256(path)
|
||||
tar.add(path, arcname=relative, recursive=False)
|
||||
if before != file_sha256(path):
|
||||
@@ -153,7 +179,7 @@ def archive_application(folder):
|
||||
|
||||
|
||||
def client(sql, database=None):
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()),
|
||||
'--batch', '--skip-column-names']
|
||||
if database:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
|
||||
@@ -181,7 +207,7 @@ def counts(database):
|
||||
def query_evidence(database):
|
||||
"""Reproduce release membership and monthly formal input components in SQL."""
|
||||
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
||||
if not current:
|
||||
if not current or current == 'NULL':
|
||||
return {'releaseId': None, 'months': []}
|
||||
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
||||
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
||||
@@ -332,25 +358,29 @@ def _snapshot_counts(sql):
|
||||
return snapshot_counts
|
||||
|
||||
|
||||
@backup_retention.serialized
|
||||
def create():
|
||||
# Validate the full, fixed application input set before creating any backup files.
|
||||
selected_application = application_files()
|
||||
config = database_config()
|
||||
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
|
||||
before_queries = query_evidence(config['mysqlDatabase'])
|
||||
before_counts = counts(config['mysqlDatabase'])
|
||||
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
||||
folder.mkdir(parents=True, exist_ok=False)
|
||||
folder.chmod(0o700)
|
||||
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
|
||||
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
|
||||
sql = folder / 'database.sql.gz'
|
||||
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
||||
'--skip-extended-insert', config['mysqlDatabase']]
|
||||
'--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']]
|
||||
process = None
|
||||
try:
|
||||
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
||||
try:
|
||||
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
|
||||
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
|
||||
shutil.copyfileobj(process.stdout, compressed)
|
||||
process.stdout.close()
|
||||
if process.wait() != 0:
|
||||
@@ -373,7 +403,11 @@ def create():
|
||||
sql.chmod(0o600)
|
||||
if query_evidence(config['mysqlDatabase']) != before_queries:
|
||||
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
||||
table_counts = _snapshot_counts(sql)
|
||||
# The dump is a single consistent transaction. Restore verification below
|
||||
# independently checks these counts, including a net-zero concurrent race.
|
||||
if counts(config['mysqlDatabase']) != before_counts:
|
||||
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
||||
table_counts = before_counts
|
||||
archive = folder / 'archive.tar.gz'
|
||||
source = ROOT / 'runtimedata/archive'
|
||||
files = {}
|
||||
@@ -413,6 +447,9 @@ def create():
|
||||
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
|
||||
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
||||
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
||||
accepted = backup_retention.accepted_backend(ROOT)
|
||||
if accepted:
|
||||
manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]}
|
||||
try:
|
||||
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
||||
except Exception:
|
||||
@@ -420,7 +457,7 @@ def create():
|
||||
return folder
|
||||
|
||||
|
||||
def verify(folder):
|
||||
def _verify(folder):
|
||||
started = time.monotonic()
|
||||
folder = Path(folder)
|
||||
if folder.is_symlink():
|
||||
@@ -451,7 +488,7 @@ def verify(folder):
|
||||
try:
|
||||
if file_sha256(app_archive) != manifest['applicationSha256']:
|
||||
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp:
|
||||
Path(temp).chmod(0o700)
|
||||
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
|
||||
application_verified = actual_app == expected_app
|
||||
@@ -465,14 +502,17 @@ def verify(folder):
|
||||
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
||||
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
||||
created_scratch = False
|
||||
scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip())
|
||||
try:
|
||||
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||
created_scratch = True
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
|
||||
backup_retention.scratch_created(scratch_registration)
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
|
||||
process = None
|
||||
try:
|
||||
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
||||
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
||||
backup_retention.scratch_child(scratch_registration, process.pid)
|
||||
try:
|
||||
shutil.copyfileobj(data, process.stdin)
|
||||
process.stdin.write(b'\nCOMMIT;\n')
|
||||
@@ -494,7 +534,7 @@ def verify(folder):
|
||||
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
||||
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
||||
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-') as temp:
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp:
|
||||
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
|
||||
relocated_root.parent.mkdir(parents=True, exist_ok=True)
|
||||
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
|
||||
@@ -534,10 +574,27 @@ def verify(folder):
|
||||
if created_scratch:
|
||||
try:
|
||||
client('DROP DATABASE `' + scratch + '`;')
|
||||
scratch_registration.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
|
||||
else:
|
||||
scratch_registration.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@backup_retention.serialized
|
||||
def verify(folder):
|
||||
if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1':
|
||||
import backup_verifier
|
||||
with backup_verifier.instance(ROOT) as configuration:
|
||||
token = CLIENT_CONFIG_OVERRIDE.set(configuration)
|
||||
try:
|
||||
return _verify(folder)
|
||||
finally:
|
||||
CLIENT_CONFIG_OVERRIDE.reset(token)
|
||||
return _verify(folder)
|
||||
|
||||
|
||||
@backup_retention.serialized
|
||||
def restore(folder, target, database):
|
||||
"""Restore a verified format-v3 backup to a new project directory and new database.
|
||||
|
||||
@@ -709,13 +766,14 @@ def restore(folder, target, database):
|
||||
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
def _main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
|
||||
parser.add_argument('--directory', type=Path)
|
||||
parser.add_argument('--target', type=Path)
|
||||
parser.add_argument('--database')
|
||||
args = parser.parse_args(argv)
|
||||
previous_backups = backup_retention.backup_ids(ROOT)
|
||||
try:
|
||||
if args.action == 'verify':
|
||||
if not args.directory:
|
||||
@@ -730,17 +788,31 @@ def main(argv=None):
|
||||
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
||||
if args.action == 'create-and-verify':
|
||||
result = verify(folder)
|
||||
if args.action in ('verify', 'create-and-verify'):
|
||||
selected = args.directory if args.action == 'verify' else folder
|
||||
backup_retention.rotate(ROOT, selected)
|
||||
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
|
||||
separators=(',', ':')), flush=True)
|
||||
return 0
|
||||
except BackupFailure as error:
|
||||
backup_retention.discard_failed_new(ROOT, previous_backups)
|
||||
emit_failure(error)
|
||||
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
|
||||
except Exception:
|
||||
backup_retention.discard_failed_new(ROOT, previous_backups)
|
||||
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
|
||||
emit_failure(error)
|
||||
return 1
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
with backup_retention.lock(ROOT):
|
||||
return _main(argv)
|
||||
except backup_retention.Busy:
|
||||
emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR'))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user