新增:Linux Docker 部署与自动回收、备份及容量监测

This commit is contained in:
2026-11-10 14:53:07 +08:00
parent 3962521a5c
commit dc5b9b76ec
45 changed files with 5591 additions and 35 deletions
+84 -12
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
"""Create and verify a local backup using an isolated database and archive tree."""
import argparse
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime, timezone
import gzip
import hashlib
@@ -10,11 +12,13 @@ from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import uuid
import project
import backup_retention
ROOT = project.ROOT
BACKUP_ROOT = ROOT / 'output/backups'
@@ -55,6 +59,17 @@ SAFE_FAILURES = {
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
}
CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None)
def client_config():
return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf'
def verification_temp_root():
# 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。
configuration = CLIENT_CONFIG_OVERRIDE.get()
return configuration.parent if configuration is not None else None
class BackupFailure(RuntimeError):
@@ -101,14 +116,14 @@ def application_files():
if missing:
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
paths = set()
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist'):
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'):
folder = ROOT / relative
for path in folder.rglob('*'):
if path.is_symlink():
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
if path.is_file() and '__pycache__' not in path.parts:
paths.add(path)
for relative in (*REQUIRED_APPLICATION_FILES, '启动项目.command', '停止项目.command'):
for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'):
path = ROOT / relative
if path.exists():
if path.is_symlink() or not path.is_file():
@@ -124,6 +139,10 @@ def application_files():
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
paths.add(path)
accepted = backup_retention.accepted_backend(ROOT)
if accepted:
paths.add(accepted[0])
paths.update(backup_retention.native_files(ROOT))
if not set(required_paths).issubset(paths):
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
return sorted(paths)
@@ -139,6 +158,13 @@ def archive_application(folder):
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
relative = path.relative_to(ROOT).as_posix()
if path.name == 'nao.sqlite':
with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary:
copied = Path(temporary) / 'nao.sqlite'
backup_retention.snapshot_sqlite(path, copied)
files[relative] = file_sha256(copied)
tar.add(copied, arcname=relative, recursive=False)
continue
before = file_sha256(path)
tar.add(path, arcname=relative, recursive=False)
if before != file_sha256(path):
@@ -153,7 +179,7 @@ def archive_application(folder):
def client(sql, database=None):
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()),
'--batch', '--skip-column-names']
if database:
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
@@ -181,7 +207,7 @@ def counts(database):
def query_evidence(database):
"""Reproduce release membership and monthly formal input components in SQL."""
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
if not current:
if not current or current == 'NULL':
return {'releaseId': None, 'months': []}
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
@@ -332,25 +358,29 @@ def _snapshot_counts(sql):
return snapshot_counts
@backup_retention.serialized
def create():
# Validate the full, fixed application input set before creating any backup files.
selected_application = application_files()
config = database_config()
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
before_queries = query_evidence(config['mysqlDatabase'])
before_counts = counts(config['mysqlDatabase'])
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
'--skip-extended-insert', config['mysqlDatabase']]
'--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']]
process = None
try:
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
try:
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
shutil.copyfileobj(process.stdout, compressed)
process.stdout.close()
if process.wait() != 0:
@@ -373,7 +403,11 @@ def create():
sql.chmod(0o600)
if query_evidence(config['mysqlDatabase']) != before_queries:
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
table_counts = _snapshot_counts(sql)
# The dump is a single consistent transaction. Restore verification below
# independently checks these counts, including a net-zero concurrent race.
if counts(config['mysqlDatabase']) != before_counts:
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
table_counts = before_counts
archive = folder / 'archive.tar.gz'
source = ROOT / 'runtimedata/archive'
files = {}
@@ -413,6 +447,9 @@ def create():
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
accepted = backup_retention.accepted_backend(ROOT)
if accepted:
manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]}
try:
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
except Exception:
@@ -420,7 +457,7 @@ def create():
return folder
def verify(folder):
def _verify(folder):
started = time.monotonic()
folder = Path(folder)
if folder.is_symlink():
@@ -451,7 +488,7 @@ def verify(folder):
try:
if file_sha256(app_archive) != manifest['applicationSha256']:
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp:
Path(temp).chmod(0o700)
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
application_verified = actual_app == expected_app
@@ -465,14 +502,17 @@ def verify(folder):
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
created_scratch = False
scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip())
try:
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
created_scratch = True
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
backup_retention.scratch_created(scratch_registration)
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
process = None
try:
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
backup_retention.scratch_child(scratch_registration, process.pid)
try:
shutil.copyfileobj(data, process.stdin)
process.stdin.write(b'\nCOMMIT;\n')
@@ -494,7 +534,7 @@ def verify(folder):
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-') as temp:
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp:
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
relocated_root.parent.mkdir(parents=True, exist_ok=True)
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
@@ -534,10 +574,27 @@ def verify(folder):
if created_scratch:
try:
client('DROP DATABASE `' + scratch + '`;')
scratch_registration.unlink(missing_ok=True)
except Exception:
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
else:
scratch_registration.unlink(missing_ok=True)
@backup_retention.serialized
def verify(folder):
if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1':
import backup_verifier
with backup_verifier.instance(ROOT) as configuration:
token = CLIENT_CONFIG_OVERRIDE.set(configuration)
try:
return _verify(folder)
finally:
CLIENT_CONFIG_OVERRIDE.reset(token)
return _verify(folder)
@backup_retention.serialized
def restore(folder, target, database):
"""Restore a verified format-v3 backup to a new project directory and new database.
@@ -709,13 +766,14 @@ def restore(folder, target, database):
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
def main(argv=None):
def _main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
parser.add_argument('--directory', type=Path)
parser.add_argument('--target', type=Path)
parser.add_argument('--database')
args = parser.parse_args(argv)
previous_backups = backup_retention.backup_ids(ROOT)
try:
if args.action == 'verify':
if not args.directory:
@@ -730,17 +788,31 @@ def main(argv=None):
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
if args.action == 'create-and-verify':
result = verify(folder)
if args.action in ('verify', 'create-and-verify'):
selected = args.directory if args.action == 'verify' else folder
backup_retention.rotate(ROOT, selected)
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
separators=(',', ':')), flush=True)
return 0
except BackupFailure as error:
backup_retention.discard_failed_new(ROOT, previous_backups)
emit_failure(error)
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
except Exception:
backup_retention.discard_failed_new(ROOT, previous_backups)
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
emit_failure(error)
return 1
def main(argv=None):
try:
with backup_retention.lock(ROOT):
return _main(argv)
except backup_retention.Busy:
emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR'))
return 1
if __name__ == '__main__':
raise SystemExit(main())
+381
View File
@@ -0,0 +1,381 @@
"""Shared backup/maintenance exclusion and verified-only backup rotation."""
from contextlib import contextmanager
from datetime import datetime, timezone
from functools import wraps
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import threading
import time
import uuid
_thread_lock = threading.RLock()
_held = {}
ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$')
CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified',
'archiveFilesVerified', 'archiveReferencesVerified')
ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256',
'application.tar.gz': 'applicationSha256'}
class Busy(RuntimeError):
pass
def sha(path):
digest = hashlib.sha256()
with Path(path).open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''):
digest.update(block)
return digest.hexdigest()
def atomic(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
with tmp.open('w') as output:
os.chmod(tmp, 0o600)
json.dump(value, output, ensure_ascii=False, indent=2)
output.flush()
os.fsync(output.fileno())
tmp.replace(path)
if os.name != 'nt':
fd = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
@contextmanager
def lock(root, timeout=1800):
"""One lock for UI backups, restores, scheduled backups and garbage collection."""
root = Path(root).resolve()
key = str(root)
with _thread_lock:
if key in _held:
yield
return
path = root / '.runtime/data-maintenance.lock'
path.parent.mkdir(parents=True, exist_ok=True)
if path.parent.is_symlink() or path.is_symlink():
raise ValueError('Unsafe maintenance lock path')
stream = path.open('a+b')
os.chmod(path, 0o600)
deadline = time.monotonic() + timeout
try:
while True:
try:
if os.name == 'nt':
import msvcrt
stream.seek(0)
if not path.stat().st_size:
stream.write(b'0'); stream.flush()
stream.seek(0)
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except OSError:
if time.monotonic() >= deadline:
raise Busy('Another backup or maintenance operation is active')
time.sleep(.2)
_held[key] = stream
try:
yield
finally:
_held.pop(key, None)
if os.name == 'nt':
import msvcrt
stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(stream, fcntl.LOCK_UN)
finally:
stream.close()
def serialized(function):
@wraps(function)
def wrapped(*args, **kwargs):
with lock(function.__globals__['ROOT']):
return function(*args, **kwargs)
return wrapped
def verified(folder, checksums=True):
folder = Path(folder)
if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
return None
try:
if (folder / 'manifest.json').is_symlink():
return None
manifest = json.loads((folder / 'manifest.json').read_text())
if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED':
return None
if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS):
return None
for name, field in ARTIFACTS.items():
p = folder / name
if p.is_symlink() or not p.is_file() or not manifest.get(field):
return None
if checksums and sha(p) != manifest[field]:
return None
return manifest
except (OSError, ValueError, TypeError):
return None
def _rotate(root, selected):
"""Publish the new verified backup first. A failed candidate never deletes old backups."""
root = Path(root).resolve()
selected = Path(selected)
if selected.is_symlink():
raise ValueError('Backup directory cannot be a symbolic link')
selected = selected.resolve()
base = root / 'output/backups'
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):
raise ValueError('Unsafe managed backup root')
if selected.parent.resolve() != base.resolve():
raise ValueError('Backup is outside the managed backup directory')
manifest = verified(selected)
if manifest is None:
raise ValueError('Backup has not passed restore and artifact verification')
previous = latest(root)
if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''):
selected, manifest = previous
for name in (*ARTIFACTS, 'manifest.json'):
with (selected / name).open('r+b') as stream:
stream.flush(); os.fsync(stream.fileno())
if os.name!='nt':
for directory in (selected,base,base.parent):
fd=os.open(directory,os.O_RDONLY)
try:os.fsync(fd)
finally:os.close(fd)
pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'),
'releaseId': manifest.get('queryEvidence', {}).get('releaseId'),
'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()}
atomic(root / '.runtime/retention/latest.json', pointer)
removed = []
for folder in base.iterdir():
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
continue
# A restore-verified older folder is safe to retire under the shared lock.
# Incomplete candidates are kept for 24h and never treated as good backups.
older = verified(folder, checksums=False)
try:
owner = json.loads((folder / '.owner.json').read_text())
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
except (OSError, ValueError):
owned = False
stale = owned and time.time() - folder.stat().st_mtime > 86400
if older is not None or stale:
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents):
continue
try:
retire_folder(root,folder,'backup')
removed.append(folder.name)
except OSError:
pass # Latest stays usable; a later maintenance pass retries retirement.
# A one-off snapshot from the previous manual cleanup is retired only when
# explicitly registered and its complete original file set still matches.
registration = root / '.runtime/retention/legacy.json'
if registration.is_file():
legacy_info = json.loads(registration.read_text())
if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'):
legacy = root / legacy_info['path']
if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']:
old = json.loads((legacy/'manifest.json').read_text())
expected = old.get('filesSha256', {})
actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()}
if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()):
evidence = root/'.runtime/retention/setup-evidence'
evidence.mkdir(parents=True,exist_ok=True,mode=0o700)
for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'):
if (legacy/name).is_file():
shutil.copy2(legacy/name,evidence/name)
retire_folder(root,legacy,'legacy')
legacy_info['retired']=True
atomic(registration,legacy_info)
clean_retired(root)
return pointer, removed
def retire_folder(root, folder, kind):
"""Atomic rename makes a crash during removal safely resumable."""
root=Path(root).resolve();folder=Path(folder).resolve()
if kind=='backup':
if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source')
elif kind=='legacy':
if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source')
else:raise ValueError('Unknown retirement kind')
area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700)
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):
raise ValueError('Unsafe backup retirement directory')
token=uuid.uuid4().hex
# Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。
storage = root/'output/backups/.retired' if kind=='backup' else area
if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage')
storage.mkdir(parents=True,exist_ok=True,mode=0o700)
atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token,
'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'})
folder.rename(storage/token)
if os.name!='nt':
fd=os.open(storage,os.O_RDONLY)
try:os.fsync(fd)
finally:os.close(fd)
def clean_retired(root):
root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0
for record in area.glob('*.json'):
try:
if record.is_symlink():continue
value=json.loads(record.read_text());token=value.get('target','')
if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue
storage=value.get('storage','state-volume')
if storage not in ('state-volume','backup-volume'):continue
base=root/'output/backups/.retired' if storage=='backup-volume' else area
if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue
folder=base/token
if folder.is_symlink():continue
if folder.exists():
if any(p.is_symlink() for p in folder.rglob('*')):continue
shutil.rmtree(folder);count+=1
record.unlink()
except (OSError,ValueError):continue
return count
def rotate(root, selected):
with lock(root):
return _rotate(root, selected)
def latest(root, checksums=True):
root = Path(root).resolve()
try:
base=root/'output/backups'
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None
pointer = json.loads((root / '.runtime/retention/latest.json').read_text())
if not ID.fullmatch(pointer['id']):
return None
folder = root / 'output/backups' / pointer['id']
if sha(folder / 'manifest.json') != pointer['manifestSha256']:
return None
manifest = verified(folder, checksums=checksums)
return (folder, manifest) if manifest else None
except (OSError, ValueError, KeyError, TypeError):
return None
def backup_ids(root):
base=Path(root)/'output/backups'
return {p.name for p in base.iterdir()} if base.exists() else set()
def discard_failed_new(root, before):
"""Delete only this call's registered, incomplete candidate after it has stopped."""
root=Path(root).resolve();base=root/'output/backups'
for folder in base.iterdir() if base.exists() else []:
if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue
try:
owner=json.loads((folder/'.owner.json').read_text())
if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue
if verified(folder,checksums=False) is not None:continue
if any(p.is_symlink() for p in folder.rglob('*')):continue
shutil.rmtree(folder)
except (OSError,ValueError):continue
def accepted_backend(root):
"""The running pinned jar is part of a valid application backup on this Mac."""
root = Path(root).resolve()
# 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。
if os.environ.get('QUALITY_CONTAINER') == '1':
path = root / 'app/backend.jar'
digest = (root / 'app/backend.sha256').read_text().strip()
if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest:
raise ValueError('Container backend digest mismatch')
return path, digest
config = Path.home() / 'Library/Application Support/QualityCenterService/service.json'
if not config.exists():
return None
value = json.loads(config.read_text())
if Path(value['root']).resolve() != root:
return None
digest = value['jarSha256']
if not re.fullmatch(r'[a-f0-9]{64}', digest):
raise ValueError('Invalid accepted backend digest')
path = root / '.runtime/backend-builds' / (digest + '.jar')
if path.is_symlink() or sha(path) != digest:
raise ValueError('Accepted backend digest mismatch')
return path, digest
def native_files(root):
"""Retain restorable native settings/conversations; exclude live access tokens."""
for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'):
path=Path(root)/relative
if path.is_symlink():raise ValueError('Unsafe operations configuration link')
if path.is_file():yield path
base=Path(root)/'.runtime/nao-users'
ignored={'logs','backups','__pycache__'}
ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'}
for path in base.rglob('*') if base.exists() else []:
if path.is_symlink():
raise ValueError('Unsafe native workspace link')
if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts):
continue
if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')):
continue
yield path
def snapshot_sqlite(source, target):
original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True)
copied=sqlite3.connect(target)
try:
original.backup(copied)
copied.execute('PRAGMA foreign_keys=OFF')
copied.execute('PRAGMA secure_delete=ON')
tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")}
for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token',
'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'):
if name in tables:
copied.execute('DELETE FROM "'+name+'"')
copied.commit();copied.execute('VACUUM')
if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
raise ValueError('Native SQLite backup integrity failed')
finally:
original.close();copied.close()
def scratch_register(root, database, server):
import psutil
if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database):
raise ValueError('Unexpected verification database name')
path=Path(root)/'.runtime/retention/scratch'/(database+'.json')
value={'kind':'quality-restore-check','database':database,'server':server,
'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False}
atomic(path,value)
return path
def scratch_created(path):
value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value)
def scratch_child(path, pid):
import psutil
value=json.loads(Path(path).read_text())
value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()})
atomic(path,value)
+112
View File
@@ -0,0 +1,112 @@
"""Private, socket-only MySQL instance for low-priority restore verification."""
from contextlib import contextmanager
import json
import os
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import uuid
import psutil
import backup_retention as br
def stop_owned(value):
if value.get('initPid'):
try:
init=psutil.Process(value['initPid'])
if abs(init.create_time()-value['initCreated'])<.01 and init.is_running() and init.status()!=psutil.STATUS_ZOMBIE:return False
except psutil.NoSuchProcess:pass
except psutil.Error:return False
pid=value.get('serverPid')
if not pid:
p=Path(value['directory'])/'server.pid'
if not p.exists():return True
pid=int(p.read_text())
try:
process=psutil.Process(pid)
if value.get('serverCreated') and abs(process.create_time()-value['serverCreated'])>.01:return True
command=process.cmdline()
if process.uids().real!=os.getuid() or Path(process.exe()).resolve()!=Path(value['binary']).resolve() or '--datadir='+value['directory']+'/data' not in command or '--skip-networking' not in command:
return False
process.terminate()
try:process.wait(timeout=180)
except psutil.TimeoutExpired:return False
return True
except psutil.NoSuchProcess:return True
except psutil.Error:return False
def remove_owned(root, directory, value):
directory=Path(directory);base=Path(root)/'.runtime/retention'
if directory.parent!=base or not directory.name.startswith('verify-') or directory.is_symlink():
return False
if not stop_owned(value):return False
socket=Path(value['socketDirectory'])
if socket.parent.resolve()!=Path('/tmp').resolve() or not socket.name.startswith('qret-') or socket.is_symlink():return False
shutil.rmtree(socket,ignore_errors=True)
shutil.rmtree(directory)
return True
def clean_abandoned(root):
base=Path(root)/'.runtime/retention';removed=0
for directory in base.glob('verify-*'):
path=directory/'owner.json'
if directory.is_symlink() or not path.is_file() or path.is_symlink():continue
value=json.loads(path.read_text())
if value.get('kind')!='private-quality-restore' or value.get('directory')!=str(directory):continue
try:
p=psutil.Process(value['ownerPid'])
if abs(p.create_time()-value['ownerCreated'])<.01 and p.is_running() and p.status()!=psutil.STATUS_ZOMBIE:continue
except psutil.NoSuchProcess:pass
except psutil.Error:continue
if time.time()-value['createdAt']<300:continue
if remove_owned(root,directory,value):removed+=1
return removed
@contextmanager
def instance(root):
root=Path(root).resolve();base=root/'.runtime/retention';base.mkdir(parents=True,exist_ok=True,mode=0o700)
directory=base/('verify-'+uuid.uuid4().hex);directory.mkdir(mode=0o700)
socket=Path(tempfile.mkdtemp(prefix='qret-',dir='/tmp')).resolve();socket.chmod(0o700)
binary=root/'.runtime/mysql/bin/mysqld'
value={'kind':'private-quality-restore','directory':str(directory),'socketDirectory':str(socket),
'binary':str(binary),'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time()}
br.atomic(directory/'owner.json',value)
process=None
try:
with (directory/'initialize.log').open('ab') as output:
init=subprocess.Popen([str(binary),'--no-defaults','--initialize-insecure','--basedir='+str(binary.parent.parent),
'--datadir='+str(directory/'data'),'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--skip-log-bin'],stdout=output,stderr=output)
value.update({'initPid':init.pid,'initCreated':psutil.Process(init.pid).create_time()});br.atomic(directory/'owner.json',value)
try:code=init.wait(timeout=180)
except subprocess.TimeoutExpired:
init.kill();init.wait();raise
if code:raise RuntimeError('Private restore server initialization failed')
args=[str(binary),'--no-defaults','--basedir='+str(binary.parent.parent),'--datadir='+str(directory/'data'),
'--socket='+str(socket/'mysql.sock'),'--pid-file='+str(directory/'server.pid'),
'--log-error='+str(directory/'server.log'),'--skip-networking','--mysqlx=OFF','--skip-log-bin',
'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--innodb-buffer-pool-size=128M','--innodb-redo-log-capacity=64M','--max-connections=8']
with (directory/'stdout.log').open('ab') as output:
process=subprocess.Popen(args,stdout=output,stderr=output,stdin=subprocess.DEVNULL)
value.update({'serverPid':process.pid,'serverCreated':psutil.Process(process.pid).create_time()});br.atomic(directory/'owner.json',value)
config=directory/'client.cnf'
config.write_text('[client]\nuser=root\nsocket='+str(socket/'mysql.sock')+'\n');config.chmod(0o600)
deadline=time.monotonic()+90
while time.monotonic()<deadline:
if process.poll() is not None:raise RuntimeError('Private restore server exited')
ready=subprocess.run([str(binary.parent/'mysql'),'--defaults-file='+str(config),'--batch','--skip-column-names','-e','SELECT 1'],capture_output=True)
if ready.returncode==0:break
time.sleep(.2)
else:raise RuntimeError('Private restore server not ready')
yield config
finally:
if not remove_owned(root,directory,value):
raise RuntimeError('Private restore server cleanup requires inspection')
if process is not None:process.wait(timeout=5)
+158
View File
@@ -0,0 +1,158 @@
"""Conservative retention of application logs, with native MySQL reopening."""
import gzip
import json
import os
from pathlib import Path
import re
import time
import uuid
import psutil
MAX_BYTES = 100 * 1024**2
MAX_AGE = 7 * 86400
def candidates(root, support):
root, support = Path(root).resolve(), Path(support).resolve()
paths = set()
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
if not base.is_dir() or base.is_symlink():
continue
for directory, dirs, files in os.walk(base, followlinks=False):
parent = Path(directory)
if len(parent.relative_to(base).parts) >= depth - 1:
dirs[:] = []
else:
dirs[:] = [d for d in dirs if not (parent / d).is_symlink() and d not in
('backups', 'mysql-data', 'mysql', 'jdk', 'm2', 'backend-builds', 'node_modules')]
for name in files:
if re.fullmatch(r'.+\.(?:log|out|err)(?:\.[a-zA-Z0-9_-]+)*(?:\.gz)?', name):
p = parent / name
if not p.is_symlink() and p.is_file() and p.stat().st_nlink == 1:
paths.add(p)
return sorted(paths)
def opened_files():
"""Unknown access is a reason to defer deletion, never a reason to assume closed."""
paths = set()
for process in psutil.process_iter(['uids']):
try:
if process.info['uids'].real != os.getuid():
continue
for f in process.open_files():
paths.add(str(Path(f.path).absolute()))
except psutil.NoSuchProcess:
continue
except psutil.Error as error:
# macOS restricts system/GUI agents unrelated to these application log
# directories. A denied project runtime/interpreter still defers cleanup.
try:
executable = Path(process.exe()).resolve()
interpreter = re.fullmatch(r'(?:python.*|Python|java|node|bash|zsh|sh|uv|mvn)', executable.name)
if executable.is_relative_to(Path.cwd().resolve()) or interpreter:
raise RuntimeError('Project writer identity check unavailable') from error
except psutil.NoSuchProcess:
continue
except psutil.Error:
raise RuntimeError('Writer identity check unavailable') from error
return paths
def fingerprint(path):
s = path.stat()
return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns
def category(path):
name = path.name.lstrip('.')
name = re.sub(r'\.\d{4}-\d{2}-\d{2}.*', '', name)
name = re.sub(r'\.roll-.*', '', name)
name = re.sub(r'\.(?:log|out|err)(?:\..*)?$', '', name)
return name
def rotate_mysql(root, execute, br, now=None):
root = Path(root).resolve(); now = time.time() if now is None else now
source = root / 'output/logs/mysql-server.log'
journal = root / '.runtime/operations/mysql-log-rotation.json'
if any(p.is_symlink() for p in (root / 'output', source.parent, source, root / '.runtime', journal.parent, journal)):
raise ValueError('Unsafe MySQL log path')
pending = None
if journal.exists():
pending = json.loads(journal.read_text())
if (pending.get('root') != str(root) or pending.get('kind') != 'quality-mysql-log-rotation' or
not re.fullmatch(r'mysql-server\.roll-\d+-[a-f0-9]{8}\.log', pending.get('archive', ''))):
raise ValueError('Unknown MySQL log rotation intent')
elif source.is_file() and source.stat().st_size:
old_day = time.localtime(source.stat().st_mtime)[:3]
if source.stat().st_size < 10 * 1024**2 and old_day == time.localtime(now)[:3]:
return 0
pending = {'kind': 'quality-mysql-log-rotation', 'root': str(root),
'archive': 'mysql-server.roll-' + str(int(now)) + '-' + uuid.uuid4().hex[:8] + '.log',
'fingerprint': list(fingerprint(source))}
br.atomic(journal, pending)
if pending is None:
return 0
target = source.with_name(pending['archive'])
if target.is_symlink():
raise ValueError('Unsafe MySQL rotated log')
if not target.exists():
if not source.is_file() or list(fingerprint(source)) != pending['fingerprint']:
journal.unlink(); return 0
source.rename(target)
# MySQL closes the old descriptor and reopens its configured path. No truncation.
execute('FLUSH ERROR LOGS;', None)
if not source.is_file() or source.is_symlink() or source.stat().st_ino == target.stat().st_ino:
raise RuntimeError('MySQL error log reopening not confirmed')
source.chmod(0o600)
target.chmod(0o600)
journal.unlink()
return 1
def retain(root, support, now=None, open_paths=None, max_bytes=MAX_BYTES, max_age=MAX_AGE):
now = time.time() if now is None else now
opened = opened_files() if open_paths is None else set(open_paths)
groups = {}; result = {'logsRemoved': 0, 'logBytesRemoved': 0, 'logsDeferred': 0}
for p in candidates(root, support):
groups.setdefault(category(p), []).append(p)
for files in groups.values():
files.sort(key=lambda p: p.stat().st_mtime)
total = sum(p.stat().st_size for p in files)
for p in files:
s = p.stat(); stamp = fingerprint(p)
if now - s.st_mtime <= max_age and total <= max_bytes:
continue
# Native appenders own these live files. The monitor handles archives;
# MySQL is reopened by its dedicated rotation function above.
if ((p.parent == Path(root) / 'output/logs' and p.name in ('backend.log', 'mysql-server.log', 'backend-console.log')) or
(p.parent == Path(support) / 'logs' and p.name in ('guardian.log', 'retention.log', 'operations.log'))):
result['logsDeferred'] += 1; continue
if str(p.absolute()) in opened:
result['logsDeferred'] += 1; continue
# Recheck descriptors immediately before mutation. Data/config/archives
# never enter this allowlist, and active writers are never truncated.
if p.is_symlink() or fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
result['logsDeferred'] += 1; continue
if s.st_size > max_bytes and now - s.st_mtime <= max_age and len(files) == 1:
# Keep the newest bounded tail of a closed oversized console log.
temp = p.with_name(p.name + '.' + uuid.uuid4().hex + '.tmp')
try:
with p.open('rb') as source, temp.open('xb') as output:
temp.chmod(0o600); source.seek(max(0, s.st_size - max_bytes))
source.readline() # Drop a possible partial leading line.
while block := source.read(1024 * 1024): output.write(block)
output.flush(); os.fsync(output.fileno())
if fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
result['logsDeferred'] += 1; continue
os.utime(temp, ns=(s.st_atime_ns, s.st_mtime_ns)); temp.replace(p)
removed = s.st_size - p.stat().st_size
total -= removed; result['logBytesRemoved'] += removed
finally:
temp.unlink(missing_ok=True)
else:
p.unlink(); total -= s.st_size
result['logsRemoved'] += 1; result['logBytesRemoved'] += s.st_size
return result
+207
View File
@@ -0,0 +1,207 @@
"""Ownership records for disposable test schemas; never infer ownership from a prefix."""
from pathlib import Path
import json
import os
import re
import time
import uuid
import psutil
import backup_retention as br
PREFIXES = ('quality_verify_', 'quality_scale_', 'quality_retention_test_',
'quality_classification_source_', 'quality_classification_preview_',
'quality_sales_type_test_', 'quality_unclassified_delta_', 'quality_sales_metric_test_',
'quality_source_column_test_', 'quality_protocol_test_', 'quality_fine_rollup_')
MARKER = '__quality_test_owner'
def valid_name(name):
return (isinstance(name, str) and len(name) <= 64 and
re.fullmatch(r'[a-z0-9_]+_[a-f0-9]{8,32}', name) is not None and
name.startswith(PREFIXES))
def registry(root):
root = Path(root).resolve(); base = root / '.runtime/test-resources'
for path in (root / '.runtime', base):
if path.is_symlink() or not path.resolve().is_relative_to(root):
raise ValueError('Unsafe test registry path')
return base
def protect(root, name):
cfg = json.loads((Path(root) / '.runtime/local-config.json').read_text())
if not valid_name(name) or name == cfg['mysqlDatabase']:
raise ValueError('Not a disposable test database')
def active_process(pid, created):
try:
p = psutil.Process(int(pid))
return (abs(p.create_time() - float(created)) < .02 and p.is_running() and
p.status() != psutil.STATUS_ZOMBIE)
except psutil.NoSuchProcess:
return False
except (psutil.Error, TypeError, ValueError):
return True
def load(root, name):
path = registry(root) / (name + '.json')
if not path.is_file() or path.is_symlink():
return None
try:
value = json.loads(path.read_text())
if (not isinstance(value, dict) or value.get('kind') != 'quality-owned-test-db' or
value.get('root') != str(Path(root).resolve()) or value.get('database') != name or
not valid_name(name) or not re.fullmatch(r'[a-f0-9]{32}', value.get('token', ''))):
return None
return value
except (OSError, ValueError, TypeError):
return None
def create(root, name, execute, ddl=None, empty=False):
root = Path(root).resolve(); protect(root, name)
base = registry(root); base.mkdir(parents=True, exist_ok=True, mode=0o700)
path = base / (name + '.json')
# Exclusive registration also prevents a second process claiming an existing name.
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd)
value = {'kind': 'quality-owned-test-db', 'formatVersion': 1, 'root': str(root),
'database': name, 'token': uuid.uuid4().hex, 'createdAt': time.time(),
'expiresAt': time.time() + 86400, 'ownerPid': os.getpid(),
'ownerCreated': psutil.Process().create_time(), 'children': [], 'users': [],
'serverUuid': execute('SELECT @@server_uuid;', None).strip(), 'created': False}
br.atomic(path, value)
try:
result = execute(ddl or 'CREATE DATABASE `' + name + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', None)
except BaseException:
path.unlink(missing_ok=True); raise
value['created'] = True; br.atomic(path, value)
directory = root / '.runtime/mysql-data' / name
if directory.is_dir() and not directory.is_symlink():
stat = directory.stat()
value['directoryIdentity'] = [stat.st_dev, stat.st_ino]
br.atomic(path, value)
if not empty:
seal(root, name, execute)
return result
def seal(root, name, execute):
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
execute('CREATE TABLE `' + name + '`.`' + MARKER + '` (token CHAR(32) PRIMARY KEY);'
'INSERT INTO `' + name + '`.`' + MARKER + "` VALUES ('" + value['token'] + "');", None)
value['ready'] = True; br.atomic(registry(root) / (name + '.json'), value)
def same_directory(root, value):
p = Path(root) / '.runtime/mysql-data' / value['database']
if p.is_symlink() or not p.is_dir(): return False
stat = p.stat()
return value.get('directoryIdentity') == [stat.st_dev, stat.st_ino]
def attach_child(root, name, pid):
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
value.setdefault('children', []).append({'pid': pid, 'created': psutil.Process(pid).create_time()})
br.atomic(registry(root) / (name + '.json'), value)
def attach_user(root, name, user):
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
raise ValueError('Unsafe test account')
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
value.setdefault('users', []).append(user)
br.atomic(registry(root) / (name + '.json'), value)
def marker_matches(value, execute):
if not value.get('ready') or execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
return False
name = value['database']
found = execute("SELECT COUNT(*) FROM information_schema.tables WHERE TABLE_SCHEMA='" + name +
"' AND TABLE_NAME='" + MARKER + "';", None).strip()
return found == '1' and execute('SELECT token FROM `' + name + '`.`' + MARKER + '`;', None).strip() == value['token']
def drop(root, name, execute):
protect(root, name); value = load(root, name)
if value is None:
raise ValueError('Registered test schema identity not confirmed')
same_server = execute('SELECT @@server_uuid;', None).strip() == value['serverUuid']
own_process = (value['ownerPid'] == os.getpid() and active_process(value['ownerPid'], value['ownerCreated']))
if not same_server or not (marker_matches(value, execute) or
(value.get('created') and not value.get('ready') and (own_process or same_directory(root, value)))):
raise ValueError('Registered test schema identity not confirmed')
result = execute('DROP DATABASE `' + name + '`;', None)
value['databaseDropped'] = True
path = registry(root) / (name + '.json')
if value.get('users'):
br.atomic(path, value)
else:
path.unlink()
return result
def user_dropped(root, user):
for path in registry(root).glob('*.json'):
value = load(root, path.stem)
if value is None or value['ownerPid'] != os.getpid() or user not in value.get('users', []):
continue
value['users'].remove(user)
if value.get('databaseDropped') and not value['users']:
path.unlink()
else:
br.atomic(path, value)
def cleanup(root, execute, now=None, limit=3):
"""Called while maintenance owns its shared lock and the business application is idle."""
now = time.time() if now is None else now
result = {'testDatabasesRemoved': 0, 'testAccountsRemoved': 0, 'testResourcesDeferred': 0}
for path in sorted(registry(root).glob('*.json')):
if result['testDatabasesRemoved'] >= limit:
break
value = load(root, path.stem)
if value is None or not value.get('created') or now < value.get('expiresAt', now + 1):
continue
protect(root, value['database'])
if (active_process(value.get('ownerPid'), value.get('ownerCreated')) or
any(active_process(p.get('pid'), p.get('created')) for p in value.get('children', []))):
result['testResourcesDeferred'] += 1; continue
if execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
result['testResourcesDeferred'] += 1; continue
name = value['database']
if execute("SELECT COUNT(*) FROM information_schema.processlist WHERE DB='" + name + "';", None).strip() != '0':
result['testResourcesDeferred'] += 1; continue
if not value.get('databaseDropped'):
if not marker_matches(value, execute) and not (not value.get('ready') and same_directory(root, value)):
result['testResourcesDeferred'] += 1; continue
drop(root, name, execute); result['testDatabasesRemoved'] += 1
value['databaseDropped'] = True
for user in list(value.get('users', [])):
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
continue
grantee = "'" + user + "'@'127.0.0.1'"
literal = "CONVERT(0x" + grantee.encode().hex() + " USING utf8mb4)"
unsafe = execute("SELECT (SELECT COUNT(*) FROM information_schema.USER_PRIVILEGES WHERE GRANTEE=" + literal +
" AND PRIVILEGE_TYPE<>'USAGE')+(SELECT COUNT(*) FROM information_schema.SCHEMA_PRIVILEGES WHERE GRANTEE=" + literal +
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.TABLE_PRIVILEGES WHERE GRANTEE=" + literal +
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.processlist WHERE USER='" + user + "');", None).strip()
if unsafe != '0':
result['testResourcesDeferred'] += 1; continue
execute("DROP USER IF EXISTS '" + user + "'@'127.0.0.1';", None)
value['users'].remove(user); result['testAccountsRemoved'] += 1
if not value.get('users'):
path.unlink(missing_ok=True)
else:
br.atomic(path, value)
return result
+2 -1
View File
@@ -145,7 +145,8 @@ def start_mysql():
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M'])
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M',
*(['--skip-log-bin'] if value.get('binaryLogEnabled', True) is False else [])])
client_config = RUNTIME / 'mysql-root.cnf'
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
for _ in range(120):
+213
View File
@@ -0,0 +1,213 @@
"""Reclaim only unreferenced, completed source revisions and their owned archives."""
import hashlib
import json
from contextlib import contextmanager
from pathlib import Path
import re
import time
import uuid
# These are measured business values, not documents that refer to source IDs.
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
class Cleaner:
def __init__(self, db, br, safe_file, fingerprint, defer):
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
self.root = db.root.resolve()
self.boundary = self.root / 'runtimedata/archive/source-revisions'
self.journals = self.root / '.runtime/retention/source-files'
self.reference_deadline = None
@contextmanager
def quick_checks(self):
self.reference_deadline = time.monotonic() + .25
try:
yield
finally:
self.reference_deadline = None
@staticmethod
def ident(value):
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
raise ValueError('Invalid SQL identifier')
return '`' + value + '`'
def exists(self, table, condition, args):
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
try:
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
return result
except Exception as error:
if error.args and error.args[0] in (1317, 3024):
raise self.defer('Source reference check exceeded its short query budget') from error
raise
def metadata_mentions(self, needles, excluded=None):
for table, columns in self.db.column_types.items():
for column, kind in columns.items():
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
continue
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
args = list(needles)
if excluded and table == excluded[0]:
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
args.append(excluded[1])
condition = ' AND '.join(conditions)
else:
condition = ' OR '.join(conditions)
if self.exists(table, condition, args):
return True
return False
def referenced(self, revision):
rid = revision['id']
for fk in self.db.fks:
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
return True
# Replacement, withdrawal and restore records include IDs without FKs.
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('revision_id'):
condition = self.ident(column) + '=%s'
args = [rid]
if table == 'source_revision':
condition += ' AND id<>%s'; args.append(rid)
if self.exists(table, condition, args):
return True
# A catalog source can be imported again. Retain its original even when
# its current source_path points to an external workbook rather than archive.
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
return True
return self.metadata_mentions([rid], ('source_revision', rid))
def file_referenced(self, path, exclude_revision=None):
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('_path'):
condition = self.ident(column) + '=%s'; args = [str(path)]
if table == 'source_revision' and exclude_revision:
condition += ' AND id<>%s'; args.append(exclude_revision)
if self.exists(table, condition, args):
return True
# Search both raw and JSON-escaped spelling, including saved snapshots.
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
def owned_file(self, revision):
path = Path(revision['archive_path'])
if not self.safe_file(path, self.boundary):
return None
relative = path.relative_to(self.boundary).parts
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
not re.fullmatch(r'\d{4}', relative[1]) or
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
relative[3] != revision['sha256'] + '.xlsx'):
return None
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
return None
return path
def safe_journals(self):
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
raise ValueError('Unsafe source file journal directory')
def drain(self, current, limit=5):
"""Resume only registered deletions after database commit; never scan stray files."""
self.safe_journals()
removed = size = 0
for record in sorted(self.journals.glob('*.json'))[:limit]:
if record.is_symlink():
continue
try:
value = json.loads(record.read_text())
except (OSError, ValueError):
continue
token = value.get('token', '')
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
value.get('database') != self.db.name or not isinstance(token, str) or
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
continue
revision = value.get('revision')
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
continue
path = Path(revision['archive_path'])
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
record.unlink() # An interrupted/rolled-back delete still owns its file.
continue
if not path.exists() and not path.is_symlink():
record.unlink(); continue
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
continue
if self.br.sha(path) != revision['sha256']:
continue
with self.db.frozen(current), self.quick_checks():
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
self.owned_file(revision) != path or
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
continue
size += path.stat().st_size
path.unlink(); removed += 1
record.unlink()
return removed, size
def collect(self, current, state, now, grace=1800, limit=3):
self.safe_journals()
seen = state.setdefault('sourceRevisions', {})
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
# All registered/current revisions remain readable until both ownership and
# dependency checks pass. Unknown states and missing/damaged files are kept.
revisions = self.db.rows('SELECT * FROM source_revision')
present = {row['id'] for row in revisions}
for stale in set(seen) - present:
seen.pop(stale, None)
deadline = time.monotonic() + 5
for revision in revisions:
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
break
rid = revision['id']
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
seen.pop(rid, None); continue
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
if seen.get(rid, {}).get('identity') != identity:
seen[rid] = {'identity': identity, 'since': now}
if now - seen[rid]['since'] < grace:
continue
path = self.owned_file(revision)
if path is None:
continue
stamp = self.fingerprint(path)
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
continue
# Durable intent first: a crash after row deletion must not leave an
# unregistered orphan. The journal is harmless if the row still exists.
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
'database': self.db.name, 'token': token, 'createdAt': now,
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
'fingerprint': stamp})
with self.db.frozen(current), self.quick_checks():
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
if (not live or live[0] != revision or self.referenced(revision) or
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
record.unlink(); seen.pop(rid, None); continue
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
files, size = self.drain(current)
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
return result