Files
quality_analyse/scripts/backup_retention.py
T

382 lines
17 KiB
Python

"""Shared backup/maintenance exclusion and verified-only backup rotation."""
from contextlib import contextmanager
from datetime import datetime, timezone
from functools import wraps
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import threading
import time
import uuid
_thread_lock = threading.RLock()
_held = {}
ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$')
CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified',
'archiveFilesVerified', 'archiveReferencesVerified')
ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256',
'application.tar.gz': 'applicationSha256'}
class Busy(RuntimeError):
pass
def sha(path):
digest = hashlib.sha256()
with Path(path).open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''):
digest.update(block)
return digest.hexdigest()
def atomic(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
with tmp.open('w') as output:
os.chmod(tmp, 0o600)
json.dump(value, output, ensure_ascii=False, indent=2)
output.flush()
os.fsync(output.fileno())
tmp.replace(path)
if os.name != 'nt':
fd = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
@contextmanager
def lock(root, timeout=1800):
"""One lock for UI backups, restores, scheduled backups and garbage collection."""
root = Path(root).resolve()
key = str(root)
with _thread_lock:
if key in _held:
yield
return
path = root / '.runtime/data-maintenance.lock'
path.parent.mkdir(parents=True, exist_ok=True)
if path.parent.is_symlink() or path.is_symlink():
raise ValueError('Unsafe maintenance lock path')
stream = path.open('a+b')
os.chmod(path, 0o600)
deadline = time.monotonic() + timeout
try:
while True:
try:
if os.name == 'nt':
import msvcrt
stream.seek(0)
if not path.stat().st_size:
stream.write(b'0'); stream.flush()
stream.seek(0)
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except OSError:
if time.monotonic() >= deadline:
raise Busy('Another backup or maintenance operation is active')
time.sleep(.2)
_held[key] = stream
try:
yield
finally:
_held.pop(key, None)
if os.name == 'nt':
import msvcrt
stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(stream, fcntl.LOCK_UN)
finally:
stream.close()
def serialized(function):
@wraps(function)
def wrapped(*args, **kwargs):
with lock(function.__globals__['ROOT']):
return function(*args, **kwargs)
return wrapped
def verified(folder, checksums=True):
folder = Path(folder)
if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
return None
try:
if (folder / 'manifest.json').is_symlink():
return None
manifest = json.loads((folder / 'manifest.json').read_text())
if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED':
return None
if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS):
return None
for name, field in ARTIFACTS.items():
p = folder / name
if p.is_symlink() or not p.is_file() or not manifest.get(field):
return None
if checksums and sha(p) != manifest[field]:
return None
return manifest
except (OSError, ValueError, TypeError):
return None
def _rotate(root, selected):
"""Publish the new verified backup first. A failed candidate never deletes old backups."""
root = Path(root).resolve()
selected = Path(selected)
if selected.is_symlink():
raise ValueError('Backup directory cannot be a symbolic link')
selected = selected.resolve()
base = root / 'output/backups'
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):
raise ValueError('Unsafe managed backup root')
if selected.parent.resolve() != base.resolve():
raise ValueError('Backup is outside the managed backup directory')
manifest = verified(selected)
if manifest is None:
raise ValueError('Backup has not passed restore and artifact verification')
previous = latest(root)
if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''):
selected, manifest = previous
for name in (*ARTIFACTS, 'manifest.json'):
with (selected / name).open('r+b') as stream:
stream.flush(); os.fsync(stream.fileno())
if os.name!='nt':
for directory in (selected,base,base.parent):
fd=os.open(directory,os.O_RDONLY)
try:os.fsync(fd)
finally:os.close(fd)
pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'),
'releaseId': manifest.get('queryEvidence', {}).get('releaseId'),
'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()}
atomic(root / '.runtime/retention/latest.json', pointer)
removed = []
for folder in base.iterdir():
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
continue
# A restore-verified older folder is safe to retire under the shared lock.
# Incomplete candidates are kept for 24h and never treated as good backups.
older = verified(folder, checksums=False)
try:
owner = json.loads((folder / '.owner.json').read_text())
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
except (OSError, ValueError):
owned = False
stale = owned and time.time() - folder.stat().st_mtime > 86400
if older is not None or stale:
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents):
continue
try:
retire_folder(root,folder,'backup')
removed.append(folder.name)
except OSError:
pass # Latest stays usable; a later maintenance pass retries retirement.
# A one-off snapshot from the previous manual cleanup is retired only when
# explicitly registered and its complete original file set still matches.
registration = root / '.runtime/retention/legacy.json'
if registration.is_file():
legacy_info = json.loads(registration.read_text())
if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'):
legacy = root / legacy_info['path']
if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']:
old = json.loads((legacy/'manifest.json').read_text())
expected = old.get('filesSha256', {})
actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()}
if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()):
evidence = root/'.runtime/retention/setup-evidence'
evidence.mkdir(parents=True,exist_ok=True,mode=0o700)
for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'):
if (legacy/name).is_file():
shutil.copy2(legacy/name,evidence/name)
retire_folder(root,legacy,'legacy')
legacy_info['retired']=True
atomic(registration,legacy_info)
clean_retired(root)
return pointer, removed
def retire_folder(root, folder, kind):
"""Atomic rename makes a crash during removal safely resumable."""
root=Path(root).resolve();folder=Path(folder).resolve()
if kind=='backup':
if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source')
elif kind=='legacy':
if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source')
else:raise ValueError('Unknown retirement kind')
area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700)
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):
raise ValueError('Unsafe backup retirement directory')
token=uuid.uuid4().hex
# Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。
storage = root/'output/backups/.retired' if kind=='backup' else area
if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage')
storage.mkdir(parents=True,exist_ok=True,mode=0o700)
atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token,
'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'})
folder.rename(storage/token)
if os.name!='nt':
fd=os.open(storage,os.O_RDONLY)
try:os.fsync(fd)
finally:os.close(fd)
def clean_retired(root):
root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0
for record in area.glob('*.json'):
try:
if record.is_symlink():continue
value=json.loads(record.read_text());token=value.get('target','')
if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue
storage=value.get('storage','state-volume')
if storage not in ('state-volume','backup-volume'):continue
base=root/'output/backups/.retired' if storage=='backup-volume' else area
if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue
folder=base/token
if folder.is_symlink():continue
if folder.exists():
if any(p.is_symlink() for p in folder.rglob('*')):continue
shutil.rmtree(folder);count+=1
record.unlink()
except (OSError,ValueError):continue
return count
def rotate(root, selected):
with lock(root):
return _rotate(root, selected)
def latest(root, checksums=True):
root = Path(root).resolve()
try:
base=root/'output/backups'
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None
pointer = json.loads((root / '.runtime/retention/latest.json').read_text())
if not ID.fullmatch(pointer['id']):
return None
folder = root / 'output/backups' / pointer['id']
if sha(folder / 'manifest.json') != pointer['manifestSha256']:
return None
manifest = verified(folder, checksums=checksums)
return (folder, manifest) if manifest else None
except (OSError, ValueError, KeyError, TypeError):
return None
def backup_ids(root):
base=Path(root)/'output/backups'
return {p.name for p in base.iterdir()} if base.exists() else set()
def discard_failed_new(root, before):
"""Delete only this call's registered, incomplete candidate after it has stopped."""
root=Path(root).resolve();base=root/'output/backups'
for folder in base.iterdir() if base.exists() else []:
if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue
try:
owner=json.loads((folder/'.owner.json').read_text())
if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue
if verified(folder,checksums=False) is not None:continue
if any(p.is_symlink() for p in folder.rglob('*')):continue
shutil.rmtree(folder)
except (OSError,ValueError):continue
def accepted_backend(root):
"""The running pinned jar is part of a valid application backup on this Mac."""
root = Path(root).resolve()
# 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。
if os.environ.get('QUALITY_CONTAINER') == '1':
path = root / 'app/backend.jar'
digest = (root / 'app/backend.sha256').read_text().strip()
if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest:
raise ValueError('Container backend digest mismatch')
return path, digest
config = Path.home() / 'Library/Application Support/QualityCenterService/service.json'
if not config.exists():
return None
value = json.loads(config.read_text())
if Path(value['root']).resolve() != root:
return None
digest = value['jarSha256']
if not re.fullmatch(r'[a-f0-9]{64}', digest):
raise ValueError('Invalid accepted backend digest')
path = root / '.runtime/backend-builds' / (digest + '.jar')
if path.is_symlink() or sha(path) != digest:
raise ValueError('Accepted backend digest mismatch')
return path, digest
def native_files(root):
"""Retain restorable native settings/conversations; exclude live access tokens."""
for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'):
path=Path(root)/relative
if path.is_symlink():raise ValueError('Unsafe operations configuration link')
if path.is_file():yield path
base=Path(root)/'.runtime/nao-users'
ignored={'logs','backups','__pycache__'}
ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'}
for path in base.rglob('*') if base.exists() else []:
if path.is_symlink():
raise ValueError('Unsafe native workspace link')
if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts):
continue
if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')):
continue
yield path
def snapshot_sqlite(source, target):
original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True)
copied=sqlite3.connect(target)
try:
original.backup(copied)
copied.execute('PRAGMA foreign_keys=OFF')
copied.execute('PRAGMA secure_delete=ON')
tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")}
for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token',
'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'):
if name in tables:
copied.execute('DELETE FROM "'+name+'"')
copied.commit();copied.execute('VACUUM')
if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
raise ValueError('Native SQLite backup integrity failed')
finally:
original.close();copied.close()
def scratch_register(root, database, server):
import psutil
if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database):
raise ValueError('Unexpected verification database name')
path=Path(root)/'.runtime/retention/scratch'/(database+'.json')
value={'kind':'quality-restore-check','database':database,'server':server,
'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False}
atomic(path,value)
return path
def scratch_created(path):
value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value)
def scratch_child(path, pid):
import psutil
value=json.loads(Path(path).read_text())
value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()})
atomic(path,value)