"""Shared backup/maintenance exclusion and verified-only backup rotation.""" from contextlib import contextmanager from datetime import datetime, timezone from functools import wraps import hashlib import json import math import os from pathlib import Path import re import shutil import sqlite3 import threading import time import uuid _thread_lock = threading.RLock() _held = {} ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$') CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified', 'archiveFilesVerified', 'archiveReferencesVerified') ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256', 'application.tar.gz': 'applicationSha256'} class Busy(RuntimeError): pass def sha(path): digest = hashlib.sha256() with Path(path).open('rb') as stream: for block in iter(lambda: stream.read(1024 * 1024), b''): digest.update(block) return digest.hexdigest() def atomic(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp') with tmp.open('w') as output: os.chmod(tmp, 0o600) json.dump(value, output, ensure_ascii=False, indent=2) output.flush() os.fsync(output.fileno()) tmp.replace(path) if os.name != 'nt': fd = os.open(path.parent, os.O_RDONLY) try: os.fsync(fd) finally: os.close(fd) @contextmanager def lock(root, timeout=1800): """One lock for UI backups, restores, scheduled backups and garbage collection.""" root = Path(root).resolve() key = str(root) with _thread_lock: if key in _held: yield return path = root / '.runtime/data-maintenance.lock' path.parent.mkdir(parents=True, exist_ok=True) if path.parent.is_symlink() or path.is_symlink(): raise ValueError('Unsafe maintenance lock path') stream = path.open('a+b') os.chmod(path, 0o600) deadline = time.monotonic() + timeout try: while True: try: if os.name == 'nt': import msvcrt stream.seek(0) if not path.stat().st_size: stream.write(b'0'); stream.flush() stream.seek(0) msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) else: import fcntl fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) break except OSError: if time.monotonic() >= deadline: raise Busy('Another backup or maintenance operation is active') time.sleep(.2) _held[key] = stream try: yield finally: _held.pop(key, None) if os.name == 'nt': import msvcrt stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) else: import fcntl fcntl.flock(stream, fcntl.LOCK_UN) finally: stream.close() def serialized(function): @wraps(function) def wrapped(*args, **kwargs): with lock(function.__globals__['ROOT']): return function(*args, **kwargs) return wrapped def verified(folder, checksums=True): folder = Path(folder) if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): return None try: if (folder / 'manifest.json').is_symlink(): return None manifest = json.loads((folder / 'manifest.json').read_text()) if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED': return None if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS): return None for name, field in ARTIFACTS.items(): p = folder / name if p.is_symlink() or not p.is_file() or not manifest.get(field): return None if checksums and sha(p) != manifest[field]: return None return manifest except (OSError, ValueError, TypeError): return None def _rotate(root, selected): """Publish the new verified backup first. A failed candidate never deletes old backups.""" root = Path(root).resolve() selected = Path(selected) if selected.is_symlink(): raise ValueError('Backup directory cannot be a symbolic link') selected = selected.resolve() base = root / 'output/backups' if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root): raise ValueError('Unsafe managed backup root') if selected.parent.resolve() != base.resolve(): raise ValueError('Backup is outside the managed backup directory') manifest = verified(selected) if manifest is None: raise ValueError('Backup has not passed restore and artifact verification') previous = latest(root) if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''): selected, manifest = previous for name in (*ARTIFACTS, 'manifest.json'): with (selected / name).open('r+b') as stream: stream.flush(); os.fsync(stream.fileno()) if os.name!='nt': for directory in (selected,base,base.parent): fd=os.open(directory,os.O_RDONLY) try:os.fsync(fd) finally:os.close(fd) pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'), 'releaseId': manifest.get('queryEvidence', {}).get('releaseId'), 'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()} atomic(root / '.runtime/retention/latest.json', pointer) removed = [] for folder in base.iterdir(): if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue # 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。 older = verified(folder, checksums=False) if older is not None: contents = list(folder.rglob('*')) if any(p.is_symlink() for p in contents): continue try: retire_folder(root,folder,'backup') removed.append(folder.name) except OSError: pass # Latest stays usable; a later maintenance pass retries retirement. # A one-off snapshot from the previous manual cleanup is retired only when # explicitly registered and its complete original file set still matches. registration = root / '.runtime/retention/legacy.json' if registration.is_file(): legacy_info = json.loads(registration.read_text()) if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'): legacy = root / legacy_info['path'] if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']: old = json.loads((legacy/'manifest.json').read_text()) expected = old.get('filesSha256', {}) actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()} if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()): evidence = root/'.runtime/retention/setup-evidence' evidence.mkdir(parents=True,exist_ok=True,mode=0o700) for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'): if (legacy/name).is_file(): shutil.copy2(legacy/name,evidence/name) retire_folder(root,legacy,'legacy') legacy_info['retired']=True atomic(registration,legacy_info) clean_incomplete(root) clean_retired(root) return pointer, removed def retire_folder(root, folder, kind): """Atomic rename makes a crash during removal safely resumable.""" root=Path(root).resolve();folder=Path(folder).resolve() if kind=='backup': if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source') elif kind=='legacy': if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source') else:raise ValueError('Unknown retirement kind') area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700) if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')): raise ValueError('Unsafe backup retirement directory') token=uuid.uuid4().hex # Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。 storage = root/'output/backups/.retired' if kind=='backup' else area if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage') storage.mkdir(parents=True,exist_ok=True,mode=0o700) atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token, 'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'}) folder.rename(storage/token) if os.name!='nt': fd=os.open(storage,os.O_RDONLY) try:os.fsync(fd) finally:os.close(fd) def clean_retired(root): root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0 if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0 for record in area.glob('*.json'): try: if record.is_symlink():continue value=json.loads(record.read_text());token=value.get('target','') if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue storage=value.get('storage','state-volume') if storage not in ('state-volume','backup-volume'):continue base=root/'output/backups/.retired' if storage=='backup-volume' else area if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue folder=base/token if folder.is_symlink():continue if folder.exists(): if any(p.is_symlink() for p in folder.rglob('*')):continue shutil.rmtree(folder);count+=1 record.unlink() except (OSError,ValueError):continue return count def rotate(root, selected): with lock(root): return _rotate(root, selected) def latest(root, checksums=True): root = Path(root).resolve() try: base=root/'output/backups' if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None pointer = json.loads((root / '.runtime/retention/latest.json').read_text()) if not ID.fullmatch(pointer['id']): return None folder = root / 'output/backups' / pointer['id'] if sha(folder / 'manifest.json') != pointer['manifestSha256']: return None manifest = verified(folder, checksums=checksums) return (folder, manifest) if manifest else None except (OSError, ValueError, KeyError, TypeError): return None def backup_ids(root): base=Path(root)/'output/backups' return {p.name for p in base.iterdir()} if base.exists() else set() def register_candidate(folder, root): import psutil folder = Path(folder); info = folder.stat() atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2, 'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(), 'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False}) def candidate_child(folder, pid=None, pending=False): import psutil path = Path(folder) / '.owner.json'; value = json.loads(path.read_text()) if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed') value['childPending'] = pending if pid is not None: value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()}) atomic(path, value) def clean_incomplete(root, now=None): """独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。""" import psutil root = Path(root).resolve(); now = time.time() if now is None else now result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0} base = root / 'output/backups' if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root') def exited(item, pid_key, created_key): if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or not isinstance(item.get(created_key), (float, int)) or not math.isfinite(item[created_key]) or item[created_key] <= 0): return False try: process = psutil.Process(item[pid_key]) return (abs(process.create_time() - item[created_key]) >= .02 or not process.is_running() or process.status() == psutil.STATUS_ZOMBIE) except psutil.NoSuchProcess: return True except psutil.Error: return False with lock(root, timeout=0): pointer = root / '.runtime/retention/latest.json' try: if pointer.is_symlink(): return result protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None except (OSError, ValueError, KeyError, TypeError): return result for folder in base.iterdir() if base.exists() else []: if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue owner = folder / '.owner.json' if owner.is_symlink(): continue try: value = json.loads(owner.read_text()); info = folder.stat() if not isinstance(value, dict): continue if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue if verified(folder, checksums=False) is not None: continue if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or value.get('childPending') is not False or not isinstance(value.get('children'), list) or not exited(value, 'ownerPid', 'ownerCreated') or not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])): result['incompleteBackupsDeferred'] += 1; continue contents = list(folder.rglob('*')) if any(p.is_symlink() for p in contents): continue size = sum(p.stat().st_size for p in contents if p.is_file()) if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue retire_folder(root, folder, 'backup') result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size except (OSError, ValueError, TypeError): continue clean_retired(root) return result def discard_failed_new(root, before): """Delete only this call's registered, incomplete candidate after it has stopped.""" root=Path(root).resolve();base=root/'output/backups' for folder in base.iterdir() if base.exists() else []: if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue try: owner=json.loads((folder/'.owner.json').read_text()) if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue if verified(folder,checksums=False) is not None:continue if any(p.is_symlink() for p in folder.rglob('*')):continue shutil.rmtree(folder) except (OSError,ValueError):continue def accepted_backend(root): """The running pinned jar is part of a valid application backup on this Mac.""" root = Path(root).resolve() # 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。 if os.environ.get('QUALITY_CONTAINER') == '1': path = root / 'app/backend.jar' digest = (root / 'app/backend.sha256').read_text().strip() if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest: raise ValueError('Container backend digest mismatch') return path, digest config = Path.home() / 'Library/Application Support/QualityCenterService/service.json' if not config.exists(): return None value = json.loads(config.read_text()) if Path(value['root']).resolve() != root: return None digest = value['jarSha256'] if not re.fullmatch(r'[a-f0-9]{64}', digest): raise ValueError('Invalid accepted backend digest') path = root / '.runtime/backend-builds' / (digest + '.jar') if path.is_symlink() or sha(path) != digest: raise ValueError('Accepted backend digest mismatch') return path, digest def native_files(root): """Retain restorable native settings/conversations; exclude live access tokens.""" for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'): path=Path(root)/relative if path.is_symlink():raise ValueError('Unsafe operations configuration link') if path.is_file():yield path base=Path(root)/'.runtime/nao-users' ignored={'logs','backups','__pycache__'} ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'} for path in base.rglob('*') if base.exists() else []: if path.is_symlink(): raise ValueError('Unsafe native workspace link') if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts): continue if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')): continue yield path def snapshot_sqlite(source, target): original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True) copied=sqlite3.connect(target) try: original.backup(copied) copied.execute('PRAGMA foreign_keys=OFF') copied.execute('PRAGMA secure_delete=ON') tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")} for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token', 'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'): if name in tables: copied.execute('DELETE FROM "'+name+'"') copied.commit();copied.execute('VACUUM') if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok': raise ValueError('Native SQLite backup integrity failed') finally: original.close();copied.close() def scratch_register(root, database, server): import psutil if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database): raise ValueError('Unexpected verification database name') path=Path(root)/'.runtime/retention/scratch'/(database+'.json') value={'kind':'quality-restore-check','database':database,'server':server, 'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False} atomic(path,value) return path def scratch_created(path): value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value) def scratch_child(path, pid): import psutil value=json.loads(Path(path).read_text()) value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()}) atomic(path,value)