446 lines
21 KiB
Python
446 lines
21 KiB
Python
"""Shared backup/maintenance exclusion and verified-only backup rotation."""
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from functools import wraps
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
_thread_lock = threading.RLock()
|
|
_held = {}
|
|
ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$')
|
|
CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified',
|
|
'archiveFilesVerified', 'archiveReferencesVerified')
|
|
ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256',
|
|
'application.tar.gz': 'applicationSha256'}
|
|
|
|
|
|
class Busy(RuntimeError):
|
|
pass
|
|
|
|
|
|
def sha(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def atomic(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
|
with tmp.open('w') as output:
|
|
os.chmod(tmp, 0o600)
|
|
json.dump(value, output, ensure_ascii=False, indent=2)
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
tmp.replace(path)
|
|
if os.name != 'nt':
|
|
fd = os.open(path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
@contextmanager
|
|
def lock(root, timeout=1800):
|
|
"""One lock for UI backups, restores, scheduled backups and garbage collection."""
|
|
root = Path(root).resolve()
|
|
key = str(root)
|
|
with _thread_lock:
|
|
if key in _held:
|
|
yield
|
|
return
|
|
path = root / '.runtime/data-maintenance.lock'
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.parent.is_symlink() or path.is_symlink():
|
|
raise ValueError('Unsafe maintenance lock path')
|
|
stream = path.open('a+b')
|
|
os.chmod(path, 0o600)
|
|
deadline = time.monotonic() + timeout
|
|
try:
|
|
while True:
|
|
try:
|
|
if os.name == 'nt':
|
|
import msvcrt
|
|
stream.seek(0)
|
|
if not path.stat().st_size:
|
|
stream.write(b'0'); stream.flush()
|
|
stream.seek(0)
|
|
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
break
|
|
except OSError:
|
|
if time.monotonic() >= deadline:
|
|
raise Busy('Another backup or maintenance operation is active')
|
|
time.sleep(.2)
|
|
_held[key] = stream
|
|
try:
|
|
yield
|
|
finally:
|
|
_held.pop(key, None)
|
|
if os.name == 'nt':
|
|
import msvcrt
|
|
stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(stream, fcntl.LOCK_UN)
|
|
finally:
|
|
stream.close()
|
|
|
|
|
|
def serialized(function):
|
|
@wraps(function)
|
|
def wrapped(*args, **kwargs):
|
|
with lock(function.__globals__['ROOT']):
|
|
return function(*args, **kwargs)
|
|
return wrapped
|
|
|
|
|
|
def verified(folder, checksums=True):
|
|
folder = Path(folder)
|
|
if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
|
return None
|
|
try:
|
|
if (folder / 'manifest.json').is_symlink():
|
|
return None
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED':
|
|
return None
|
|
if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS):
|
|
return None
|
|
for name, field in ARTIFACTS.items():
|
|
p = folder / name
|
|
if p.is_symlink() or not p.is_file() or not manifest.get(field):
|
|
return None
|
|
if checksums and sha(p) != manifest[field]:
|
|
return None
|
|
return manifest
|
|
except (OSError, ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _rotate(root, selected):
|
|
"""Publish the new verified backup first. A failed candidate never deletes old backups."""
|
|
root = Path(root).resolve()
|
|
selected = Path(selected)
|
|
if selected.is_symlink():
|
|
raise ValueError('Backup directory cannot be a symbolic link')
|
|
selected = selected.resolve()
|
|
base = root / 'output/backups'
|
|
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):
|
|
raise ValueError('Unsafe managed backup root')
|
|
if selected.parent.resolve() != base.resolve():
|
|
raise ValueError('Backup is outside the managed backup directory')
|
|
manifest = verified(selected)
|
|
if manifest is None:
|
|
raise ValueError('Backup has not passed restore and artifact verification')
|
|
previous = latest(root)
|
|
if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''):
|
|
selected, manifest = previous
|
|
for name in (*ARTIFACTS, 'manifest.json'):
|
|
with (selected / name).open('r+b') as stream:
|
|
stream.flush(); os.fsync(stream.fileno())
|
|
if os.name!='nt':
|
|
for directory in (selected,base,base.parent):
|
|
fd=os.open(directory,os.O_RDONLY)
|
|
try:os.fsync(fd)
|
|
finally:os.close(fd)
|
|
pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'),
|
|
'releaseId': manifest.get('queryEvidence', {}).get('releaseId'),
|
|
'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()}
|
|
atomic(root / '.runtime/retention/latest.json', pointer)
|
|
removed = []
|
|
for folder in base.iterdir():
|
|
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
|
continue
|
|
# 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。
|
|
older = verified(folder, checksums=False)
|
|
if older is not None:
|
|
contents = list(folder.rglob('*'))
|
|
if any(p.is_symlink() for p in contents):
|
|
continue
|
|
try:
|
|
retire_folder(root,folder,'backup')
|
|
removed.append(folder.name)
|
|
except OSError:
|
|
pass # Latest stays usable; a later maintenance pass retries retirement.
|
|
# A one-off snapshot from the previous manual cleanup is retired only when
|
|
# explicitly registered and its complete original file set still matches.
|
|
registration = root / '.runtime/retention/legacy.json'
|
|
if registration.is_file():
|
|
legacy_info = json.loads(registration.read_text())
|
|
if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'):
|
|
legacy = root / legacy_info['path']
|
|
if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']:
|
|
old = json.loads((legacy/'manifest.json').read_text())
|
|
expected = old.get('filesSha256', {})
|
|
actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()}
|
|
if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()):
|
|
evidence = root/'.runtime/retention/setup-evidence'
|
|
evidence.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'):
|
|
if (legacy/name).is_file():
|
|
shutil.copy2(legacy/name,evidence/name)
|
|
retire_folder(root,legacy,'legacy')
|
|
legacy_info['retired']=True
|
|
atomic(registration,legacy_info)
|
|
clean_incomplete(root)
|
|
clean_retired(root)
|
|
return pointer, removed
|
|
|
|
|
|
def retire_folder(root, folder, kind):
|
|
"""Atomic rename makes a crash during removal safely resumable."""
|
|
root=Path(root).resolve();folder=Path(folder).resolve()
|
|
if kind=='backup':
|
|
if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source')
|
|
elif kind=='legacy':
|
|
if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source')
|
|
else:raise ValueError('Unknown retirement kind')
|
|
area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):
|
|
raise ValueError('Unsafe backup retirement directory')
|
|
token=uuid.uuid4().hex
|
|
# Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。
|
|
storage = root/'output/backups/.retired' if kind=='backup' else area
|
|
if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage')
|
|
storage.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token,
|
|
'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'})
|
|
folder.rename(storage/token)
|
|
if os.name!='nt':
|
|
fd=os.open(storage,os.O_RDONLY)
|
|
try:os.fsync(fd)
|
|
finally:os.close(fd)
|
|
|
|
|
|
def clean_retired(root):
|
|
root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0
|
|
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0
|
|
for record in area.glob('*.json'):
|
|
try:
|
|
if record.is_symlink():continue
|
|
value=json.loads(record.read_text());token=value.get('target','')
|
|
if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue
|
|
storage=value.get('storage','state-volume')
|
|
if storage not in ('state-volume','backup-volume'):continue
|
|
base=root/'output/backups/.retired' if storage=='backup-volume' else area
|
|
if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue
|
|
folder=base/token
|
|
if folder.is_symlink():continue
|
|
if folder.exists():
|
|
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
|
shutil.rmtree(folder);count+=1
|
|
record.unlink()
|
|
except (OSError,ValueError):continue
|
|
return count
|
|
|
|
|
|
def rotate(root, selected):
|
|
with lock(root):
|
|
return _rotate(root, selected)
|
|
|
|
|
|
def latest(root, checksums=True):
|
|
root = Path(root).resolve()
|
|
try:
|
|
base=root/'output/backups'
|
|
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None
|
|
pointer = json.loads((root / '.runtime/retention/latest.json').read_text())
|
|
if not ID.fullmatch(pointer['id']):
|
|
return None
|
|
folder = root / 'output/backups' / pointer['id']
|
|
if sha(folder / 'manifest.json') != pointer['manifestSha256']:
|
|
return None
|
|
manifest = verified(folder, checksums=checksums)
|
|
return (folder, manifest) if manifest else None
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return None
|
|
|
|
|
|
def backup_ids(root):
|
|
base=Path(root)/'output/backups'
|
|
return {p.name for p in base.iterdir()} if base.exists() else set()
|
|
|
|
|
|
def register_candidate(folder, root):
|
|
import psutil
|
|
folder = Path(folder); info = folder.stat()
|
|
atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2,
|
|
'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(),
|
|
'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(),
|
|
'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False})
|
|
|
|
|
|
def candidate_child(folder, pid=None, pending=False):
|
|
import psutil
|
|
path = Path(folder) / '.owner.json'; value = json.loads(path.read_text())
|
|
if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed')
|
|
value['childPending'] = pending
|
|
if pid is not None:
|
|
value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()})
|
|
atomic(path, value)
|
|
|
|
|
|
def clean_incomplete(root, now=None):
|
|
"""独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。"""
|
|
import psutil
|
|
root = Path(root).resolve(); now = time.time() if now is None else now
|
|
result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0}
|
|
base = root / 'output/backups'
|
|
if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root')
|
|
def exited(item, pid_key, created_key):
|
|
if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or
|
|
not isinstance(item.get(created_key), (float, int)) or
|
|
not math.isfinite(item[created_key]) or item[created_key] <= 0): return False
|
|
try:
|
|
process = psutil.Process(item[pid_key])
|
|
return (abs(process.create_time() - item[created_key]) >= .02 or
|
|
not process.is_running() or process.status() == psutil.STATUS_ZOMBIE)
|
|
except psutil.NoSuchProcess: return True
|
|
except psutil.Error: return False
|
|
with lock(root, timeout=0):
|
|
pointer = root / '.runtime/retention/latest.json'
|
|
try:
|
|
if pointer.is_symlink(): return result
|
|
protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None
|
|
except (OSError, ValueError, KeyError, TypeError): return result
|
|
for folder in base.iterdir() if base.exists() else []:
|
|
if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue
|
|
owner = folder / '.owner.json'
|
|
if owner.is_symlink(): continue
|
|
try:
|
|
value = json.loads(owner.read_text()); info = folder.stat()
|
|
if not isinstance(value, dict): continue
|
|
if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or
|
|
value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or
|
|
not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue
|
|
if verified(folder, checksums=False) is not None: continue
|
|
if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or
|
|
value.get('childPending') is not False or not isinstance(value.get('children'), list) or
|
|
not exited(value, 'ownerPid', 'ownerCreated') or
|
|
not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])):
|
|
result['incompleteBackupsDeferred'] += 1; continue
|
|
contents = list(folder.rglob('*'))
|
|
if any(p.is_symlink() for p in contents): continue
|
|
size = sum(p.stat().st_size for p in contents if p.is_file())
|
|
if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue
|
|
retire_folder(root, folder, 'backup')
|
|
result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size
|
|
except (OSError, ValueError, TypeError): continue
|
|
clean_retired(root)
|
|
return result
|
|
|
|
|
|
def discard_failed_new(root, before):
|
|
"""Delete only this call's registered, incomplete candidate after it has stopped."""
|
|
root=Path(root).resolve();base=root/'output/backups'
|
|
for folder in base.iterdir() if base.exists() else []:
|
|
if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue
|
|
try:
|
|
owner=json.loads((folder/'.owner.json').read_text())
|
|
if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue
|
|
if verified(folder,checksums=False) is not None:continue
|
|
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
|
shutil.rmtree(folder)
|
|
except (OSError,ValueError):continue
|
|
|
|
|
|
def accepted_backend(root):
|
|
"""The running pinned jar is part of a valid application backup on this Mac."""
|
|
root = Path(root).resolve()
|
|
# 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。
|
|
if os.environ.get('QUALITY_CONTAINER') == '1':
|
|
path = root / 'app/backend.jar'
|
|
digest = (root / 'app/backend.sha256').read_text().strip()
|
|
if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest:
|
|
raise ValueError('Container backend digest mismatch')
|
|
return path, digest
|
|
config = Path.home() / 'Library/Application Support/QualityCenterService/service.json'
|
|
if not config.exists():
|
|
return None
|
|
value = json.loads(config.read_text())
|
|
if Path(value['root']).resolve() != root:
|
|
return None
|
|
digest = value['jarSha256']
|
|
if not re.fullmatch(r'[a-f0-9]{64}', digest):
|
|
raise ValueError('Invalid accepted backend digest')
|
|
path = root / '.runtime/backend-builds' / (digest + '.jar')
|
|
if path.is_symlink() or sha(path) != digest:
|
|
raise ValueError('Accepted backend digest mismatch')
|
|
return path, digest
|
|
|
|
|
|
def native_files(root):
|
|
"""Retain restorable native settings/conversations; exclude live access tokens."""
|
|
for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'):
|
|
path=Path(root)/relative
|
|
if path.is_symlink():raise ValueError('Unsafe operations configuration link')
|
|
if path.is_file():yield path
|
|
base=Path(root)/'.runtime/nao-users'
|
|
ignored={'logs','backups','__pycache__'}
|
|
ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'}
|
|
for path in base.rglob('*') if base.exists() else []:
|
|
if path.is_symlink():
|
|
raise ValueError('Unsafe native workspace link')
|
|
if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts):
|
|
continue
|
|
if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')):
|
|
continue
|
|
yield path
|
|
|
|
|
|
def snapshot_sqlite(source, target):
|
|
original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True)
|
|
copied=sqlite3.connect(target)
|
|
try:
|
|
original.backup(copied)
|
|
copied.execute('PRAGMA foreign_keys=OFF')
|
|
copied.execute('PRAGMA secure_delete=ON')
|
|
tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token',
|
|
'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'):
|
|
if name in tables:
|
|
copied.execute('DELETE FROM "'+name+'"')
|
|
copied.commit();copied.execute('VACUUM')
|
|
if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
|
|
raise ValueError('Native SQLite backup integrity failed')
|
|
finally:
|
|
original.close();copied.close()
|
|
|
|
|
|
def scratch_register(root, database, server):
|
|
import psutil
|
|
if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database):
|
|
raise ValueError('Unexpected verification database name')
|
|
path=Path(root)/'.runtime/retention/scratch'/(database+'.json')
|
|
value={'kind':'quality-restore-check','database':database,'server':server,
|
|
'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False}
|
|
atomic(path,value)
|
|
return path
|
|
|
|
|
|
def scratch_created(path):
|
|
value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value)
|
|
|
|
|
|
def scratch_child(path, pid):
|
|
import psutil
|
|
value=json.loads(Path(path).read_text())
|
|
value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()})
|
|
atomic(path,value)
|