新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,381 @@
|
||||
"""Shared backup/maintenance exclusion and verified-only backup rotation."""
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from functools import wraps
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
_thread_lock = threading.RLock()
|
||||
_held = {}
|
||||
ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$')
|
||||
CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified',
|
||||
'archiveFilesVerified', 'archiveReferencesVerified')
|
||||
ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256',
|
||||
'application.tar.gz': 'applicationSha256'}
|
||||
|
||||
|
||||
class Busy(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def sha(path):
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open('rb') as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
||||
with tmp.open('w') as output:
|
||||
os.chmod(tmp, 0o600)
|
||||
json.dump(value, output, ensure_ascii=False, indent=2)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
tmp.replace(path)
|
||||
if os.name != 'nt':
|
||||
fd = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def lock(root, timeout=1800):
|
||||
"""One lock for UI backups, restores, scheduled backups and garbage collection."""
|
||||
root = Path(root).resolve()
|
||||
key = str(root)
|
||||
with _thread_lock:
|
||||
if key in _held:
|
||||
yield
|
||||
return
|
||||
path = root / '.runtime/data-maintenance.lock'
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.parent.is_symlink() or path.is_symlink():
|
||||
raise ValueError('Unsafe maintenance lock path')
|
||||
stream = path.open('a+b')
|
||||
os.chmod(path, 0o600)
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
stream.seek(0)
|
||||
if not path.stat().st_size:
|
||||
stream.write(b'0'); stream.flush()
|
||||
stream.seek(0)
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise Busy('Another backup or maintenance operation is active')
|
||||
time.sleep(.2)
|
||||
_held[key] = stream
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_held.pop(key, None)
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(stream, fcntl.LOCK_UN)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
|
||||
def serialized(function):
|
||||
@wraps(function)
|
||||
def wrapped(*args, **kwargs):
|
||||
with lock(function.__globals__['ROOT']):
|
||||
return function(*args, **kwargs)
|
||||
return wrapped
|
||||
|
||||
|
||||
def verified(folder, checksums=True):
|
||||
folder = Path(folder)
|
||||
if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
||||
return None
|
||||
try:
|
||||
if (folder / 'manifest.json').is_symlink():
|
||||
return None
|
||||
manifest = json.loads((folder / 'manifest.json').read_text())
|
||||
if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED':
|
||||
return None
|
||||
if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS):
|
||||
return None
|
||||
for name, field in ARTIFACTS.items():
|
||||
p = folder / name
|
||||
if p.is_symlink() or not p.is_file() or not manifest.get(field):
|
||||
return None
|
||||
if checksums and sha(p) != manifest[field]:
|
||||
return None
|
||||
return manifest
|
||||
except (OSError, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _rotate(root, selected):
|
||||
"""Publish the new verified backup first. A failed candidate never deletes old backups."""
|
||||
root = Path(root).resolve()
|
||||
selected = Path(selected)
|
||||
if selected.is_symlink():
|
||||
raise ValueError('Backup directory cannot be a symbolic link')
|
||||
selected = selected.resolve()
|
||||
base = root / 'output/backups'
|
||||
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):
|
||||
raise ValueError('Unsafe managed backup root')
|
||||
if selected.parent.resolve() != base.resolve():
|
||||
raise ValueError('Backup is outside the managed backup directory')
|
||||
manifest = verified(selected)
|
||||
if manifest is None:
|
||||
raise ValueError('Backup has not passed restore and artifact verification')
|
||||
previous = latest(root)
|
||||
if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''):
|
||||
selected, manifest = previous
|
||||
for name in (*ARTIFACTS, 'manifest.json'):
|
||||
with (selected / name).open('r+b') as stream:
|
||||
stream.flush(); os.fsync(stream.fileno())
|
||||
if os.name!='nt':
|
||||
for directory in (selected,base,base.parent):
|
||||
fd=os.open(directory,os.O_RDONLY)
|
||||
try:os.fsync(fd)
|
||||
finally:os.close(fd)
|
||||
pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'),
|
||||
'releaseId': manifest.get('queryEvidence', {}).get('releaseId'),
|
||||
'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()}
|
||||
atomic(root / '.runtime/retention/latest.json', pointer)
|
||||
removed = []
|
||||
for folder in base.iterdir():
|
||||
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
||||
continue
|
||||
# A restore-verified older folder is safe to retire under the shared lock.
|
||||
# Incomplete candidates are kept for 24h and never treated as good backups.
|
||||
older = verified(folder, checksums=False)
|
||||
try:
|
||||
owner = json.loads((folder / '.owner.json').read_text())
|
||||
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
|
||||
except (OSError, ValueError):
|
||||
owned = False
|
||||
stale = owned and time.time() - folder.stat().st_mtime > 86400
|
||||
if older is not None or stale:
|
||||
contents = list(folder.rglob('*'))
|
||||
if any(p.is_symlink() for p in contents):
|
||||
continue
|
||||
try:
|
||||
retire_folder(root,folder,'backup')
|
||||
removed.append(folder.name)
|
||||
except OSError:
|
||||
pass # Latest stays usable; a later maintenance pass retries retirement.
|
||||
# A one-off snapshot from the previous manual cleanup is retired only when
|
||||
# explicitly registered and its complete original file set still matches.
|
||||
registration = root / '.runtime/retention/legacy.json'
|
||||
if registration.is_file():
|
||||
legacy_info = json.loads(registration.read_text())
|
||||
if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'):
|
||||
legacy = root / legacy_info['path']
|
||||
if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']:
|
||||
old = json.loads((legacy/'manifest.json').read_text())
|
||||
expected = old.get('filesSha256', {})
|
||||
actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()}
|
||||
if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()):
|
||||
evidence = root/'.runtime/retention/setup-evidence'
|
||||
evidence.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'):
|
||||
if (legacy/name).is_file():
|
||||
shutil.copy2(legacy/name,evidence/name)
|
||||
retire_folder(root,legacy,'legacy')
|
||||
legacy_info['retired']=True
|
||||
atomic(registration,legacy_info)
|
||||
clean_retired(root)
|
||||
return pointer, removed
|
||||
|
||||
|
||||
def retire_folder(root, folder, kind):
|
||||
"""Atomic rename makes a crash during removal safely resumable."""
|
||||
root=Path(root).resolve();folder=Path(folder).resolve()
|
||||
if kind=='backup':
|
||||
if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source')
|
||||
elif kind=='legacy':
|
||||
if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source')
|
||||
else:raise ValueError('Unknown retirement kind')
|
||||
area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):
|
||||
raise ValueError('Unsafe backup retirement directory')
|
||||
token=uuid.uuid4().hex
|
||||
# Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。
|
||||
storage = root/'output/backups/.retired' if kind=='backup' else area
|
||||
if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage')
|
||||
storage.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token,
|
||||
'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'})
|
||||
folder.rename(storage/token)
|
||||
if os.name!='nt':
|
||||
fd=os.open(storage,os.O_RDONLY)
|
||||
try:os.fsync(fd)
|
||||
finally:os.close(fd)
|
||||
|
||||
|
||||
def clean_retired(root):
|
||||
root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0
|
||||
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0
|
||||
for record in area.glob('*.json'):
|
||||
try:
|
||||
if record.is_symlink():continue
|
||||
value=json.loads(record.read_text());token=value.get('target','')
|
||||
if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue
|
||||
storage=value.get('storage','state-volume')
|
||||
if storage not in ('state-volume','backup-volume'):continue
|
||||
base=root/'output/backups/.retired' if storage=='backup-volume' else area
|
||||
if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue
|
||||
folder=base/token
|
||||
if folder.is_symlink():continue
|
||||
if folder.exists():
|
||||
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
||||
shutil.rmtree(folder);count+=1
|
||||
record.unlink()
|
||||
except (OSError,ValueError):continue
|
||||
return count
|
||||
|
||||
|
||||
def rotate(root, selected):
|
||||
with lock(root):
|
||||
return _rotate(root, selected)
|
||||
|
||||
|
||||
def latest(root, checksums=True):
|
||||
root = Path(root).resolve()
|
||||
try:
|
||||
base=root/'output/backups'
|
||||
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None
|
||||
pointer = json.loads((root / '.runtime/retention/latest.json').read_text())
|
||||
if not ID.fullmatch(pointer['id']):
|
||||
return None
|
||||
folder = root / 'output/backups' / pointer['id']
|
||||
if sha(folder / 'manifest.json') != pointer['manifestSha256']:
|
||||
return None
|
||||
manifest = verified(folder, checksums=checksums)
|
||||
return (folder, manifest) if manifest else None
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def backup_ids(root):
|
||||
base=Path(root)/'output/backups'
|
||||
return {p.name for p in base.iterdir()} if base.exists() else set()
|
||||
|
||||
|
||||
def discard_failed_new(root, before):
|
||||
"""Delete only this call's registered, incomplete candidate after it has stopped."""
|
||||
root=Path(root).resolve();base=root/'output/backups'
|
||||
for folder in base.iterdir() if base.exists() else []:
|
||||
if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue
|
||||
try:
|
||||
owner=json.loads((folder/'.owner.json').read_text())
|
||||
if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue
|
||||
if verified(folder,checksums=False) is not None:continue
|
||||
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
||||
shutil.rmtree(folder)
|
||||
except (OSError,ValueError):continue
|
||||
|
||||
|
||||
def accepted_backend(root):
|
||||
"""The running pinned jar is part of a valid application backup on this Mac."""
|
||||
root = Path(root).resolve()
|
||||
# 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。
|
||||
if os.environ.get('QUALITY_CONTAINER') == '1':
|
||||
path = root / 'app/backend.jar'
|
||||
digest = (root / 'app/backend.sha256').read_text().strip()
|
||||
if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest:
|
||||
raise ValueError('Container backend digest mismatch')
|
||||
return path, digest
|
||||
config = Path.home() / 'Library/Application Support/QualityCenterService/service.json'
|
||||
if not config.exists():
|
||||
return None
|
||||
value = json.loads(config.read_text())
|
||||
if Path(value['root']).resolve() != root:
|
||||
return None
|
||||
digest = value['jarSha256']
|
||||
if not re.fullmatch(r'[a-f0-9]{64}', digest):
|
||||
raise ValueError('Invalid accepted backend digest')
|
||||
path = root / '.runtime/backend-builds' / (digest + '.jar')
|
||||
if path.is_symlink() or sha(path) != digest:
|
||||
raise ValueError('Accepted backend digest mismatch')
|
||||
return path, digest
|
||||
|
||||
|
||||
def native_files(root):
|
||||
"""Retain restorable native settings/conversations; exclude live access tokens."""
|
||||
for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'):
|
||||
path=Path(root)/relative
|
||||
if path.is_symlink():raise ValueError('Unsafe operations configuration link')
|
||||
if path.is_file():yield path
|
||||
base=Path(root)/'.runtime/nao-users'
|
||||
ignored={'logs','backups','__pycache__'}
|
||||
ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'}
|
||||
for path in base.rglob('*') if base.exists() else []:
|
||||
if path.is_symlink():
|
||||
raise ValueError('Unsafe native workspace link')
|
||||
if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts):
|
||||
continue
|
||||
if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')):
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def snapshot_sqlite(source, target):
|
||||
original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True)
|
||||
copied=sqlite3.connect(target)
|
||||
try:
|
||||
original.backup(copied)
|
||||
copied.execute('PRAGMA foreign_keys=OFF')
|
||||
copied.execute('PRAGMA secure_delete=ON')
|
||||
tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token',
|
||||
'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'):
|
||||
if name in tables:
|
||||
copied.execute('DELETE FROM "'+name+'"')
|
||||
copied.commit();copied.execute('VACUUM')
|
||||
if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
|
||||
raise ValueError('Native SQLite backup integrity failed')
|
||||
finally:
|
||||
original.close();copied.close()
|
||||
|
||||
|
||||
def scratch_register(root, database, server):
|
||||
import psutil
|
||||
if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database):
|
||||
raise ValueError('Unexpected verification database name')
|
||||
path=Path(root)/'.runtime/retention/scratch'/(database+'.json')
|
||||
value={'kind':'quality-restore-check','database':database,'server':server,
|
||||
'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False}
|
||||
atomic(path,value)
|
||||
return path
|
||||
|
||||
|
||||
def scratch_created(path):
|
||||
value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value)
|
||||
|
||||
|
||||
def scratch_child(path, pid):
|
||||
import psutil
|
||||
value=json.loads(Path(path).read_text())
|
||||
value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()})
|
||||
atomic(path,value)
|
||||
Reference in New Issue
Block a user