98 lines
4.6 KiB
Python
98 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""与后端共享 PID 命名空间的低优先级维护任务;不向容器暴露 Docker socket。"""
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import sys
|
|
import time
|
|
import shutil
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
|
|
import backup_retention as br
|
|
import operations
|
|
import retention
|
|
|
|
|
|
def console_notify(title, message):
|
|
print(json.dumps({'event': 'capacity-alert', 'title': title, 'message': message}, ensure_ascii=False), flush=True)
|
|
return {'submitted': True, 'channel': 'container-log'}
|
|
|
|
|
|
def snapshot(root, support=None, now=None):
|
|
import backup
|
|
import managed_test_resources as resources
|
|
import log_retention
|
|
root = Path(root); now = time.time() if now is None else now
|
|
cfg = json.loads((root / '.runtime/local-config.json').read_text())
|
|
rows = backup.client('SELECT TABLE_SCHEMA,COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) FROM information_schema.tables GROUP BY TABLE_SCHEMA;')
|
|
estimates = {parts[0]: int(parts[1]) for line in rows.splitlines() if len(parts := line.split('\t')) == 2}
|
|
registered = 0; overdue = 0
|
|
for path in resources.registry(root).glob('*.json'):
|
|
item = resources.load(root, path.stem)
|
|
if not item or not item.get('created'): continue
|
|
registered += estimates.get(item['database'], 0)
|
|
if now >= item.get('expiresAt', now + 1): overdue += 1
|
|
private = sum(operations.allocated(p) for p in (root / '.runtime/retention').glob('verify-*') if p.is_dir() and not p.is_symlink())
|
|
mysql = estimates.get(cfg['mysqlDatabase'], 0)
|
|
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
|
|
native = operations.allocated(root / '.runtime/nao-users')
|
|
log_paths = log_retention.candidates(root, root / '.runtime/operations')
|
|
logs = sum(p.stat().st_blocks * 512 for p in log_paths)
|
|
native_logs = sum(p.stat().st_blocks * 512 for p in log_paths if p.is_relative_to(root / '.runtime/nao-users'))
|
|
core = mysql + archive + backups + max(0, native - native_logs) + logs
|
|
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
|
|
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
|
|
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': max(0, native - native_logs),
|
|
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
|
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
|
|
'overdueTestResources': overdue}
|
|
|
|
|
|
def once(role):
|
|
# 数据库不可连接时也要能释放中断备份占用,不能放在采样或健康检查之后。
|
|
if not (ROOT / '.runtime/retention/paused').exists():
|
|
try:
|
|
result = br.clean_incomplete(ROOT)
|
|
if result['incompleteBackupsRemoved']:
|
|
print(json.dumps({'event': 'incomplete-backup-cleanup', **result}), flush=True)
|
|
except br.Busy: pass
|
|
if role == 'monitor':
|
|
operations.snapshot = snapshot
|
|
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
|
|
return retention.run(ROOT)
|
|
|
|
|
|
def main():
|
|
role = sys.argv[1]
|
|
if role not in ('retention', 'monitor'): raise ValueError('Unknown worker role')
|
|
try: os.nice(10)
|
|
except OSError: pass
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
|
stopping = False
|
|
def stop(signum, frame):
|
|
nonlocal stopping
|
|
stopping = True
|
|
signal.signal(signal.SIGTERM, stop); signal.signal(signal.SIGINT, stop)
|
|
while not stopping:
|
|
try:
|
|
once(role)
|
|
except (br.Busy, retention.Busy): pass
|
|
except Exception as error:
|
|
if role == 'monitor': operations.failed_run(ROOT, error, notifier=console_notify)
|
|
else:
|
|
path = ROOT / '.runtime/retention/state.json'
|
|
state = json.loads(path.read_text()) if path.exists() else {}
|
|
state['consecutiveFailures'] = min(1000, state.get('consecutiveFailures', 0) + 1)
|
|
state['lastError'] = {'time': time.time(), 'type': type(error).__name__}
|
|
br.atomic(path, state)
|
|
print(json.dumps({'event': 'maintenance-error', 'role': role, 'errorType': type(error).__name__}), flush=True)
|
|
for _ in range(300):
|
|
if stopping: break
|
|
time.sleep(1)
|
|
|
|
|
|
if __name__ == '__main__': main()
|