#!/usr/bin/env python3 """Independent capacity monitoring, registered-test cleanup and log retention.""" import argparse import fcntl from datetime import datetime import hashlib import json import logging from logging.handlers import RotatingFileHandler import os from pathlib import Path import plistlib import shutil import subprocess import sys import time from zoneinfo import ZoneInfo LABEL = 'com.lyunlab.quality-center-operations' SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' GIB = 1024**3 DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB, 'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB, 'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600, 'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2, 'testMaxAgeSeconds': 86400} def allocated(path): path = Path(path) if not path.exists() or path.is_symlink(): return 0 result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45) lines = result.stdout.splitlines() if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete') return int(lines[-1].split()[0]) * 1024 def snapshot(root, support=SUPPORT, now=None): import managed_test_resources as resources import log_retention root = Path(root).resolve(); now = time.time() if now is None else now mysql = allocated(root / '.runtime/mysql-data') archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups') runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output') logs = log_retention.candidates(root, support) log_bytes = sum(p.stat().st_blocks * 512 for p in logs) output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output')) runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime')) native = allocated(root / '.runtime/nao-users') registered = 0; overdue = 0 for p in resources.registry(root).glob('*.json'): value = resources.load(root, p.stem) if not value or not value.get('created'): continue if now >= value.get('expiresAt', now + 1): overdue += 1 registered += allocated(root / '.runtime/mysql-data' / value['database']) private = 0 for p in (root / '.runtime/retention').glob('verify-*'): if p.is_symlink(): continue try: owner = json.loads((p / 'owner.json').read_text()) except (OSError, ValueError): continue if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p) # Core service data is the same scope as the earlier 11.86GiB measurement. # Software/build/delivery artifacts are visible separately and included in growth checks. core = max(0, mysql - registered) + archives + backups + log_bytes + native artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs) return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups, 'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private, 'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts, 'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue} def conditions(sample, policy, maintenance, history): alerts = {} if sample['coreBytes'] >= policy['criticalBytes']: alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"} elif sample['coreBytes'] >= policy['warningBytes']: alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"} for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)), ('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]: if size >= policy['warningBytes']: alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning', 'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'} if sample['freeBytes'] < policy['freeCriticalBytes']: alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"} elif sample['freeBytes'] < policy['freeWarningBytes']: alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"} if maintenance.get('consecutiveFailures', 0) >= 3: alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'} if sample['overdueTestResources']: alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"} old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600] if old: baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400)) changes = [] for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]: before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes']) after = sample.get(key, sample['permanentBytes'] - sample['coreBytes']) delta = after - before if delta > max(policy['growthBytes'], before * policy['growthFraction']): changes.append(f'{label}增长{delta/GIB:.1f}GiB') if changes: alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'} return alerts def notify(title, message): script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n' try: p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True, capture_output=True, timeout=10) return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode} except (OSError, subprocess.TimeoutExpired): return {'submitted': False, 'channel': 'macOS'} def dispatch(alerts, previous, now, policy, notifier=notify): updated = {}; events = [] for key, value in alerts.items(): prior = previous.get(key, {}) due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds']) entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)} if due: result = notifier('质量平台容量告警', value['message']) entry['notification'] = result # Failed submission retries on the next check; duplicates are limited after success. if result.get('submitted'): entry['lastSubmittedAt'] = now events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result}) updated[key] = entry for key in set(previous) - set(alerts): events.append({'kind': 'resolved', 'key': key, 'at': now}) return updated, events def janitor(root, support, policy): import backup_retention as br import backup_verifier import managed_test_resources as resources import log_retention from retention import Database, Busy root = Path(root).resolve() if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'} try: with br.lock(root, timeout=0): db = Database(root) try: config = json.loads((root / '.runtime/retention/config.json').read_text()) accepted = br.accepted_backend(root) if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]: return {'deferred': 'Application version changed'} db.current() if db.busy(): return {'deferred': 'Application jobs are active'} # Use a short-lived CLI connection so no observer holds a test schema open. import backup result = resources.cleanup(root, backup.client) result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root) result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br) result.update(log_retention.retain(root, support, max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds'])) return result finally: db.conn.close() except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'} def run(root, support=SUPPORT, notifier=notify): import backup_retention as br root = Path(root).resolve(); base = root / '.runtime/operations' if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path') base.mkdir(parents=True, exist_ok=True, mode=0o700) policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())} state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {} history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else [] now = time.time(); sample = snapshot(root, support, now) maintenance_path = root / '.runtime/retention/state.json' maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {} alerts = conditions(sample, policy, maintenance, history) cleanup = {}; failures = previous.get('cleanupFailures', 0) try: cleanup = janitor(root, support, policy) if 'deferred' not in cleanup: failures = 0 except Exception as error: failures += 1; cleanup = {'errorType': type(error).__name__} if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'} active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier) state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active} br.atomic(state_path, state) (base / 'failure.json').unlink(missing_ok=True) history = [v for v in history if now - v['at'] <= 27 * 3600] history.append(sample); br.atomic(history_path, history[-330:]) daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {} day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]}) if events: handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2) handler.setFormatter(logging.Formatter('%(message)s')) logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False try: for event in events: logger.info(json.dumps(event, ensure_ascii=False)) finally: logger.removeHandler(handler); handler.close() return state def failed_run(root, error, notifier=notify): import backup_retention as br base = Path(root).resolve() / '.runtime/operations' if base.is_symlink() or base.parent.is_symlink(): return path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {} now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__}) if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']: result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。') value['notification'] = result if result.get('submitted'): value['lastSubmittedAt'] = now br.atomic(path, value) def install(root): import backup_retention as br root = Path(root).resolve(); base = root / '.runtime/operations' base.mkdir(parents=True, exist_ok=True, mode=0o700) policy_path = base / 'policy.json' policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})} br.atomic(policy_path, policy) files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in ('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')] digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest() version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700) for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600) log_config = root / 'deploy/macos/logback-quality.xml' shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600) logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700) value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)], 'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background', 'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'}, 'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')} domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist') subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True) with plist.open('wb') as f: plistlib.dump(value, f) plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True) for _ in range(50): p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True) if p.returncode == 0: break time.sleep(.2) else: raise RuntimeError('Operations job registration did not complete') print('Installed independent five-minute capacity and housekeeping checks') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True) args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts')) import backup_retention as br if args.action == 'install': with br.lock(root, timeout=0): install(root) elif args.action == 'status': print((root / '.runtime/operations/monitor.json').read_text()) elif args.action == 'notify-test': print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False)) else: logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True) handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2) handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO) with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock: fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) try: state = run(root) logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s', state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB, state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts'])) except Exception as error: logging.error('operations failed safely: %s', type(error).__name__) failed_run(root, error); raise SystemExit(1) if __name__ == '__main__': main()