280 lines
17 KiB
Python
280 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""Independent capacity monitoring, registered-test cleanup and log retention."""
|
||
import argparse
|
||
import fcntl
|
||
from datetime import datetime
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
from logging.handlers import RotatingFileHandler
|
||
import os
|
||
from pathlib import Path
|
||
import plistlib
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from zoneinfo import ZoneInfo
|
||
|
||
LABEL = 'com.lyunlab.quality-center-operations'
|
||
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||
GIB = 1024**3
|
||
DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB,
|
||
'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB,
|
||
'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600,
|
||
'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2,
|
||
'testMaxAgeSeconds': 86400}
|
||
|
||
|
||
def allocated(path):
|
||
path = Path(path)
|
||
if not path.exists() or path.is_symlink(): return 0
|
||
result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45)
|
||
lines = result.stdout.splitlines()
|
||
if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete')
|
||
return int(lines[-1].split()[0]) * 1024
|
||
|
||
|
||
def snapshot(root, support=SUPPORT, now=None):
|
||
import managed_test_resources as resources
|
||
import log_retention
|
||
root = Path(root).resolve(); now = time.time() if now is None else now
|
||
mysql = allocated(root / '.runtime/mysql-data')
|
||
archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups')
|
||
runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output')
|
||
logs = log_retention.candidates(root, support)
|
||
log_bytes = sum(p.stat().st_blocks * 512 for p in logs)
|
||
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
|
||
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
|
||
native = allocated(root / '.runtime/nao-users')
|
||
native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users'))
|
||
registered = 0; overdue = 0
|
||
for p in resources.registry(root).glob('*.json'):
|
||
value = resources.load(root, p.stem)
|
||
if not value or not value.get('created'): continue
|
||
if now >= value.get('expiresAt', now + 1): overdue += 1
|
||
registered += allocated(root / '.runtime/mysql-data' / value['database'])
|
||
private = 0
|
||
for p in (root / '.runtime/retention').glob('verify-*'):
|
||
if p.is_symlink(): continue
|
||
try: owner = json.loads((p / 'owner.json').read_text())
|
||
except (OSError, ValueError): continue
|
||
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
|
||
# Core service data is the same scope as the earlier 11.86GiB measurement.
|
||
# Software/build/delivery artifacts are visible separately and included in growth checks.
|
||
native_data = max(0, native - native_logs)
|
||
core = max(0, mysql - registered) + archives + backups + log_bytes + native_data
|
||
artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs)
|
||
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
|
||
'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered,
|
||
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
|
||
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
|
||
|
||
|
||
def conditions(sample, policy, maintenance, history):
|
||
alerts = {}
|
||
if sample['coreBytes'] >= policy['criticalBytes']:
|
||
alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"}
|
||
elif sample['coreBytes'] >= policy['warningBytes']:
|
||
alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"}
|
||
for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)),
|
||
('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]:
|
||
if size >= policy['warningBytes']:
|
||
alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning',
|
||
'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'}
|
||
if sample['freeBytes'] < policy['freeCriticalBytes']:
|
||
alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"}
|
||
elif sample['freeBytes'] < policy['freeWarningBytes']:
|
||
alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"}
|
||
if maintenance.get('consecutiveFailures', 0) >= 3:
|
||
alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'}
|
||
if sample['overdueTestResources']:
|
||
alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"}
|
||
old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600]
|
||
if old:
|
||
baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400))
|
||
changes = []
|
||
for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]:
|
||
before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes'])
|
||
after = sample.get(key, sample['permanentBytes'] - sample['coreBytes'])
|
||
delta = after - before
|
||
if delta > max(policy['growthBytes'], before * policy['growthFraction']):
|
||
changes.append(f'{label}增长{delta/GIB:.1f}GiB')
|
||
if changes:
|
||
alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'}
|
||
return alerts
|
||
|
||
|
||
def notify(title, message):
|
||
script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n'
|
||
try:
|
||
p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True,
|
||
capture_output=True, timeout=10)
|
||
return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode}
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
return {'submitted': False, 'channel': 'macOS'}
|
||
|
||
|
||
def dispatch(alerts, previous, now, policy, notifier=notify):
|
||
updated = {}; events = []
|
||
for key, value in alerts.items():
|
||
prior = previous.get(key, {})
|
||
due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or
|
||
now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds'])
|
||
entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)}
|
||
if due:
|
||
result = notifier('质量平台容量告警', value['message'])
|
||
entry['notification'] = result
|
||
# Failed submission retries on the next check; duplicates are limited after success.
|
||
if result.get('submitted'): entry['lastSubmittedAt'] = now
|
||
events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result})
|
||
updated[key] = entry
|
||
for key in set(previous) - set(alerts):
|
||
events.append({'kind': 'resolved', 'key': key, 'at': now})
|
||
return updated, events
|
||
|
||
|
||
def janitor(root, support, policy):
|
||
import backup_retention as br
|
||
import backup_verifier
|
||
import managed_test_resources as resources
|
||
import log_retention
|
||
from retention import Database, Busy
|
||
root = Path(root).resolve()
|
||
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
|
||
try:
|
||
with br.lock(root, timeout=0):
|
||
result = br.clean_incomplete(root)
|
||
db = Database(root)
|
||
try:
|
||
config = json.loads((root / '.runtime/retention/config.json').read_text())
|
||
accepted = br.accepted_backend(root)
|
||
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
|
||
return {**result, 'deferred': 'Application version changed'}
|
||
db.current()
|
||
if db.busy(): return {**result, 'deferred': 'Application jobs are active'}
|
||
# Use a short-lived CLI connection so no observer holds a test schema open.
|
||
import backup
|
||
result.update(resources.cleanup(root, backup.client))
|
||
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
|
||
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
|
||
result.update(log_retention.retain(root, support,
|
||
max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds']))
|
||
return result
|
||
finally: db.conn.close()
|
||
except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'}
|
||
|
||
|
||
def run(root, support=SUPPORT, notifier=notify):
|
||
import backup_retention as br
|
||
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||
if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path')
|
||
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())}
|
||
state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||
history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else []
|
||
now = time.time(); sample = snapshot(root, support, now)
|
||
maintenance_path = root / '.runtime/retention/state.json'
|
||
maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {}
|
||
alerts = conditions(sample, policy, maintenance, history)
|
||
cleanup = {}; failures = previous.get('cleanupFailures', 0)
|
||
try:
|
||
cleanup = janitor(root, support, policy)
|
||
if 'deferred' not in cleanup: failures = 0
|
||
except Exception as error:
|
||
failures += 1; cleanup = {'errorType': type(error).__name__}
|
||
if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'}
|
||
active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier)
|
||
state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active}
|
||
br.atomic(state_path, state)
|
||
(base / 'failure.json').unlink(missing_ok=True)
|
||
history = [v for v in history if now - v['at'] <= 27 * 3600]
|
||
history.append(sample); br.atomic(history_path, history[-330:])
|
||
daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {}
|
||
day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample
|
||
br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]})
|
||
if events:
|
||
handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2)
|
||
handler.setFormatter(logging.Formatter('%(message)s'))
|
||
logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False
|
||
try:
|
||
for event in events: logger.info(json.dumps(event, ensure_ascii=False))
|
||
finally: logger.removeHandler(handler); handler.close()
|
||
return state
|
||
|
||
|
||
def failed_run(root, error, notifier=notify):
|
||
import backup_retention as br
|
||
base = Path(root).resolve() / '.runtime/operations'
|
||
if base.is_symlink() or base.parent.is_symlink(): return
|
||
path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {}
|
||
now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__})
|
||
if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']:
|
||
result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。')
|
||
value['notification'] = result
|
||
if result.get('submitted'): value['lastSubmittedAt'] = now
|
||
br.atomic(path, value)
|
||
|
||
|
||
def install(root):
|
||
import backup_retention as br
|
||
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
policy_path = base / 'policy.json'
|
||
policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})}
|
||
br.atomic(policy_path, policy)
|
||
files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in
|
||
('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')]
|
||
digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
|
||
version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600)
|
||
log_config = root / 'deploy/macos/logback-quality.xml'
|
||
shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600)
|
||
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)],
|
||
'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background',
|
||
'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
||
'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')}
|
||
domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
||
subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True)
|
||
with plist.open('wb') as f: plistlib.dump(value, f)
|
||
plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True)
|
||
for _ in range(50):
|
||
p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True)
|
||
if p.returncode == 0: break
|
||
time.sleep(.2)
|
||
else: raise RuntimeError('Operations job registration did not complete')
|
||
print('Installed independent five-minute capacity and housekeeping checks')
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True)
|
||
args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts'))
|
||
import backup_retention as br
|
||
if args.action == 'install':
|
||
with br.lock(root, timeout=0): install(root)
|
||
elif args.action == 'status':
|
||
print((root / '.runtime/operations/monitor.json').read_text())
|
||
elif args.action == 'notify-test':
|
||
print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False))
|
||
else:
|
||
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True)
|
||
handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2)
|
||
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||
logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO)
|
||
with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock:
|
||
fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
try:
|
||
state = run(root)
|
||
logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s',
|
||
state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB,
|
||
state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts']))
|
||
except Exception as error:
|
||
logging.error('operations failed safely: %s', type(error).__name__)
|
||
failed_run(root, error); raise SystemExit(1)
|
||
|
||
|
||
if __name__ == '__main__': main()
|