Files
quality_analyse/deploy/macos/operations.py
T

280 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Independent capacity monitoring, registered-test cleanup and log retention."""
import argparse
import fcntl
from datetime import datetime
import hashlib
import json
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import plistlib
import shutil
import subprocess
import sys
import time
from zoneinfo import ZoneInfo
LABEL = 'com.lyunlab.quality-center-operations'
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
GIB = 1024**3
DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB,
'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB,
'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600,
'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2,
'testMaxAgeSeconds': 86400}
def allocated(path):
path = Path(path)
if not path.exists() or path.is_symlink(): return 0
result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45)
lines = result.stdout.splitlines()
if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete')
return int(lines[-1].split()[0]) * 1024
def snapshot(root, support=SUPPORT, now=None):
import managed_test_resources as resources
import log_retention
root = Path(root).resolve(); now = time.time() if now is None else now
mysql = allocated(root / '.runtime/mysql-data')
archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups')
runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output')
logs = log_retention.candidates(root, support)
log_bytes = sum(p.stat().st_blocks * 512 for p in logs)
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
native = allocated(root / '.runtime/nao-users')
native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users'))
registered = 0; overdue = 0
for p in resources.registry(root).glob('*.json'):
value = resources.load(root, p.stem)
if not value or not value.get('created'): continue
if now >= value.get('expiresAt', now + 1): overdue += 1
registered += allocated(root / '.runtime/mysql-data' / value['database'])
private = 0
for p in (root / '.runtime/retention').glob('verify-*'):
if p.is_symlink(): continue
try: owner = json.loads((p / 'owner.json').read_text())
except (OSError, ValueError): continue
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
# Core service data is the same scope as the earlier 11.86GiB measurement.
# Software/build/delivery artifacts are visible separately and included in growth checks.
native_data = max(0, native - native_logs)
core = max(0, mysql - registered) + archives + backups + log_bytes + native_data
artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs)
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered,
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
def conditions(sample, policy, maintenance, history):
alerts = {}
if sample['coreBytes'] >= policy['criticalBytes']:
alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"}
elif sample['coreBytes'] >= policy['warningBytes']:
alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"}
for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)),
('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]:
if size >= policy['warningBytes']:
alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning',
'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'}
if sample['freeBytes'] < policy['freeCriticalBytes']:
alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"}
elif sample['freeBytes'] < policy['freeWarningBytes']:
alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"}
if maintenance.get('consecutiveFailures', 0) >= 3:
alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'}
if sample['overdueTestResources']:
alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"}
old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600]
if old:
baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400))
changes = []
for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]:
before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes'])
after = sample.get(key, sample['permanentBytes'] - sample['coreBytes'])
delta = after - before
if delta > max(policy['growthBytes'], before * policy['growthFraction']):
changes.append(f'{label}增长{delta/GIB:.1f}GiB')
if changes:
alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'}
return alerts
def notify(title, message):
script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n'
try:
p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True,
capture_output=True, timeout=10)
return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode}
except (OSError, subprocess.TimeoutExpired):
return {'submitted': False, 'channel': 'macOS'}
def dispatch(alerts, previous, now, policy, notifier=notify):
updated = {}; events = []
for key, value in alerts.items():
prior = previous.get(key, {})
due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or
now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds'])
entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)}
if due:
result = notifier('质量平台容量告警', value['message'])
entry['notification'] = result
# Failed submission retries on the next check; duplicates are limited after success.
if result.get('submitted'): entry['lastSubmittedAt'] = now
events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result})
updated[key] = entry
for key in set(previous) - set(alerts):
events.append({'kind': 'resolved', 'key': key, 'at': now})
return updated, events
def janitor(root, support, policy):
import backup_retention as br
import backup_verifier
import managed_test_resources as resources
import log_retention
from retention import Database, Busy
root = Path(root).resolve()
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
try:
with br.lock(root, timeout=0):
result = br.clean_incomplete(root)
db = Database(root)
try:
config = json.loads((root / '.runtime/retention/config.json').read_text())
accepted = br.accepted_backend(root)
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
return {**result, 'deferred': 'Application version changed'}
db.current()
if db.busy(): return {**result, 'deferred': 'Application jobs are active'}
# Use a short-lived CLI connection so no observer holds a test schema open.
import backup
result.update(resources.cleanup(root, backup.client))
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
result.update(log_retention.retain(root, support,
max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds']))
return result
finally: db.conn.close()
except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'}
def run(root, support=SUPPORT, notifier=notify):
import backup_retention as br
root = Path(root).resolve(); base = root / '.runtime/operations'
if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path')
base.mkdir(parents=True, exist_ok=True, mode=0o700)
policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())}
state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {}
history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else []
now = time.time(); sample = snapshot(root, support, now)
maintenance_path = root / '.runtime/retention/state.json'
maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {}
alerts = conditions(sample, policy, maintenance, history)
cleanup = {}; failures = previous.get('cleanupFailures', 0)
try:
cleanup = janitor(root, support, policy)
if 'deferred' not in cleanup: failures = 0
except Exception as error:
failures += 1; cleanup = {'errorType': type(error).__name__}
if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'}
active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier)
state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active}
br.atomic(state_path, state)
(base / 'failure.json').unlink(missing_ok=True)
history = [v for v in history if now - v['at'] <= 27 * 3600]
history.append(sample); br.atomic(history_path, history[-330:])
daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {}
day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample
br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]})
if events:
handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2)
handler.setFormatter(logging.Formatter('%(message)s'))
logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False
try:
for event in events: logger.info(json.dumps(event, ensure_ascii=False))
finally: logger.removeHandler(handler); handler.close()
return state
def failed_run(root, error, notifier=notify):
import backup_retention as br
base = Path(root).resolve() / '.runtime/operations'
if base.is_symlink() or base.parent.is_symlink(): return
path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {}
now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__})
if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']:
result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。')
value['notification'] = result
if result.get('submitted'): value['lastSubmittedAt'] = now
br.atomic(path, value)
def install(root):
import backup_retention as br
root = Path(root).resolve(); base = root / '.runtime/operations'
base.mkdir(parents=True, exist_ok=True, mode=0o700)
policy_path = base / 'policy.json'
policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})}
br.atomic(policy_path, policy)
files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in
('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')]
digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700)
for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600)
log_config = root / 'deploy/macos/logback-quality.xml'
shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600)
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700)
value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)],
'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background',
'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')}
domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True)
with plist.open('wb') as f: plistlib.dump(value, f)
plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True)
for _ in range(50):
p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True)
if p.returncode == 0: break
time.sleep(.2)
else: raise RuntimeError('Operations job registration did not complete')
print('Installed independent five-minute capacity and housekeeping checks')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True)
args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts'))
import backup_retention as br
if args.action == 'install':
with br.lock(root, timeout=0): install(root)
elif args.action == 'status':
print((root / '.runtime/operations/monitor.json').read_text())
elif args.action == 'notify-test':
print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False))
else:
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True)
handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2)
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO)
with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock:
fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
state = run(root)
logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s',
state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB,
state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts']))
except Exception as error:
logging.error('operations failed safely: %s', type(error).__name__)
failed_run(root, error); raise SystemExit(1)
if __name__ == '__main__': main()