新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent capacity monitoring, registered-test cleanup and log retention."""
|
||||
import argparse
|
||||
import fcntl
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
LABEL = 'com.lyunlab.quality-center-operations'
|
||||
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||
GIB = 1024**3
|
||||
DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB,
|
||||
'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB,
|
||||
'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600,
|
||||
'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2,
|
||||
'testMaxAgeSeconds': 86400}
|
||||
|
||||
|
||||
def allocated(path):
|
||||
path = Path(path)
|
||||
if not path.exists() or path.is_symlink(): return 0
|
||||
result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45)
|
||||
lines = result.stdout.splitlines()
|
||||
if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete')
|
||||
return int(lines[-1].split()[0]) * 1024
|
||||
|
||||
|
||||
def snapshot(root, support=SUPPORT, now=None):
|
||||
import managed_test_resources as resources
|
||||
import log_retention
|
||||
root = Path(root).resolve(); now = time.time() if now is None else now
|
||||
mysql = allocated(root / '.runtime/mysql-data')
|
||||
archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups')
|
||||
runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output')
|
||||
logs = log_retention.candidates(root, support)
|
||||
log_bytes = sum(p.stat().st_blocks * 512 for p in logs)
|
||||
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
|
||||
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
|
||||
native = allocated(root / '.runtime/nao-users')
|
||||
registered = 0; overdue = 0
|
||||
for p in resources.registry(root).glob('*.json'):
|
||||
value = resources.load(root, p.stem)
|
||||
if not value or not value.get('created'): continue
|
||||
if now >= value.get('expiresAt', now + 1): overdue += 1
|
||||
registered += allocated(root / '.runtime/mysql-data' / value['database'])
|
||||
private = 0
|
||||
for p in (root / '.runtime/retention').glob('verify-*'):
|
||||
if p.is_symlink(): continue
|
||||
try: owner = json.loads((p / 'owner.json').read_text())
|
||||
except (OSError, ValueError): continue
|
||||
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
|
||||
# Core service data is the same scope as the earlier 11.86GiB measurement.
|
||||
# Software/build/delivery artifacts are visible separately and included in growth checks.
|
||||
core = max(0, mysql - registered) + archives + backups + log_bytes + native
|
||||
artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs)
|
||||
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
|
||||
'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered,
|
||||
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||||
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
|
||||
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
|
||||
|
||||
|
||||
def conditions(sample, policy, maintenance, history):
|
||||
alerts = {}
|
||||
if sample['coreBytes'] >= policy['criticalBytes']:
|
||||
alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"}
|
||||
elif sample['coreBytes'] >= policy['warningBytes']:
|
||||
alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"}
|
||||
for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)),
|
||||
('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]:
|
||||
if size >= policy['warningBytes']:
|
||||
alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning',
|
||||
'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'}
|
||||
if sample['freeBytes'] < policy['freeCriticalBytes']:
|
||||
alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"}
|
||||
elif sample['freeBytes'] < policy['freeWarningBytes']:
|
||||
alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"}
|
||||
if maintenance.get('consecutiveFailures', 0) >= 3:
|
||||
alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'}
|
||||
if sample['overdueTestResources']:
|
||||
alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"}
|
||||
old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600]
|
||||
if old:
|
||||
baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400))
|
||||
changes = []
|
||||
for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]:
|
||||
before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes'])
|
||||
after = sample.get(key, sample['permanentBytes'] - sample['coreBytes'])
|
||||
delta = after - before
|
||||
if delta > max(policy['growthBytes'], before * policy['growthFraction']):
|
||||
changes.append(f'{label}增长{delta/GIB:.1f}GiB')
|
||||
if changes:
|
||||
alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'}
|
||||
return alerts
|
||||
|
||||
|
||||
def notify(title, message):
|
||||
script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n'
|
||||
try:
|
||||
p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True,
|
||||
capture_output=True, timeout=10)
|
||||
return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode}
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return {'submitted': False, 'channel': 'macOS'}
|
||||
|
||||
|
||||
def dispatch(alerts, previous, now, policy, notifier=notify):
|
||||
updated = {}; events = []
|
||||
for key, value in alerts.items():
|
||||
prior = previous.get(key, {})
|
||||
due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or
|
||||
now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds'])
|
||||
entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)}
|
||||
if due:
|
||||
result = notifier('质量平台容量告警', value['message'])
|
||||
entry['notification'] = result
|
||||
# Failed submission retries on the next check; duplicates are limited after success.
|
||||
if result.get('submitted'): entry['lastSubmittedAt'] = now
|
||||
events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result})
|
||||
updated[key] = entry
|
||||
for key in set(previous) - set(alerts):
|
||||
events.append({'kind': 'resolved', 'key': key, 'at': now})
|
||||
return updated, events
|
||||
|
||||
|
||||
def janitor(root, support, policy):
|
||||
import backup_retention as br
|
||||
import backup_verifier
|
||||
import managed_test_resources as resources
|
||||
import log_retention
|
||||
from retention import Database, Busy
|
||||
root = Path(root).resolve()
|
||||
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
|
||||
try:
|
||||
with br.lock(root, timeout=0):
|
||||
db = Database(root)
|
||||
try:
|
||||
config = json.loads((root / '.runtime/retention/config.json').read_text())
|
||||
accepted = br.accepted_backend(root)
|
||||
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
|
||||
return {'deferred': 'Application version changed'}
|
||||
db.current()
|
||||
if db.busy(): return {'deferred': 'Application jobs are active'}
|
||||
# Use a short-lived CLI connection so no observer holds a test schema open.
|
||||
import backup
|
||||
result = resources.cleanup(root, backup.client)
|
||||
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
|
||||
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
|
||||
result.update(log_retention.retain(root, support,
|
||||
max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds']))
|
||||
return result
|
||||
finally: db.conn.close()
|
||||
except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'}
|
||||
|
||||
|
||||
def run(root, support=SUPPORT, notifier=notify):
|
||||
import backup_retention as br
|
||||
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||||
if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path')
|
||||
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())}
|
||||
state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else []
|
||||
now = time.time(); sample = snapshot(root, support, now)
|
||||
maintenance_path = root / '.runtime/retention/state.json'
|
||||
maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {}
|
||||
alerts = conditions(sample, policy, maintenance, history)
|
||||
cleanup = {}; failures = previous.get('cleanupFailures', 0)
|
||||
try:
|
||||
cleanup = janitor(root, support, policy)
|
||||
if 'deferred' not in cleanup: failures = 0
|
||||
except Exception as error:
|
||||
failures += 1; cleanup = {'errorType': type(error).__name__}
|
||||
if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'}
|
||||
active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier)
|
||||
state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active}
|
||||
br.atomic(state_path, state)
|
||||
(base / 'failure.json').unlink(missing_ok=True)
|
||||
history = [v for v in history if now - v['at'] <= 27 * 3600]
|
||||
history.append(sample); br.atomic(history_path, history[-330:])
|
||||
daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {}
|
||||
day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample
|
||||
br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]})
|
||||
if events:
|
||||
handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2)
|
||||
handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False
|
||||
try:
|
||||
for event in events: logger.info(json.dumps(event, ensure_ascii=False))
|
||||
finally: logger.removeHandler(handler); handler.close()
|
||||
return state
|
||||
|
||||
|
||||
def failed_run(root, error, notifier=notify):
|
||||
import backup_retention as br
|
||||
base = Path(root).resolve() / '.runtime/operations'
|
||||
if base.is_symlink() or base.parent.is_symlink(): return
|
||||
path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {}
|
||||
now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__})
|
||||
if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']:
|
||||
result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。')
|
||||
value['notification'] = result
|
||||
if result.get('submitted'): value['lastSubmittedAt'] = now
|
||||
br.atomic(path, value)
|
||||
|
||||
|
||||
def install(root):
|
||||
import backup_retention as br
|
||||
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||||
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
policy_path = base / 'policy.json'
|
||||
policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})}
|
||||
br.atomic(policy_path, policy)
|
||||
files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in
|
||||
('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')]
|
||||
digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
|
||||
version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600)
|
||||
log_config = root / 'deploy/macos/logback-quality.xml'
|
||||
shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600)
|
||||
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)],
|
||||
'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background',
|
||||
'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
||||
'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')}
|
||||
domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
||||
subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True)
|
||||
with plist.open('wb') as f: plistlib.dump(value, f)
|
||||
plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True)
|
||||
for _ in range(50):
|
||||
p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True)
|
||||
if p.returncode == 0: break
|
||||
time.sleep(.2)
|
||||
else: raise RuntimeError('Operations job registration did not complete')
|
||||
print('Installed independent five-minute capacity and housekeeping checks')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True)
|
||||
args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts'))
|
||||
import backup_retention as br
|
||||
if args.action == 'install':
|
||||
with br.lock(root, timeout=0): install(root)
|
||||
elif args.action == 'status':
|
||||
print((root / '.runtime/operations/monitor.json').read_text())
|
||||
elif args.action == 'notify-test':
|
||||
print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False))
|
||||
else:
|
||||
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True)
|
||||
handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2)
|
||||
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||
logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO)
|
||||
with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock:
|
||||
fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
state = run(root)
|
||||
logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s',
|
||||
state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB,
|
||||
state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts']))
|
||||
except Exception as error:
|
||||
logging.error('operations failed safely: %s', type(error).__name__)
|
||||
failed_run(root, error); raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
Reference in New Issue
Block a user