Files
quality_analyse/deploy/aliyun/quality-tunnel-health.py
T

77 lines
2.7 KiB
Python

#!/usr/bin/env python3
"""只回收质量平台回连端口上连续无响应的、身份已核实的 SSH 子连接。"""
import json
import os
from pathlib import Path
import re
import signal
import subprocess
from urllib.error import HTTPError
from urllib.request import ProxyHandler, build_opener
PORT = 15281
STATE = Path('/run/quality-center-tunnel-health/failure.json')
def reachable():
try:
with build_opener(ProxyHandler({})).open(f'http://127.0.0.1:{PORT}/actuator/health', timeout=4):
return True
except HTTPError:
# 即使后端正在维护或拒绝请求,收到 HTTP 响应也证明隧道仍可通信。
return True
except OSError:
return False
def owner():
result = subprocess.run(['ss', '-H', '-ltnp', f'sport = :{PORT}'], capture_output=True, text=True, check=True)
if f'127.0.0.1:{PORT}' not in result.stdout:
return None
pids = set(re.findall(r'\bpid=(\d+),', result.stdout))
if len(pids) != 1:
return None
pid = int(pids.pop())
command = subprocess.run(['ps', '-p', str(pid), '-o', 'args='], capture_output=True, text=True, check=True).stdout.strip()
if command != 'sshd: admin':
return None
# 不结束带有远程命令或交互终端的 SSH 连接,也不结束 SSH 主服务。
children = subprocess.run(['pgrep', '-P', str(pid)], capture_output=True, text=True)
if children.returncode != 1:
return None
stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
return {'pid': pid, 'started': stat[19]}
def check_once(state_path=STATE, probe=reachable, identify=owner, terminate=os.kill):
if probe():
state_path.unlink(missing_ok=True)
return 'reachable'
current = identify()
if current is None:
state_path.unlink(missing_ok=True)
return 'no_verified_tunnel'
try:
previous = json.loads(state_path.read_text())
except (OSError, ValueError):
previous = None
if previous != current:
state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
state_path.write_text(json.dumps(current))
return 'first_failure'
# 连续两次检查失败后,再核实端口归属与进程启动标识,防止误杀复用 PID。
if identify() != current:
state_path.unlink(missing_ok=True)
return 'owner_changed'
terminate(current['pid'], signal.SIGTERM)
state_path.unlink(missing_ok=True)
return 'stale_tunnel_released'
if __name__ == '__main__':
try:
print(json.dumps({'action': check_once()}), flush=True)
except (OSError, subprocess.SubprocessError) as error:
print(json.dumps({'action': 'check_failed', 'error': type(error).__name__}), flush=True)
raise SystemExit(1)