修复:自动回收失效隧道占用并限制首次连接等待时间
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# 质量平台公网连接
|
||||
|
||||
Nginx 将 `/api/` 转发至服务器回环地址 `127.0.0.1:15281`;本机 LaunchAgent 通过 SSH 将该端口回连到本机 `127.0.0.1:5281`。
|
||||
|
||||
## 失效连接恢复
|
||||
|
||||
`quality-tunnel-health.timer` 每轮结束 15 秒后再次检查连接。探测等待最多 4 秒;收到任何 HTTP 响应均视为连接可达,后端维护状态不会触发回收。
|
||||
|
||||
只有连续两次无响应、端口确认为回环端口 15281、拥有者确认为无子进程的 `sshd: admin`,且进程启动标识和端口归属未变化时,才结束该 SSH 子连接。它不结束 SSH 主服务、交互终端或其他项目连接。状态保存在仅 root 可访问的 `/run/quality-center-tunnel-health/`。
|
||||
|
||||
部署文件分别安装到 `/usr/local/sbin/quality-tunnel-health.py` 和 `/etc/systemd/system/quality-tunnel-health.{service,timer}`;启用 timer 即可。检查日志使用 `journalctl -u quality-tunnel-health.service`。停用时只需停用该 timer。
|
||||
|
||||
本机现有 `com.lyunlab.quality-center-tunnel` LaunchAgent 保留已配置的 SSH 别名与密钥;心跳间隔 15 秒、连续失败 2 次退出,重试间隔 10 秒,并保持 `ExitOnForwardFailure=yes` 和主机身份验证。
|
||||
|
||||
前端只为连接初始化、身份读取和页面状态校验设置 8 秒期限;普通数据查询和长时间分析不共用该期限。
|
||||
|
||||
单元测试:`python3 -m unittest discover -s deploy/aliyun -p 'test_*.py'`。
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""只回收质量平台回连端口上连续无响应的、身份已核实的 SSH 子连接。"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import ProxyHandler, build_opener
|
||||
|
||||
PORT = 15281
|
||||
STATE = Path('/run/quality-center-tunnel-health/failure.json')
|
||||
|
||||
|
||||
def reachable():
|
||||
try:
|
||||
with build_opener(ProxyHandler({})).open(f'http://127.0.0.1:{PORT}/actuator/health', timeout=4):
|
||||
return True
|
||||
except HTTPError:
|
||||
# 即使后端正在维护或拒绝请求,收到 HTTP 响应也证明隧道仍可通信。
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def owner():
|
||||
result = subprocess.run(['ss', '-H', '-ltnp', f'sport = :{PORT}'], capture_output=True, text=True, check=True)
|
||||
if f'127.0.0.1:{PORT}' not in result.stdout:
|
||||
return None
|
||||
pids = set(re.findall(r'\bpid=(\d+),', result.stdout))
|
||||
if len(pids) != 1:
|
||||
return None
|
||||
pid = int(pids.pop())
|
||||
command = subprocess.run(['ps', '-p', str(pid), '-o', 'args='], capture_output=True, text=True, check=True).stdout.strip()
|
||||
if command != 'sshd: admin':
|
||||
return None
|
||||
# 不结束带有远程命令或交互终端的 SSH 连接,也不结束 SSH 主服务。
|
||||
children = subprocess.run(['pgrep', '-P', str(pid)], capture_output=True, text=True)
|
||||
if children.returncode != 1:
|
||||
return None
|
||||
stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
|
||||
return {'pid': pid, 'started': stat[19]}
|
||||
|
||||
|
||||
def check_once(state_path=STATE, probe=reachable, identify=owner, terminate=os.kill):
|
||||
if probe():
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'reachable'
|
||||
current = identify()
|
||||
if current is None:
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'no_verified_tunnel'
|
||||
try:
|
||||
previous = json.loads(state_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
previous = None
|
||||
if previous != current:
|
||||
state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
state_path.write_text(json.dumps(current))
|
||||
return 'first_failure'
|
||||
# 连续两次检查失败后,再核实端口归属与进程启动标识,防止误杀复用 PID。
|
||||
if identify() != current:
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'owner_changed'
|
||||
terminate(current['pid'], signal.SIGTERM)
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'stale_tunnel_released'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
print(json.dumps({'action': check_once()}), flush=True)
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
print(json.dumps({'action': 'check_failed', 'error': type(error).__name__}), flush=True)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Check the Quality Center reverse tunnel
|
||||
After=network-online.target ssh.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /usr/local/sbin/quality-tunnel-health.py
|
||||
RuntimeDirectory=quality-center-tunnel-health
|
||||
RuntimeDirectoryMode=0700
|
||||
RuntimeDirectoryPreserve=yes
|
||||
UMask=0077
|
||||
TimeoutStartSec=12
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/run/quality-center-tunnel-health
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Detect unresponsive Quality Center tunnel connections
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15
|
||||
OnUnitInactiveSec=15
|
||||
AccuracySec=1
|
||||
Unit=quality-tunnel-health.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,63 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('guard', Path(__file__).with_name('quality-tunnel-health.py'))
|
||||
guard = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(guard)
|
||||
|
||||
|
||||
class TunnelHealthTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.folder = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.folder.cleanup)
|
||||
self.state = Path(self.folder.name) / 'failure.json'
|
||||
self.process = {'pid': 123, 'started': '456'}
|
||||
self.kill = Mock()
|
||||
|
||||
def check(self, healthy=False, identify=None):
|
||||
return guard.check_once(self.state, lambda: healthy, identify or (lambda: self.process), self.kill)
|
||||
|
||||
def test_two_failures_release_only_the_same_verified_process(self):
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
self.assertEqual(self.check(), 'stale_tunnel_released')
|
||||
self.kill.assert_called_once_with(123, signal.SIGTERM)
|
||||
|
||||
def test_reachable_tunnel_clears_failure_history(self):
|
||||
self.check()
|
||||
self.assertEqual(self.check(healthy=True), 'reachable')
|
||||
self.assertFalse(self.state.exists())
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_unverified_owner_is_never_killed(self):
|
||||
self.check()
|
||||
self.assertEqual(self.check(identify=lambda: None), 'no_verified_tunnel')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_new_connection_or_reused_pid_restarts_confirmation(self):
|
||||
self.check()
|
||||
self.process = {'pid': 123, 'started': '789'}
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_process_change_just_before_termination_is_rejected(self):
|
||||
self.check()
|
||||
identify = Mock(side_effect=[self.process, {'pid': 987, 'started': '789'}])
|
||||
self.assertEqual(self.check(identify=identify), 'owner_changed')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_main_sshd_or_interactive_connection_is_not_a_tunnel_owner(self):
|
||||
for command,children in [('sshd: admin', 0), ('sshd: admin@pts/0', 1), ('sshd: /usr/sbin/sshd -D [listener]', 1)]:
|
||||
with self.subTest(command=command), patch.object(guard.subprocess, 'run') as run:
|
||||
run.side_effect = [Mock(stdout='LISTEN 0 128 127.0.0.1:15281 0.0.0.0:* users:(("sshd",pid=123,fd=7))'),
|
||||
Mock(stdout=command), Mock(returncode=children)]
|
||||
self.assertIsNone(guard.owner())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user