修复:自动回收失效隧道占用并限制首次连接等待时间

This commit is contained in:
2026-10-23 17:47:29 +08:00
parent 3519dd0891
commit 8684ce1a4d
8 changed files with 216 additions and 13 deletions
+17
View File
@@ -0,0 +1,17 @@
# 质量平台公网连接
Nginx 将 `/api/` 转发至服务器回环地址 `127.0.0.1:15281`;本机 LaunchAgent 通过 SSH 将该端口回连到本机 `127.0.0.1:5281`。
## 失效连接恢复
`quality-tunnel-health.timer` 每轮结束 15 秒后再次检查连接。探测等待最多 4 秒;收到任何 HTTP 响应均视为连接可达,后端维护状态不会触发回收。
只有连续两次无响应、端口确认为回环端口 15281、拥有者确认为无子进程的 `sshd: admin`,且进程启动标识和端口归属未变化时,才结束该 SSH 子连接。它不结束 SSH 主服务、交互终端或其他项目连接。状态保存在仅 root 可访问的 `/run/quality-center-tunnel-health/`。
部署文件分别安装到 `/usr/local/sbin/quality-tunnel-health.py` 和 `/etc/systemd/system/quality-tunnel-health.{service,timer}`;启用 timer 即可。检查日志使用 `journalctl -u quality-tunnel-health.service`。停用时只需停用该 timer。
本机现有 `com.lyunlab.quality-center-tunnel` LaunchAgent 保留已配置的 SSH 别名与密钥;心跳间隔 15 秒、连续失败 2 次退出,重试间隔 10 秒,并保持 `ExitOnForwardFailure=yes` 和主机身份验证。
前端只为连接初始化、身份读取和页面状态校验设置 8 秒期限;普通数据查询和长时间分析不共用该期限。
单元测试:`python3 -m unittest discover -s deploy/aliyun -p 'test_*.py'`。
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""只回收质量平台回连端口上连续无响应的、身份已核实的 SSH 子连接。"""
import json
import os
from pathlib import Path
import re
import signal
import subprocess
from urllib.error import HTTPError
from urllib.request import ProxyHandler, build_opener
PORT = 15281
STATE = Path('/run/quality-center-tunnel-health/failure.json')
def reachable():
try:
with build_opener(ProxyHandler({})).open(f'http://127.0.0.1:{PORT}/actuator/health', timeout=4):
return True
except HTTPError:
# 即使后端正在维护或拒绝请求,收到 HTTP 响应也证明隧道仍可通信。
return True
except OSError:
return False
def owner():
result = subprocess.run(['ss', '-H', '-ltnp', f'sport = :{PORT}'], capture_output=True, text=True, check=True)
if f'127.0.0.1:{PORT}' not in result.stdout:
return None
pids = set(re.findall(r'\bpid=(\d+),', result.stdout))
if len(pids) != 1:
return None
pid = int(pids.pop())
command = subprocess.run(['ps', '-p', str(pid), '-o', 'args='], capture_output=True, text=True, check=True).stdout.strip()
if command != 'sshd: admin':
return None
# 不结束带有远程命令或交互终端的 SSH 连接,也不结束 SSH 主服务。
children = subprocess.run(['pgrep', '-P', str(pid)], capture_output=True, text=True)
if children.returncode != 1:
return None
stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
return {'pid': pid, 'started': stat[19]}
def check_once(state_path=STATE, probe=reachable, identify=owner, terminate=os.kill):
if probe():
state_path.unlink(missing_ok=True)
return 'reachable'
current = identify()
if current is None:
state_path.unlink(missing_ok=True)
return 'no_verified_tunnel'
try:
previous = json.loads(state_path.read_text())
except (OSError, ValueError):
previous = None
if previous != current:
state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
state_path.write_text(json.dumps(current))
return 'first_failure'
# 连续两次检查失败后,再核实端口归属与进程启动标识,防止误杀复用 PID。
if identify() != current:
state_path.unlink(missing_ok=True)
return 'owner_changed'
terminate(current['pid'], signal.SIGTERM)
state_path.unlink(missing_ok=True)
return 'stale_tunnel_released'
if __name__ == '__main__':
try:
print(json.dumps({'action': check_once()}), flush=True)
except (OSError, subprocess.SubprocessError) as error:
print(json.dumps({'action': 'check_failed', 'error': type(error).__name__}), flush=True)
raise SystemExit(1)
@@ -0,0 +1,17 @@
[Unit]
Description=Check the Quality Center reverse tunnel
After=network-online.target ssh.service
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 /usr/local/sbin/quality-tunnel-health.py
RuntimeDirectory=quality-center-tunnel-health
RuntimeDirectoryMode=0700
RuntimeDirectoryPreserve=yes
UMask=0077
TimeoutStartSec=12
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
ReadWritePaths=/run/quality-center-tunnel-health
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Detect unresponsive Quality Center tunnel connections
[Timer]
OnBootSec=15
OnUnitInactiveSec=15
AccuracySec=1
Unit=quality-tunnel-health.service
[Install]
WantedBy=timers.target
@@ -0,0 +1,63 @@
import importlib.util
from pathlib import Path
import signal
import tempfile
import unittest
from unittest.mock import Mock, patch
spec = importlib.util.spec_from_file_location('guard', Path(__file__).with_name('quality-tunnel-health.py'))
guard = importlib.util.module_from_spec(spec)
spec.loader.exec_module(guard)
class TunnelHealthTest(unittest.TestCase):
def setUp(self):
self.folder = tempfile.TemporaryDirectory()
self.addCleanup(self.folder.cleanup)
self.state = Path(self.folder.name) / 'failure.json'
self.process = {'pid': 123, 'started': '456'}
self.kill = Mock()
def check(self, healthy=False, identify=None):
return guard.check_once(self.state, lambda: healthy, identify or (lambda: self.process), self.kill)
def test_two_failures_release_only_the_same_verified_process(self):
self.assertEqual(self.check(), 'first_failure')
self.kill.assert_not_called()
self.assertEqual(self.check(), 'stale_tunnel_released')
self.kill.assert_called_once_with(123, signal.SIGTERM)
def test_reachable_tunnel_clears_failure_history(self):
self.check()
self.assertEqual(self.check(healthy=True), 'reachable')
self.assertFalse(self.state.exists())
self.assertEqual(self.check(), 'first_failure')
self.kill.assert_not_called()
def test_unverified_owner_is_never_killed(self):
self.check()
self.assertEqual(self.check(identify=lambda: None), 'no_verified_tunnel')
self.kill.assert_not_called()
def test_new_connection_or_reused_pid_restarts_confirmation(self):
self.check()
self.process = {'pid': 123, 'started': '789'}
self.assertEqual(self.check(), 'first_failure')
self.kill.assert_not_called()
def test_process_change_just_before_termination_is_rejected(self):
self.check()
identify = Mock(side_effect=[self.process, {'pid': 987, 'started': '789'}])
self.assertEqual(self.check(identify=identify), 'owner_changed')
self.kill.assert_not_called()
def test_main_sshd_or_interactive_connection_is_not_a_tunnel_owner(self):
for command,children in [('sshd: admin', 0), ('sshd: admin@pts/0', 1), ('sshd: /usr/sbin/sshd -D [listener]', 1)]:
with self.subTest(command=command), patch.object(guard.subprocess, 'run') as run:
run.side_effect = [Mock(stdout='LISTEN 0 128 127.0.0.1:15281 0.0.0.0:* users:(("sshd",pid=123,fd=7))'),
Mock(stdout=command), Mock(returncode=children)]
self.assertIsNone(guard.owner())
if __name__ == '__main__':
unittest.main()
+3 -2
View File
@@ -45,6 +45,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
if (revisions.current && state.dataRevision !== revisions.current.dataRevision) setRefreshKey(value => value + 1);
revisions.current = state;
setError('');
setReady(true);
return true;
} catch (value) {
if (alive.current && generation.current === epoch && !onUnauthorized(value)) setError(value instanceof Error ? value.message : '页面状态校验失败,请重试');
@@ -84,7 +85,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
} }));
useEffect(() => {
alive.current = true;
void validate().then(ok => { if (ok) setReady(true); });
void validate();
const invalidate = () => setRefreshKey(value => value + 1);
window.addEventListener(WORKSPACE_INVALIDATED, invalidate);
return () => { alive.current = false; generation.current++; validation.current = null; window.removeEventListener(WORKSPACE_INVALIDATED, invalidate); };
@@ -95,7 +96,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
const frame = requestAnimationFrame(() => window.scrollTo(0, entries.current.get(page)?.scroll || 0));
return () => cancelAnimationFrame(frame);
}, [page, ready, activeEntryKey]);
const connectionNotice = error && <Alert type="warning" showIcon message="页面状态暂未更新" description={error} action={<Button onClick={() => { void validate().then(ok => { if (ok) setReady(true); }); }}>重新连接</Button>} />;
const connectionNotice = error && <Alert type="warning" showIcon message="页面状态暂未更新" description={error} action={<Button onClick={() => { void validate(); }}>重新连接</Button>} />;
if (!ready) return connectionNotice || <div className="loading-panel"><Spin />正在打开工作台…</div>;
return <>{connectionNotice}{Array.from(entries.current.entries()).filter(([name]) => retained(name) || name === page).map(([name, entry]) => {
const active = name === page;
+14 -10
View File
@@ -1,6 +1,7 @@
import { WORKSPACE_INVALIDATED } from './pageCache';
import { consumeEventStream, type StreamEvent } from './eventStream';
import { publicErrorMessage } from './publicDataDisplay';
import { RequestTimeoutError, withRequestDeadline } from './requestDeadline';
import type {
AuthUser,
FilterField,
@@ -65,16 +66,19 @@ async function readResponse<T>(response: Response): Promise<T> {
return payload as T;
}
async function request<T>(path: string, init: RequestInit = {}): Promise<T> {
async function request<T>(path: string, init: RequestInit = {}, timeoutMs?: number): Promise<T> {
const headers = new Headers(init.headers);
if (init.body && !(init.body instanceof FormData) && !headers.has('Content-Type')) headers.set('Content-Type', 'application/json');
const response = await fetchApi(path, {
...init,
headers,
credentials: 'same-origin',
cache: 'no-store',
});
const send = async (signal = init.signal) => {
const response = await fetchApi(path, { ...init, signal, headers, credentials: 'same-origin', cache: 'no-store' });
return readResponse<T>(response);
};
if (!timeoutMs) return send();
try { return await withRequestDeadline(signal => send(signal), timeoutMs); }
catch (error) {
if (error instanceof RequestTimeoutError) throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
throw error;
}
}
async function requestBlob(path: string, signal?: AbortSignal): Promise<{ blob: Blob; filename: string }> {
@@ -113,7 +117,7 @@ async function readEventStream(path: string, options: { signal?: AbortSignal; la
}
export async function refreshCsrf(): Promise<string> {
const result = await request<{ token: string }>('/api/auth/csrf');
const result = await request<{ token: string }>('/api/auth/csrf', {}, 8000);
csrfToken = result.token;
return csrfToken;
}
@@ -143,14 +147,14 @@ async function mutateForm<T>(path: string, body: FormData): Promise<T> {
}
export const api = {
workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state'); },
workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state', {}, 8000); },
filterFields(manage = false) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`); },
filterFieldOptions(key: string, search = '') { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search})}`); },
filterFieldColumns() { return request<{ black: string[]; bright: string[] }>('/api/filter-fields/columns'); },
saveFilterFields(fields: FilterField[]) { return mutate<FilterField[]>('/api/filter-fields', fields, 'PUT'); },
refreshCsrf,
async me(): Promise<AuthUser> {
const result = await request<{ user: AuthUser }>('/api/auth/me');
const result = await request<{ user: AuthUser }>('/api/auth/me', {}, 8000);
return result.user;
},
login(username: string, password: string) {
+14
View File
@@ -0,0 +1,14 @@
export class RequestTimeoutError extends Error {
constructor() { super('请求超时'); this.name = 'RequestTimeoutError'; }
}
/** 限制整个请求(包括读取响应正文)的等待时间,并终止底层网络连接。 */
export async function withRequestDeadline<T>(operation: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T> {
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const expired = new Promise<never>((_, reject) => {
timer = setTimeout(() => { reject(new RequestTimeoutError()); controller.abort(); }, timeoutMs);
});
try { return await Promise.race([operation(controller.signal), expired]); }
finally { clearTimeout(timer); }
}