修复:自动回收失效隧道占用并限制首次连接等待时间
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# 质量平台公网连接
|
||||
|
||||
Nginx 将 `/api/` 转发至服务器回环地址 `127.0.0.1:15281`;本机 LaunchAgent 通过 SSH 将该端口回连到本机 `127.0.0.1:5281`。
|
||||
|
||||
## 失效连接恢复
|
||||
|
||||
`quality-tunnel-health.timer` 每轮结束 15 秒后再次检查连接。探测等待最多 4 秒;收到任何 HTTP 响应均视为连接可达,后端维护状态不会触发回收。
|
||||
|
||||
只有连续两次无响应、端口确认为回环端口 15281、拥有者确认为无子进程的 `sshd: admin`,且进程启动标识和端口归属未变化时,才结束该 SSH 子连接。它不结束 SSH 主服务、交互终端或其他项目连接。状态保存在仅 root 可访问的 `/run/quality-center-tunnel-health/`。
|
||||
|
||||
部署文件分别安装到 `/usr/local/sbin/quality-tunnel-health.py` 和 `/etc/systemd/system/quality-tunnel-health.{service,timer}`;启用 timer 即可。检查日志使用 `journalctl -u quality-tunnel-health.service`。停用时只需停用该 timer。
|
||||
|
||||
本机现有 `com.lyunlab.quality-center-tunnel` LaunchAgent 保留已配置的 SSH 别名与密钥;心跳间隔 15 秒、连续失败 2 次退出,重试间隔 10 秒,并保持 `ExitOnForwardFailure=yes` 和主机身份验证。
|
||||
|
||||
前端只为连接初始化、身份读取和页面状态校验设置 8 秒期限;普通数据查询和长时间分析不共用该期限。
|
||||
|
||||
单元测试:`python3 -m unittest discover -s deploy/aliyun -p 'test_*.py'`。
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""只回收质量平台回连端口上连续无响应的、身份已核实的 SSH 子连接。"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import ProxyHandler, build_opener
|
||||
|
||||
PORT = 15281
|
||||
STATE = Path('/run/quality-center-tunnel-health/failure.json')
|
||||
|
||||
|
||||
def reachable():
|
||||
try:
|
||||
with build_opener(ProxyHandler({})).open(f'http://127.0.0.1:{PORT}/actuator/health', timeout=4):
|
||||
return True
|
||||
except HTTPError:
|
||||
# 即使后端正在维护或拒绝请求,收到 HTTP 响应也证明隧道仍可通信。
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def owner():
|
||||
result = subprocess.run(['ss', '-H', '-ltnp', f'sport = :{PORT}'], capture_output=True, text=True, check=True)
|
||||
if f'127.0.0.1:{PORT}' not in result.stdout:
|
||||
return None
|
||||
pids = set(re.findall(r'\bpid=(\d+),', result.stdout))
|
||||
if len(pids) != 1:
|
||||
return None
|
||||
pid = int(pids.pop())
|
||||
command = subprocess.run(['ps', '-p', str(pid), '-o', 'args='], capture_output=True, text=True, check=True).stdout.strip()
|
||||
if command != 'sshd: admin':
|
||||
return None
|
||||
# 不结束带有远程命令或交互终端的 SSH 连接,也不结束 SSH 主服务。
|
||||
children = subprocess.run(['pgrep', '-P', str(pid)], capture_output=True, text=True)
|
||||
if children.returncode != 1:
|
||||
return None
|
||||
stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
|
||||
return {'pid': pid, 'started': stat[19]}
|
||||
|
||||
|
||||
def check_once(state_path=STATE, probe=reachable, identify=owner, terminate=os.kill):
|
||||
if probe():
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'reachable'
|
||||
current = identify()
|
||||
if current is None:
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'no_verified_tunnel'
|
||||
try:
|
||||
previous = json.loads(state_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
previous = None
|
||||
if previous != current:
|
||||
state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
state_path.write_text(json.dumps(current))
|
||||
return 'first_failure'
|
||||
# 连续两次检查失败后,再核实端口归属与进程启动标识,防止误杀复用 PID。
|
||||
if identify() != current:
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'owner_changed'
|
||||
terminate(current['pid'], signal.SIGTERM)
|
||||
state_path.unlink(missing_ok=True)
|
||||
return 'stale_tunnel_released'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
print(json.dumps({'action': check_once()}), flush=True)
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
print(json.dumps({'action': 'check_failed', 'error': type(error).__name__}), flush=True)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Check the Quality Center reverse tunnel
|
||||
After=network-online.target ssh.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /usr/local/sbin/quality-tunnel-health.py
|
||||
RuntimeDirectory=quality-center-tunnel-health
|
||||
RuntimeDirectoryMode=0700
|
||||
RuntimeDirectoryPreserve=yes
|
||||
UMask=0077
|
||||
TimeoutStartSec=12
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/run/quality-center-tunnel-health
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Detect unresponsive Quality Center tunnel connections
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15
|
||||
OnUnitInactiveSec=15
|
||||
AccuracySec=1
|
||||
Unit=quality-tunnel-health.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,63 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('guard', Path(__file__).with_name('quality-tunnel-health.py'))
|
||||
guard = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(guard)
|
||||
|
||||
|
||||
class TunnelHealthTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.folder = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.folder.cleanup)
|
||||
self.state = Path(self.folder.name) / 'failure.json'
|
||||
self.process = {'pid': 123, 'started': '456'}
|
||||
self.kill = Mock()
|
||||
|
||||
def check(self, healthy=False, identify=None):
|
||||
return guard.check_once(self.state, lambda: healthy, identify or (lambda: self.process), self.kill)
|
||||
|
||||
def test_two_failures_release_only_the_same_verified_process(self):
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
self.assertEqual(self.check(), 'stale_tunnel_released')
|
||||
self.kill.assert_called_once_with(123, signal.SIGTERM)
|
||||
|
||||
def test_reachable_tunnel_clears_failure_history(self):
|
||||
self.check()
|
||||
self.assertEqual(self.check(healthy=True), 'reachable')
|
||||
self.assertFalse(self.state.exists())
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_unverified_owner_is_never_killed(self):
|
||||
self.check()
|
||||
self.assertEqual(self.check(identify=lambda: None), 'no_verified_tunnel')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_new_connection_or_reused_pid_restarts_confirmation(self):
|
||||
self.check()
|
||||
self.process = {'pid': 123, 'started': '789'}
|
||||
self.assertEqual(self.check(), 'first_failure')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_process_change_just_before_termination_is_rejected(self):
|
||||
self.check()
|
||||
identify = Mock(side_effect=[self.process, {'pid': 987, 'started': '789'}])
|
||||
self.assertEqual(self.check(identify=identify), 'owner_changed')
|
||||
self.kill.assert_not_called()
|
||||
|
||||
def test_main_sshd_or_interactive_connection_is_not_a_tunnel_owner(self):
|
||||
for command,children in [('sshd: admin', 0), ('sshd: admin@pts/0', 1), ('sshd: /usr/sbin/sshd -D [listener]', 1)]:
|
||||
with self.subTest(command=command), patch.object(guard.subprocess, 'run') as run:
|
||||
run.side_effect = [Mock(stdout='LISTEN 0 128 127.0.0.1:15281 0.0.0.0:* users:(("sshd",pid=123,fd=7))'),
|
||||
Mock(stdout=command), Mock(returncode=children)]
|
||||
self.assertIsNone(guard.owner())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -45,6 +45,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
|
||||
if (revisions.current && state.dataRevision !== revisions.current.dataRevision) setRefreshKey(value => value + 1);
|
||||
revisions.current = state;
|
||||
setError('');
|
||||
setReady(true);
|
||||
return true;
|
||||
} catch (value) {
|
||||
if (alive.current && generation.current === epoch && !onUnauthorized(value)) setError(value instanceof Error ? value.message : '页面状态校验失败,请重试');
|
||||
@@ -84,7 +85,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
|
||||
} }));
|
||||
useEffect(() => {
|
||||
alive.current = true;
|
||||
void validate().then(ok => { if (ok) setReady(true); });
|
||||
void validate();
|
||||
const invalidate = () => setRefreshKey(value => value + 1);
|
||||
window.addEventListener(WORKSPACE_INVALIDATED, invalidate);
|
||||
return () => { alive.current = false; generation.current++; validation.current = null; window.removeEventListener(WORKSPACE_INVALIDATED, invalidate); };
|
||||
@@ -95,7 +96,7 @@ export const CachedWorkspacePages = forwardRef<WorkspaceNavigation, {
|
||||
const frame = requestAnimationFrame(() => window.scrollTo(0, entries.current.get(page)?.scroll || 0));
|
||||
return () => cancelAnimationFrame(frame);
|
||||
}, [page, ready, activeEntryKey]);
|
||||
const connectionNotice = error && <Alert type="warning" showIcon message="页面状态暂未更新" description={error} action={<Button onClick={() => { void validate().then(ok => { if (ok) setReady(true); }); }}>重新连接</Button>} />;
|
||||
const connectionNotice = error && <Alert type="warning" showIcon message="页面状态暂未更新" description={error} action={<Button onClick={() => { void validate(); }}>重新连接</Button>} />;
|
||||
if (!ready) return connectionNotice || <div className="loading-panel"><Spin />正在打开工作台…</div>;
|
||||
return <>{connectionNotice}{Array.from(entries.current.entries()).filter(([name]) => retained(name) || name === page).map(([name, entry]) => {
|
||||
const active = name === page;
|
||||
|
||||
+15
-11
@@ -1,6 +1,7 @@
|
||||
import { WORKSPACE_INVALIDATED } from './pageCache';
|
||||
import { consumeEventStream, type StreamEvent } from './eventStream';
|
||||
import { publicErrorMessage } from './publicDataDisplay';
|
||||
import { RequestTimeoutError, withRequestDeadline } from './requestDeadline';
|
||||
import type {
|
||||
AuthUser,
|
||||
FilterField,
|
||||
@@ -65,16 +66,19 @@ async function readResponse<T>(response: Response): Promise<T> {
|
||||
return payload as T;
|
||||
}
|
||||
|
||||
async function request<T>(path: string, init: RequestInit = {}): Promise<T> {
|
||||
async function request<T>(path: string, init: RequestInit = {}, timeoutMs?: number): Promise<T> {
|
||||
const headers = new Headers(init.headers);
|
||||
if (init.body && !(init.body instanceof FormData) && !headers.has('Content-Type')) headers.set('Content-Type', 'application/json');
|
||||
const response = await fetchApi(path, {
|
||||
...init,
|
||||
headers,
|
||||
credentials: 'same-origin',
|
||||
cache: 'no-store',
|
||||
});
|
||||
return readResponse<T>(response);
|
||||
const send = async (signal = init.signal) => {
|
||||
const response = await fetchApi(path, { ...init, signal, headers, credentials: 'same-origin', cache: 'no-store' });
|
||||
return readResponse<T>(response);
|
||||
};
|
||||
if (!timeoutMs) return send();
|
||||
try { return await withRequestDeadline(signal => send(signal), timeoutMs); }
|
||||
catch (error) {
|
||||
if (error instanceof RequestTimeoutError) throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function requestBlob(path: string, signal?: AbortSignal): Promise<{ blob: Blob; filename: string }> {
|
||||
@@ -113,7 +117,7 @@ async function readEventStream(path: string, options: { signal?: AbortSignal; la
|
||||
}
|
||||
|
||||
export async function refreshCsrf(): Promise<string> {
|
||||
const result = await request<{ token: string }>('/api/auth/csrf');
|
||||
const result = await request<{ token: string }>('/api/auth/csrf', {}, 8000);
|
||||
csrfToken = result.token;
|
||||
return csrfToken;
|
||||
}
|
||||
@@ -143,14 +147,14 @@ async function mutateForm<T>(path: string, body: FormData): Promise<T> {
|
||||
}
|
||||
|
||||
export const api = {
|
||||
workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state'); },
|
||||
workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state', {}, 8000); },
|
||||
filterFields(manage = false) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`); },
|
||||
filterFieldOptions(key: string, search = '') { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search})}`); },
|
||||
filterFieldColumns() { return request<{ black: string[]; bright: string[] }>('/api/filter-fields/columns'); },
|
||||
saveFilterFields(fields: FilterField[]) { return mutate<FilterField[]>('/api/filter-fields', fields, 'PUT'); },
|
||||
refreshCsrf,
|
||||
async me(): Promise<AuthUser> {
|
||||
const result = await request<{ user: AuthUser }>('/api/auth/me');
|
||||
const result = await request<{ user: AuthUser }>('/api/auth/me', {}, 8000);
|
||||
return result.user;
|
||||
},
|
||||
login(username: string, password: string) {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
export class RequestTimeoutError extends Error {
|
||||
constructor() { super('请求超时'); this.name = 'RequestTimeoutError'; }
|
||||
}
|
||||
|
||||
/** 限制整个请求(包括读取响应正文)的等待时间,并终止底层网络连接。 */
|
||||
export async function withRequestDeadline<T>(operation: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T> {
|
||||
const controller = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const expired = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => { reject(new RequestTimeoutError()); controller.abort(); }, timeoutMs);
|
||||
});
|
||||
try { return await Promise.race([operation(controller.signal), expired]); }
|
||||
finally { clearTimeout(timer); }
|
||||
}
|
||||
Reference in New Issue
Block a user