From 8684ce1a4d239301819986055c31f9877b00c45c Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 23 Oct 2026 17:47:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9A=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=9B=9E=E6=94=B6=E5=A4=B1=E6=95=88=E9=9A=A7=E9=81=93=E5=8D=A0?= =?UTF-8?q?=E7=94=A8=E5=B9=B6=E9=99=90=E5=88=B6=E9=A6=96=E6=AC=A1=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E7=AD=89=E5=BE=85=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/aliyun/README.md | 17 +++++ deploy/aliyun/quality-tunnel-health.py | 76 +++++++++++++++++++++ deploy/aliyun/quality-tunnel-health.service | 17 +++++ deploy/aliyun/quality-tunnel-health.timer | 11 +++ deploy/aliyun/test_quality_tunnel_health.py | 63 +++++++++++++++++ web/src/CachedWorkspacePages.tsx | 5 +- web/src/api.ts | 26 ++++--- web/src/requestDeadline.ts | 14 ++++ 8 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 deploy/aliyun/README.md create mode 100644 deploy/aliyun/quality-tunnel-health.py create mode 100644 deploy/aliyun/quality-tunnel-health.service create mode 100644 deploy/aliyun/quality-tunnel-health.timer create mode 100644 deploy/aliyun/test_quality_tunnel_health.py create mode 100644 web/src/requestDeadline.ts diff --git a/deploy/aliyun/README.md b/deploy/aliyun/README.md new file mode 100644 index 0000000..c794595 --- /dev/null +++ b/deploy/aliyun/README.md @@ -0,0 +1,17 @@ +# 质量平台公网连接 + +Nginx 将 `/api/` 转发至服务器回环地址 `127.0.0.1:15281`;本机 LaunchAgent 通过 SSH 将该端口回连到本机 `127.0.0.1:5281`。 + +## 失效连接恢复 + +`quality-tunnel-health.timer` 每轮结束 15 秒后再次检查连接。探测等待最多 4 秒;收到任何 HTTP 响应均视为连接可达,后端维护状态不会触发回收。 + +只有连续两次无响应、端口确认为回环端口 15281、拥有者确认为无子进程的 `sshd: admin`,且进程启动标识和端口归属未变化时,才结束该 SSH 子连接。它不结束 SSH 主服务、交互终端或其他项目连接。状态保存在仅 root 可访问的 `/run/quality-center-tunnel-health/`。 + +部署文件分别安装到 `/usr/local/sbin/quality-tunnel-health.py` 和 `/etc/systemd/system/quality-tunnel-health.{service,timer}`;启用 timer 即可。检查日志使用 `journalctl -u quality-tunnel-health.service`。停用时只需停用该 timer。 + +本机现有 `com.lyunlab.quality-center-tunnel` LaunchAgent 保留已配置的 SSH 别名与密钥;心跳间隔 15 秒、连续失败 2 次退出,重试间隔 10 秒,并保持 `ExitOnForwardFailure=yes` 和主机身份验证。 + +前端只为连接初始化、身份读取和页面状态校验设置 8 秒期限;普通数据查询和长时间分析不共用该期限。 + +单元测试:`python3 -m unittest discover -s deploy/aliyun -p 'test_*.py'`。 diff --git a/deploy/aliyun/quality-tunnel-health.py b/deploy/aliyun/quality-tunnel-health.py new file mode 100644 index 0000000..3a5ef18 --- /dev/null +++ b/deploy/aliyun/quality-tunnel-health.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""只回收质量平台回连端口上连续无响应的、身份已核实的 SSH 子连接。""" +import json +import os +from pathlib import Path +import re +import signal +import subprocess +from urllib.error import HTTPError +from urllib.request import ProxyHandler, build_opener + +PORT = 15281 +STATE = Path('/run/quality-center-tunnel-health/failure.json') + + +def reachable(): + try: + with build_opener(ProxyHandler({})).open(f'http://127.0.0.1:{PORT}/actuator/health', timeout=4): + return True + except HTTPError: + # 即使后端正在维护或拒绝请求,收到 HTTP 响应也证明隧道仍可通信。 + return True + except OSError: + return False + + +def owner(): + result = subprocess.run(['ss', '-H', '-ltnp', f'sport = :{PORT}'], capture_output=True, text=True, check=True) + if f'127.0.0.1:{PORT}' not in result.stdout: + return None + pids = set(re.findall(r'\bpid=(\d+),', result.stdout)) + if len(pids) != 1: + return None + pid = int(pids.pop()) + command = subprocess.run(['ps', '-p', str(pid), '-o', 'args='], capture_output=True, text=True, check=True).stdout.strip() + if command != 'sshd: admin': + return None + # 不结束带有远程命令或交互终端的 SSH 连接,也不结束 SSH 主服务。 + children = subprocess.run(['pgrep', '-P', str(pid)], capture_output=True, text=True) + if children.returncode != 1: + return None + stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + return {'pid': pid, 'started': stat[19]} + + +def check_once(state_path=STATE, probe=reachable, identify=owner, terminate=os.kill): + if probe(): + state_path.unlink(missing_ok=True) + return 'reachable' + current = identify() + if current is None: + state_path.unlink(missing_ok=True) + return 'no_verified_tunnel' + try: + previous = json.loads(state_path.read_text()) + except (OSError, ValueError): + previous = None + if previous != current: + state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + state_path.write_text(json.dumps(current)) + return 'first_failure' + # 连续两次检查失败后,再核实端口归属与进程启动标识,防止误杀复用 PID。 + if identify() != current: + state_path.unlink(missing_ok=True) + return 'owner_changed' + terminate(current['pid'], signal.SIGTERM) + state_path.unlink(missing_ok=True) + return 'stale_tunnel_released' + + +if __name__ == '__main__': + try: + print(json.dumps({'action': check_once()}), flush=True) + except (OSError, subprocess.SubprocessError) as error: + print(json.dumps({'action': 'check_failed', 'error': type(error).__name__}), flush=True) + raise SystemExit(1) diff --git a/deploy/aliyun/quality-tunnel-health.service b/deploy/aliyun/quality-tunnel-health.service new file mode 100644 index 0000000..d144620 --- /dev/null +++ b/deploy/aliyun/quality-tunnel-health.service @@ -0,0 +1,17 @@ +[Unit] +Description=Check the Quality Center reverse tunnel +After=network-online.target ssh.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 /usr/local/sbin/quality-tunnel-health.py +RuntimeDirectory=quality-center-tunnel-health +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +UMask=0077 +TimeoutStartSec=12 +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ReadWritePaths=/run/quality-center-tunnel-health diff --git a/deploy/aliyun/quality-tunnel-health.timer b/deploy/aliyun/quality-tunnel-health.timer new file mode 100644 index 0000000..a7d74a1 --- /dev/null +++ b/deploy/aliyun/quality-tunnel-health.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Detect unresponsive Quality Center tunnel connections + +[Timer] +OnBootSec=15 +OnUnitInactiveSec=15 +AccuracySec=1 +Unit=quality-tunnel-health.service + +[Install] +WantedBy=timers.target diff --git a/deploy/aliyun/test_quality_tunnel_health.py b/deploy/aliyun/test_quality_tunnel_health.py new file mode 100644 index 0000000..64049ab --- /dev/null +++ b/deploy/aliyun/test_quality_tunnel_health.py @@ -0,0 +1,63 @@ +import importlib.util +from pathlib import Path +import signal +import tempfile +import unittest +from unittest.mock import Mock, patch + +spec = importlib.util.spec_from_file_location('guard', Path(__file__).with_name('quality-tunnel-health.py')) +guard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(guard) + + +class TunnelHealthTest(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + self.state = Path(self.folder.name) / 'failure.json' + self.process = {'pid': 123, 'started': '456'} + self.kill = Mock() + + def check(self, healthy=False, identify=None): + return guard.check_once(self.state, lambda: healthy, identify or (lambda: self.process), self.kill) + + def test_two_failures_release_only_the_same_verified_process(self): + self.assertEqual(self.check(), 'first_failure') + self.kill.assert_not_called() + self.assertEqual(self.check(), 'stale_tunnel_released') + self.kill.assert_called_once_with(123, signal.SIGTERM) + + def test_reachable_tunnel_clears_failure_history(self): + self.check() + self.assertEqual(self.check(healthy=True), 'reachable') + self.assertFalse(self.state.exists()) + self.assertEqual(self.check(), 'first_failure') + self.kill.assert_not_called() + + def test_unverified_owner_is_never_killed(self): + self.check() + self.assertEqual(self.check(identify=lambda: None), 'no_verified_tunnel') + self.kill.assert_not_called() + + def test_new_connection_or_reused_pid_restarts_confirmation(self): + self.check() + self.process = {'pid': 123, 'started': '789'} + self.assertEqual(self.check(), 'first_failure') + self.kill.assert_not_called() + + def test_process_change_just_before_termination_is_rejected(self): + self.check() + identify = Mock(side_effect=[self.process, {'pid': 987, 'started': '789'}]) + self.assertEqual(self.check(identify=identify), 'owner_changed') + self.kill.assert_not_called() + + def test_main_sshd_or_interactive_connection_is_not_a_tunnel_owner(self): + for command,children in [('sshd: admin', 0), ('sshd: admin@pts/0', 1), ('sshd: /usr/sbin/sshd -D [listener]', 1)]: + with self.subTest(command=command), patch.object(guard.subprocess, 'run') as run: + run.side_effect = [Mock(stdout='LISTEN 0 128 127.0.0.1:15281 0.0.0.0:* users:(("sshd",pid=123,fd=7))'), + Mock(stdout=command), Mock(returncode=children)] + self.assertIsNone(guard.owner()) + + +if __name__ == '__main__': + unittest.main() diff --git a/web/src/CachedWorkspacePages.tsx b/web/src/CachedWorkspacePages.tsx index bc5cd78..f8ef3f4 100644 --- a/web/src/CachedWorkspacePages.tsx +++ b/web/src/CachedWorkspacePages.tsx @@ -45,6 +45,7 @@ export const CachedWorkspacePages = forwardRef value + 1); revisions.current = state; setError(''); + setReady(true); return true; } catch (value) { if (alive.current && generation.current === epoch && !onUnauthorized(value)) setError(value instanceof Error ? value.message : '页面状态校验失败,请重试'); @@ -84,7 +85,7 @@ export const CachedWorkspacePages = forwardRef { alive.current = true; - void validate().then(ok => { if (ok) setReady(true); }); + void validate(); const invalidate = () => setRefreshKey(value => value + 1); window.addEventListener(WORKSPACE_INVALIDATED, invalidate); return () => { alive.current = false; generation.current++; validation.current = null; window.removeEventListener(WORKSPACE_INVALIDATED, invalidate); }; @@ -95,7 +96,7 @@ export const CachedWorkspacePages = forwardRef window.scrollTo(0, entries.current.get(page)?.scroll || 0)); return () => cancelAnimationFrame(frame); }, [page, ready, activeEntryKey]); - const connectionNotice = error && { void validate().then(ok => { if (ok) setReady(true); }); }}>重新连接} />; + const connectionNotice = error && { void validate(); }}>重新连接} />; if (!ready) return connectionNotice ||
正在打开工作台…
; return <>{connectionNotice}{Array.from(entries.current.entries()).filter(([name]) => retained(name) || name === page).map(([name, entry]) => { const active = name === page; diff --git a/web/src/api.ts b/web/src/api.ts index 49e2a3c..5258578 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,6 +1,7 @@ import { WORKSPACE_INVALIDATED } from './pageCache'; import { consumeEventStream, type StreamEvent } from './eventStream'; import { publicErrorMessage } from './publicDataDisplay'; +import { RequestTimeoutError, withRequestDeadline } from './requestDeadline'; import type { AuthUser, FilterField, @@ -65,16 +66,19 @@ async function readResponse(response: Response): Promise { return payload as T; } -async function request(path: string, init: RequestInit = {}): Promise { +async function request(path: string, init: RequestInit = {}, timeoutMs?: number): Promise { const headers = new Headers(init.headers); if (init.body && !(init.body instanceof FormData) && !headers.has('Content-Type')) headers.set('Content-Type', 'application/json'); - const response = await fetchApi(path, { - ...init, - headers, - credentials: 'same-origin', - cache: 'no-store', - }); - return readResponse(response); + const send = async (signal = init.signal) => { + const response = await fetchApi(path, { ...init, signal, headers, credentials: 'same-origin', cache: 'no-store' }); + return readResponse(response); + }; + if (!timeoutMs) return send(); + try { return await withRequestDeadline(signal => send(signal), timeoutMs); } + catch (error) { + if (error instanceof RequestTimeoutError) throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); + throw error; + } } async function requestBlob(path: string, signal?: AbortSignal): Promise<{ blob: Blob; filename: string }> { @@ -113,7 +117,7 @@ async function readEventStream(path: string, options: { signal?: AbortSignal; la } export async function refreshCsrf(): Promise { - const result = await request<{ token: string }>('/api/auth/csrf'); + const result = await request<{ token: string }>('/api/auth/csrf', {}, 8000); csrfToken = result.token; return csrfToken; } @@ -143,14 +147,14 @@ async function mutateForm(path: string, body: FormData): Promise { } export const api = { - workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state'); }, + workspaceState() { return request<{ authRevision: string; dataRevision: string }>('/api/workspace/state', {}, 8000); }, filterFields(manage = false) { return request(`/api/filter-fields?manage=${manage}`); }, filterFieldOptions(key: string, search = '') { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search})}`); }, filterFieldColumns() { return request<{ black: string[]; bright: string[] }>('/api/filter-fields/columns'); }, saveFilterFields(fields: FilterField[]) { return mutate('/api/filter-fields', fields, 'PUT'); }, refreshCsrf, async me(): Promise { - const result = await request<{ user: AuthUser }>('/api/auth/me'); + const result = await request<{ user: AuthUser }>('/api/auth/me', {}, 8000); return result.user; }, login(username: string, password: string) { diff --git a/web/src/requestDeadline.ts b/web/src/requestDeadline.ts new file mode 100644 index 0000000..0b3456e --- /dev/null +++ b/web/src/requestDeadline.ts @@ -0,0 +1,14 @@ +export class RequestTimeoutError extends Error { + constructor() { super('请求超时'); this.name = 'RequestTimeoutError'; } +} + +/** 限制整个请求(包括读取响应正文)的等待时间,并终止底层网络连接。 */ +export async function withRequestDeadline(operation: (signal: AbortSignal) => Promise, timeoutMs: number): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => { reject(new RequestTimeoutError()); controller.abort(); }, timeoutMs); + }); + try { return await Promise.race([operation(controller.signal), expired]); } + finally { clearTimeout(timer); } +}