diff --git a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java index 05f1f57..6de9c1a 100644 --- a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java +++ b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java @@ -55,9 +55,13 @@ public class SystemUpdateService { if (!enabled) return false; try { Map state = readMetadata("status.json"); + Map journal = readMetadata("journal.json"); + // 同一任务的持久提交记录优先于尚未成功刷新的进度提示。 + if (journal.get("operationId") instanceof String operationId && !operationId.isBlank() + && operationId.equals(state.get("operationId")) + && Set.of("complete", "rolledBack").contains(journal.get("phase"))) return false; if (state.get("state") instanceof String phase && Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true; - Map journal = readMetadata("journal.json"); return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase); } catch (Exception failure) { return true; diff --git a/backend/src/main/resources/update-protocol.json b/backend/src/main/resources/update-protocol.json new file mode 100644 index 0000000..0c0de21 --- /dev/null +++ b/backend/src/main/resources/update-protocol.json @@ -0,0 +1 @@ +{"protocol":2} diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java index a7ad4c1..f5c670b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java @@ -60,4 +60,21 @@ class SystemUpdateServiceTest { Files.writeString(directory.resolve("status.json"), "broken JSON"); assertTrue(service.maintenanceActive()); } + @Test + void committedOperationOverridesItsStaleProgressButNotAnotherOperation(@TempDir Path root) throws Exception { + var service = new SystemUpdateService(JsonMapper.builder().build(), root.toString(), false); + var enabled = SystemUpdateService.class.getDeclaredField("enabled"); + enabled.setAccessible(true); + enabled.setBoolean(service, true); + var directory = Files.createDirectories(root.resolve(".runtime/update")); + String current = "a".repeat(32), previous = "b".repeat(32); + for (String phase : new String[]{"complete", "rolledBack"}) { + Files.writeString(directory.resolve("status.json"), "{\"state\":\"restarting\",\"operationId\":\"" + current + "\"}"); + Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + current + "\"}"); + assertFalse(service.maintenanceActive(), phase); + Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + previous + "\"}"); + assertTrue(service.maintenanceActive(), "another operation must not release maintenance"); + } + } + } diff --git a/deploy/offline/README.md b/deploy/offline/README.md index c48d870..28340df 100644 --- a/deploy/offline/README.md +++ b/deploy/offline/README.md @@ -64,3 +64,16 @@ python3 deploy/offline/publish_update.py \ Windows 的刷盘操作要求写句柄。程序及私有配置备份统一使用写入、flush、fsync 后再原子替换,移除复制后以只读句柄刷盘的步骤。备份失败仍停止更新并保留旧程序与数据库;不降低备份持久化要求。 “修复Windows更新.cmd”适配原始更新器和已安装检查超时修复的更新器,保留原清单、缓存修复和版本信息。更新失败诊断写入 `.runtime/update/failure.json`,只包含阶段、异常类型、系统错误码和源码位置,不含业务正文、凭据或异常参数;回退异常单独记录在 `rollback-failure.json`。 + +### Windows 更新流程加固(本地验证,尚未发布) + +- 停服前完成签名、下载完整性、现有运行环境、备份路径长度及读写权限检查;预检失败保留当前服务运行。完整环境检查预算为30分钟,更新后仅核验程序和应用脚本(5分钟),不重复读取约4.6GB运行环境。 +- 安全停服内部最多等待5分钟,更新器外层等待6分钟;已登记进程状态无法确认时停止后续操作,保留状态记录,不误报成功或结束未知进程。 +- 仅当对应源码与原清单摘要一致时,忽略可再生的运行库字节码,不忽略真实源码损坏。 +- `journal.phase=complete` 是不可逆提交点;状态展示或清理失败只记诊断,不再回退。后端按同一 operationId 的提交记录解除维护状态,旧任务记录不能解除新任务的维护。 +- 新后端尚未启动时,回退不重新导入没有变化的数据库。可能发生数据库变更后,恢复进度写入独立 `recovery.json`,绑定备份程序清单及 SQL 摘要。导入及安全关闭确认后复用同一个恢复库,重复启动不重复导入,并幂等恢复配置和迁移回执。 +- 不复用断电前未确认完整的导入;最多分配两次独立恢复尝试,且每次检查磁盘空间。超过次数停止自动重试并保留现场,不删除任何数据库或按前缀清理未知库。 +- 旧安装修复入口同时更新 updater、control 和清单摘要;签名公钥、不可变 bootstrap、版本、业务数据和迁移回执不变,修复入口不启动服务。必须先安装该入口,保证不可变 bootstrap 未来恢复出的旧程序也包含加固后的恢复器。 +- Windows 真机的 NTFS、进程权限和断电恢复仍需隔离副本验收。本轮本地故障注入及真实隔离 MySQL 验证不等同于生产 Windows 验收。 + +加固修复入口不能搭配已发布的旧0.3包使用:停服前会同时核对更新器、控制器及 JAR 中的恢复协议标记。不兼容的旧包会在服务仍运行时被拒绝;配套新包须包含本轮后端、控制器与更新器,禁止混装。 diff --git a/deploy/offline/control.py b/deploy/offline/control.py index d47033e..1a075de 100644 --- a/deploy/offline/control.py +++ b/deploy/offline/control.py @@ -6,6 +6,7 @@ import argparse import contextlib import hashlib import json +import math import os from pathlib import Path import platform @@ -15,8 +16,12 @@ import signal import socket import subprocess import sys +_old_dont_write_bytecode = sys.dont_write_bytecode +# Importing the controller must not create caches in the immutable package. +sys.dont_write_bytecode = True import threading import time +from pathlib import PurePosixPath from urllib.request import ProxyHandler, build_opener import psutil @@ -25,12 +30,15 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) import runtime_platform as host from gateway import make_server +sys.dont_write_bytecode = _old_dont_write_bytecode +del _old_dont_write_bytecode PRIVATE = ROOT / '.runtime' DATA = ROOT / 'runtimedata' CONFIG = PRIVATE / 'deployment.json' STATE = PRIVATE / 'service.json' STOP = PRIVATE / 'stop-request.json' +UPDATE_CONTROL_PROTOCOL = 2 IS_WINDOWS = os.name == 'nt' EXE = '.exe' if IS_WINDOWS else '' PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3') @@ -39,6 +47,7 @@ MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE) MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE) ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE) OPENER = build_opener(ProxyHandler({})) +STOP_TIMEOUT_SECONDS = 300 def atomic(path, value): @@ -65,7 +74,7 @@ def private_directory(path): path.chmod(0o700) -def configuration(): +def configuration(create=True): if CONFIG.exists(): value = json.loads(CONFIG.read_text(encoding='utf-8')) else: @@ -73,7 +82,8 @@ def configuration(): 'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1', 'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32), 'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)} - atomic(CONFIG, value) + if create: + atomic(CONFIG, value) value.setdefault('mysqlDatabase', 'quality_platform') if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']): raise RuntimeError('部署数据库名称无效。') @@ -137,14 +147,159 @@ def record(process, command): def matches(info): + """Return True for this process, False for gone/replaced, None if unknown.""" + if not isinstance(info, dict): + return None try: - process = psutil.Process(int(info['pid'])) - return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE - and abs(process.create_time() - info['created']) < .01 - and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable']) - and process.cmdline()[1:] == info['args']) - except (psutil.Error, KeyError, ValueError, TypeError): + pid = info['pid'] + created = info['created'] + executable = info['executable'] + arguments = info['args'] + if (type(pid) is not int or pid <= 0 or type(created) not in (int, float) + or not math.isfinite(created) or not isinstance(executable, str) + or not isinstance(arguments, list) or not all(isinstance(item, str) for item in arguments)): + return None + except (KeyError, ValueError, TypeError): + return None + try: + process = psutil.Process(pid) + except psutil.NoSuchProcess: return False + except Exception: + return None + try: + if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE: + return False + if abs(process.create_time() - created) >= .01: + return False + if os.path.normcase(str(Path(process.exe()).resolve())) != os.path.normcase(executable): + return False + if process.cmdline()[1:] != arguments: + return False + return True + except psutil.NoSuchProcess: + return False + except Exception: + return None + + +def registered_alive(records): + """Keep both confirmed-live and uninspectable registered process records.""" + if not isinstance(records, list): + return [], True + alive, unknown = [], False + for info in records: + status = matches(info) + if status is True: + alive.append(info) + elif status is None: + alive.append(info) + unknown = True + return alive, unknown + + +def retain_registered_children(state, records, unresolved=False): + survivors, unknown = registered_alive(records) + state['children'] = survivors + unresolved = unresolved or unknown + if unresolved: + state['unknownChildren'] = True + return survivors, unresolved + + +def registered_named(records, name): + confirmed, unknown = [], [] + for info in records: + if not isinstance(info, dict) or info.get('name') != name: + continue + status = matches(info) + if status is True: + confirmed.append(info) + elif status is None: + unknown.append(info) + return confirmed, unknown + + +def snapshot_process(process, name): + try: + result = record(process, process.cmdline()) + result['name'] = name + return result + except (psutil.Error, OSError): + return None + + +def process_object_matches(process, created): + try: + return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE + and abs(process.create_time() - created) < .01) + except psutil.NoSuchProcess: + return False + except Exception: + # Permission failures and unexpected inspection errors are unknown, + # not evidence that the observed child has exited. + return None + + +def terminate_observed(processes, timeout): + """Stop only process objects captured below a verified service child.""" + unverified = False + for process, created, info in processes: + observed = process_object_matches(process, created) + if observed is None: + if info is None: + unverified = True + elif observed is True: + with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied): + process.terminate() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + states = [process_object_matches(process, created) for process, created, _info in processes] + if any(observed is None for observed in states): + unverified = unverified or any(observed is None and info is None + for observed, (_process, _created, info) in zip(states, processes)) + break + if not any(states): + break + time.sleep(.1) + for process, created, info in processes: + observed = process_object_matches(process, created) + if observed is None: + if info is None: + unverified = True + elif observed is True: + with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied): + process.kill() + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + states = [process_object_matches(process, created) for process, created, _info in processes] + if any(observed is None for observed in states): + unverified = unverified or any(observed is None and info is None + for observed, (_process, _created, info) in zip(states, processes)) + break + if not any(states): + break + time.sleep(.1) + survivors = [] + for process, created, info in processes: + observed = process_object_matches(process, created) + if observed is None: + if info is None: + unverified = True + else: + survivors.append(info) + elif observed is True: + identity = matches(info) if info is not None else None + if info is None: + unverified = True + elif identity is None: + survivors.append(info) + elif identity is False: + survivors.append(info) + unverified = True + else: + survivors.append(info) + return survivors, unverified def read_state(): @@ -154,6 +309,45 @@ def read_state(): return {} +def wait_for_stop(supervisor, request, timeout_seconds=STOP_TIMEOUT_SECONDS, interval=.5): + request_sent = False + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + supervisor_status = matches(supervisor) + if supervisor_status is None: + raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。') + if supervisor_status is True: + if not request_sent: + atomic(STOP, request) + request_sent = True + time.sleep(interval) + continue + + state = read_state() + if 'supervisor' in state: + current_supervisor = state['supervisor'] + current_status = matches(current_supervisor) + if current_status is None: + raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。') + if current_status is True: + if current_supervisor != supervisor: + raise RuntimeError('服务状态已变化,未停止任何不明进程。') + if not request_sent: + atomic(STOP, request) + request_sent = True + time.sleep(interval) + continue + survivors, unknown_children = registered_alive(state.get('children', [])) + if unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'): + raise RuntimeError('服务监督程序已退出,但仍有未确认停止的已记录进程;未结束任何不明进程。') + if survivors: + time.sleep(interval) + continue + print('服务已停止。') + return + raise RuntimeError('服务仍在安全关闭,请稍后检查状态。') + + def free_port(port, bind='127.0.0.1'): # A recently closed TCP connection may remain in TIME_WAIT. Reject a live # listener, but allow a clean restart without waiting for that state to expire. @@ -182,22 +376,136 @@ def sql(value, query): env=env, timeout=10) -def verify(check_hashes=True): +def manifest_name(name): + if (not isinstance(name, str) or not name or '\\' in name + or PurePosixPath(name).is_absolute() + or any(part in ('', '.', '..') for part in name.split('/'))): + raise RuntimeError('文件清单包含无效路径。') + return PurePosixPath(name) + + +def manifest_path(name): + parts = manifest_name(name).parts + path = ROOT.joinpath(*parts) + root = ROOT.resolve() + if not path.resolve(strict=False).is_relative_to(root): + raise RuntimeError('文件路径异常,已停止校验:' + name) + parent = path.parent + while parent != ROOT and parent != parent.parent: + if parent.is_symlink(): + raise RuntimeError('文件路径异常,已停止校验:' + name) + parent = parent.parent + return path + + +def cache_source(name): + """Return only timestamp-based bytecode sources from the bundled Python.""" + if not isinstance(name, str) or '\\' in name: + return None + path = PurePosixPath(name) + if path.parts[:2] != ('runtimes', 'python') or any(part in ('.', '..') for part in name.split('/')): + return None + if path.parent.name != '__pycache__': + return None + match = re.fullmatch(r'(.+)\.cpython-\d+(?:\.opt-\d+)?\.pyc', path.name) + return (path.parent.parent / (match[1] + '.py')).as_posix() if match else None + + +def file_digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def regular_manifest_file(name): + path = manifest_path(name) + if path.is_symlink(): + raise RuntimeError('文件路径异常,已停止校验:' + name) + return path + + +def program_file_path(name): + """Mirror update_manager.program_path without importing code during a read-only check.""" + path = manifest_name(name) + return (name == 'app/backend.jar' or name.startswith('app/web/') + or name == 'backend/src/main/resources/analysis-runtime-policy.json' + or name.startswith('scripts/') and path.suffix == '.py' + or name.startswith('ai/nao-quality/') + or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md')) + + +def verify_manifest_files(manifest, program_only=False): + entries = manifest.get('files') + if not isinstance(entries, list): + raise RuntimeError('文件清单无效。') + by_name = {} + for entry in entries: + if not isinstance(entry, dict) or not isinstance(entry.get('path'), str): + raise RuntimeError('文件清单无效。') + name = entry['path'] + manifest_name(name) + if name in by_name: + raise RuntimeError('文件清单包含重复路径。') + by_name[name] = entry + + ignored_caches = set() + if not program_only: + # Timestamp-based bytecode varies between Windows copies. Ignore only + # cache entries whose exact source is present and matches its manifest + # digest; ordinary Python source files are still hashed below. + for name in by_name: + source = cache_source(name) + if source is None: + continue + source_entry = by_name.get(source) + expected = source_entry.get('sha256') if isinstance(source_entry, dict) else None + source_path = regular_manifest_file(source) + if (not isinstance(expected, str) or not source_path.is_file() + or file_digest(source_path) != expected): + raise RuntimeError('对应源码缺失或校验失败,已停止校验:' + source) + cache_path = regular_manifest_file(name) + if cache_path.exists() and not cache_path.is_file(): + raise RuntimeError('缓存文件路径异常,已停止校验:' + name) + ignored_caches.add(name) + + for name, entry in by_name.items(): + if program_only and not program_file_path(name): + continue + if name in ignored_caches: + continue + path = manifest_path(name) + expected_link = entry.get('symlink') + if expected_link is not None: + valid = path.is_symlink() and os.readlink(path) == expected_link + else: + if path.is_symlink(): + valid = False + elif path.is_file(): + with path.open('rb') as stream: + valid = hashlib.file_digest(stream, 'sha256').hexdigest() == entry.get('sha256') + else: + valid = False + if not valid: + raise RuntimeError('文件缺失或校验失败:' + name) + + +def verify_runtime(check_hashes=True, program_only=False): manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8')) expected_os = 'windows' if IS_WINDOWS else platform.system().lower() if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'): raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。') if check_hashes: - for entry in manifest['files']: - path = ROOT / entry['path'] - if entry.get('symlink') is not None: - valid = path.is_symlink() and os.readlink(path) == entry['symlink'] - else: - valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256'] - if not valid: - raise RuntimeError('文件缺失或校验失败:' + entry['path']) - value = configuration() + verify_manifest_files(manifest, program_only=program_only) + value = configuration(create=False) env = environment(value) + if program_only: + application_probe = subprocess.run([str(PYTHON), '-c', + 'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;' + 'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;' + 'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64', + str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45) + if application_probe.returncode: + raise RuntimeError('AI 应用脚本或策略资源检查失败。') + return value for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']): result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30) if result.returncode: @@ -221,6 +529,13 @@ def verify(check_hashes=True): return value +def verify(check_hashes=True): + # Keep the historical full verification entry point and its configuration + # initialization behavior for the installer and migration recovery tools. + verify_runtime(check_hashes=check_hashes) + return configuration() + + def addresses(value): ips = set() if value['webBind'] == '0.0.0.0': @@ -360,38 +675,90 @@ def serve(database_only=False): finally: state['ready'] = False atomic(STATE, state) - # Stop backend first, then its isolated AI processes, then the database. + cleanup_failures, extra_survivors = [], [] + unknown_children = False + # Only signal the backend and descendants observed beneath its verified + # PID/start-time/executable/arguments identity. for info in reversed(children): - if info['name'] != 'backend' or not matches(info): + if not isinstance(info, dict) or info.get('name') != 'backend': + continue + backend_status = matches(info) + if backend_status is None: + extra_survivors.append(info) + continue + if backend_status is not True: continue - process = psutil.Process(info['pid']) - descendants = process.children(recursive=True) - process.terminate() try: - process.wait(35) - except psutil.TimeoutExpired: - if matches(info): - process.kill() + process = psutil.Process(info['pid']) + except psutil.NoSuchProcess: + continue + except Exception: + extra_survivors.append(info) + continue + observed = [] + try: + descendants = process.children(recursive=True) + except psutil.NoSuchProcess: + descendants = [] + except psutil.Error: + descendants = [] + unknown_children = True for child in descendants: - with contextlib.suppress(psutil.NoSuchProcess): - child.terminate() - with contextlib.suppress(OSError, subprocess.SubprocessError): - subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'], - env=environment(value), cwd=ROOT, capture_output=True, timeout=25) - if any(info['name'] == 'mysql' and matches(info) for info in children): + try: + created = child.create_time() + observed.append((child, created, snapshot_process(child, 'backend-descendant'))) + except (psutil.Error, OSError): + unknown_children = True + observed.append((process, info['created'], info)) + survivors, unverified = terminate_observed(observed, 35) + extra_survivors.extend(item for item in survivors if item.get('name') == 'backend-descendant') + unknown_children = unknown_children or unverified + + try: + stopped = subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'], + env=environment(value), cwd=ROOT, capture_output=True, timeout=25) + if stopped.returncode: + cleanup_failures.append('analysisProcesses') + except (OSError, subprocess.SubprocessError): + cleanup_failures.append('analysisProcesses') + mysql_infos, unknown_mysql = registered_named(children, 'mysql') + extra_survivors.extend(unknown_mysql) + mysql_infos = [info for info in mysql_infos if matches(info) is True] + if mysql_infos: env = environment(value) env['MYSQL_PWD'] = value['rootPassword'] - with contextlib.suppress(OSError, subprocess.SubprocessError): - subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env, - capture_output=True, timeout=60) + try: + shutdown = subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env, + capture_output=True, timeout=60) + if shutdown.returncode: + cleanup_failures.append('databaseShutdown') + except (OSError, subprocess.SubprocessError): + cleanup_failures.append('databaseShutdown') # mysqladmin can return before InnoDB has finished closing files. - for info in children: - if info['name'] == 'mysql' and matches(info): - with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired): + for info in mysql_infos: + mysql_status = matches(info) + if mysql_status is None: + extra_survivors.append(info) + elif mysql_status is True: + try: psutil.Process(info['pid']).wait(60) - survivors = [info for info in children if matches(info)] - if survivors: - state['children'] = survivors + except (psutil.NoSuchProcess, psutil.TimeoutExpired): + pass + except Exception: + extra_survivors.append(info) + unique_records, seen_records = [], set() + for info in [*children, *extra_survivors]: + if not isinstance(info, dict): + unknown_children = True + continue + key = (info.get('pid'), info.get('created')) + if key not in seen_records: + seen_records.add(key) + unique_records.append(info) + survivors, unknown_children = retain_registered_children(state, unique_records, unknown_children) + if cleanup_failures: + state['cleanupFailures'] = cleanup_failures + if survivors or unknown_children or cleanup_failures: atomic(STATE, state) else: STATE.unlink(missing_ok=True) @@ -402,10 +769,25 @@ def serve(database_only=False): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve')) + parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'verify-runtime', 'serve')) parser.add_argument('--lan', action='store_true') + parser.add_argument('--program-only', action='store_true') parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS) args = parser.parse_args() + if args.program_only and args.action != 'verify-runtime': + parser.error('--program-only 只能与 verify-runtime 一起使用。') + if args.action == 'verify-runtime': + verify_runtime(program_only=args.program_only) + print('程序文件与 AI 应用脚本检查通过。' if args.program_only else + '文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。') + return + if args.action == 'start': + initial_state = read_state() + if 'supervisor' in initial_state and matches(initial_state['supervisor']) is None: + raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。') + _children, unknown_children = registered_alive(initial_state.get('children', [])) + if unknown_children or initial_state.get('unknownChildren') or initial_state.get('cleanupFailures'): + raise RuntimeError('发现未确认停止的已记录进程,未启动任何服务。') if args.action in ('start', 'serve', 'verify'): private_directory(PRIVATE) private_directory(DATA) @@ -426,23 +808,24 @@ def main(): message = None raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。') state = read_state() - running = matches(state.get('supervisor', {})) + running = matches(state['supervisor']) if 'supervisor' in state else False if args.action == 'status': + if running is None: + print('无法确认服务状态') + return print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止') return + if running is None: + raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。') if args.action == 'stop': if not running: - if any(matches(info) for info in state.get('children', [])): - raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。') + children, unknown_children = registered_alive(state.get('children', [])) + if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'): + raise RuntimeError('发现上次退出后仍有未确认停止的进程,请根据日志检查;未结束任何不明进程。') print('服务已停止。') return - atomic(STOP, {'token': state['token']}) - for _ in range(240): - if not matches(state['supervisor']): - print('服务已停止。') - return - time.sleep(.5) - raise RuntimeError('服务仍在安全关闭,请稍后检查状态。') + wait_for_stop(state['supervisor'], {'token': state['token']}) + return if running: if args.lan and configuration()['webBind'] != '0.0.0.0': raise RuntimeError('请先停止服务,再使用 --lan 启动。') @@ -450,7 +833,8 @@ def main(): for url in addresses(configuration()): print(url) return - if any(matches(info) for info in state.get('children', [])): + children, unknown_children = registered_alive(state.get('children', [])) + if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'): raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。') value = verify() if args.action == 'verify': diff --git a/deploy/offline/repair_windows_update_check.py b/deploy/offline/repair_windows_update_check.py index 2ae643a..38300c0 100644 --- a/deploy/offline/repair_windows_update_check.py +++ b/deploy/offline/repair_windows_update_check.py @@ -1,8 +1,8 @@ -"""原地修复 Windows 更新检查与备份兼容问题,只替换已知版本程序及清单摘要。""" +"""生成兼容旧 Windows 包的自包含 first-aid 修复入口。""" from __future__ import annotations + from contextlib import contextmanager from datetime import datetime -import base64 import hashlib import json import os @@ -10,49 +10,79 @@ from pathlib import Path import tempfile import uuid -SCRIPT = 'deploy/offline/update_manager.py' -OLD_SHA = '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b' -PREVIOUS_SHA = 'b91cb7fec10e183d2904d86f032ff0ffcf7f19f7cb27aecd2d803d3448685482' -NEW_SHA = 'a4bd05789f559fede0bed083ce61123307d6b5cf52fa638b6bde1fa112a0348a' -PATCHED_SCRIPT = base64.b64decode('IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiJTaWduZWQgV2luZG93cyBzdGFuZGFsb25lIGFwcGxpY2F0aW9uIHVwZGF0ZXMuIE5vIGNhbGxlci1zdXBwbGllZCBVUkxzIG9yIHBhdGhzLgoKVGhlIHJlbGVhc2UgZW52ZWxvcGUgc2lnbnMgdGhlICpkZWNvZGVkIHBheWxvYWQgYnl0ZXMqIHdpdGggRWQyNTUxOS4gUnVudGltZQppZGVudGl0eSBpcyBTSEEyNTYoY2Fub25pY2FsKHtydW50aW1lcywgcHl0aG9uV2hlZWxzfSkpLiBQcm9ncmFtIGZpbGVzIGFyZSBhIGZ1bGwKc25hcHNob3Q7IHByaXZhdGUgZGF0YSBhbmQgcnVudGltZXMgYXJlIG5ldmVyIHVwZGF0ZSBkZXN0aW5hdGlvbnMuCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgppbXBvcnQgYXJncGFyc2UKaW1wb3J0IGJhc2U2NAppbXBvcnQgY29udGV4dGxpYgpmcm9tIGRhdGV0aW1lIGltcG9ydCBkYXRldGltZSwgdGltZXpvbmUKaW1wb3J0IGhhc2hsaWIKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmZyb20gcGF0aGxpYiBpbXBvcnQgUGF0aCwgUHVyZVBvc2l4UGF0aAppbXBvcnQgcmUKaW1wb3J0IHNlY3JldHMKaW1wb3J0IHNodXRpbAppbXBvcnQgc3RhdAppbXBvcnQgc3VicHJvY2VzcwppbXBvcnQgc3lzCmltcG9ydCB0aW1lCmltcG9ydCB0cmFjZWJhY2sKaW1wb3J0IHppcGZpbGUKZnJvbSB1cmxsaWIucmVxdWVzdCBpbXBvcnQgUHJveHlIYW5kbGVyLCBIVFRQUmVkaXJlY3RIYW5kbGVyLCBidWlsZF9vcGVuZXIKCmltcG9ydCBwc3V0aWwKZnJvbSBjcnlwdG9ncmFwaHkuaGF6bWF0LnByaW1pdGl2ZXMuc2VyaWFsaXphdGlvbiBpbXBvcnQgbG9hZF9wZW1fcHVibGljX2tleQpmcm9tIGNyeXB0b2dyYXBoeS5oYXptYXQucHJpbWl0aXZlcy5hc3ltbWV0cmljLmVkMjU1MTkgaW1wb3J0IEVkMjU1MTlQdWJsaWNLZXkKClJPT1QgPSBQYXRoKF9fZmlsZV9fKS5yZXNvbHZlKCkucGFyZW50c1syXQpCQVNFID0gJ2h0dHBzOi8vcXVhbGl0eS5seXVubGFiLm1lL3F1YWxpdHktdXBkYXRlcy93aW5kb3dzLXg2NC8nCkxBVEVTVCA9IEJBU0UgKyAnbGF0ZXN0Lmpzb24nCktFWSA9ICdkZXBsb3kvb2ZmbGluZS91cGRhdGUtcHVibGljLWtleS5wZW0nCk1BWF9NQU5JRkVTVCA9IDE2ICogMTAyNCAqIDEwMjQKTUFYX0FSQ0hJVkUgPSAyICogMTAyNCoqMwpNQVhfRVhQQU5ERUQgPSA0ICogMTAyNCoqMwpBQ1RJVkUgPSB7J3F1ZXVlZCcsICdkb3dubG9hZGluZycsICdiYWNraW5nVXAnLCAnaW5zdGFsbGluZycsICdyZXN0YXJ0aW5nJywgJ3JvbGxpbmdCYWNrJ30KU0hBID0gcmUuY29tcGlsZShyJ1thLWYwLTldezY0fScpCgoKY2xhc3MgVXBkYXRlRXJyb3IoUnVudGltZUVycm9yKToKICAgIHBhc3MKCgpjbGFzcyBCdXN5VXBkYXRlKFVwZGF0ZUVycm9yKToKICAgIHBhc3MKCgpkZWYgY2Fub25pY2FsKHZhbHVlKToKICAgIHJldHVybiBqc29uLmR1bXBzKHZhbHVlLCBlbnN1cmVfYXNjaWk9RmFsc2UsIHNvcnRfa2V5cz1UcnVlLCBzZXBhcmF0b3JzPSgnLCcsICc6JykpLmVuY29kZSgndXRmLTgnKQoKCmRlZiB1bmlxdWVfb2JqZWN0KHBhaXJzKToKICAgIHZhbHVlID0ge30KICAgIGZvciBrZXksIGl0ZW0gaW4gcGFpcnM6CiAgICAgICAgaWYga2V5IGluIHZhbHVlOgogICAgICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5riF5Y2V5YyF5ZCr6YeN5aSN5a2X5q6144CCJykKICAgICAgICB2YWx1ZVtrZXldID0gaXRlbQogICAgcmV0dXJuIHZhbHVlCgoKZGVmIGRlY29kZShyYXcpOgogICAgcmV0dXJuIGpzb24ubG9hZHMocmF3LCBvYmplY3RfcGFpcnNfaG9vaz11bmlxdWVfb2JqZWN0KQoKCmRlZiBkaWdlc3QocGF0aCk6CiAgICB3aXRoIHBhdGgub3BlbigncmInKSBhcyBzdHJlYW06CiAgICAgICAgcmV0dXJuIGhhc2hsaWIuZmlsZV9kaWdlc3Qoc3RyZWFtLCAnc2hhMjU2JykuaGV4ZGlnZXN0KCkKCgpkZWYgYXRvbWljKHBhdGgsIHZhbHVlKToKICAgIHBhdGgucGFyZW50Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSwgbW9kZT0wbzcwMCkKICAgIHRlbXBvcmFyeSA9IHBhdGgud2l0aF9uYW1lKHBhdGgubmFtZSArICcuJyArIHNlY3JldHMudG9rZW5faGV4KDYpICsgJy50bXAnKQogICAgd2l0aCB0ZW1wb3Jhcnkub3BlbigneGInKSBhcyBzdHJlYW06CiAgICAgICAgc3RyZWFtLndyaXRlKGNhbm9uaWNhbCh2YWx1ZSkpCiAgICAgICAgc3RyZWFtLmZsdXNoKCkKICAgICAgICBvcy5mc3luYyhzdHJlYW0uZmlsZW5vKCkpCiAgICB0ZW1wb3JhcnkucmVwbGFjZShwYXRoKQoKCmRlZiBjb3B5X2F0b21pYyhzb3VyY2UsIHRhcmdldCk6CiAgICBzb3VyY2UsIHRhcmdldCA9IFBhdGgoc291cmNlKSwgUGF0aCh0YXJnZXQpCiAgICB0YXJnZXQucGFyZW50Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKICAgIHRlbXBvcmFyeSA9IHRhcmdldC53aXRoX25hbWUodGFyZ2V0Lm5hbWUgKyAnLnVwZGF0ZS0nICsgc2VjcmV0cy50b2tlbl9oZXgoNikpCiAgICB0cnk6CiAgICAgICAgd2l0aCBzb3VyY2Uub3BlbigncmInKSBhcyByZWFkZXIsIHRlbXBvcmFyeS5vcGVuKCd4YicpIGFzIHdyaXRlcjoKICAgICAgICAgICAgc2h1dGlsLmNvcHlmaWxlb2JqKHJlYWRlciwgd3JpdGVyKQogICAgICAgICAgICB3cml0ZXIuZmx1c2goKQogICAgICAgICAgICBvcy5mc3luYyh3cml0ZXIuZmlsZW5vKCkpCiAgICAgICAgc2h1dGlsLmNvcHlzdGF0KHNvdXJjZSwgdGVtcG9yYXJ5KQogICAgICAgIHRlbXBvcmFyeS5yZXBsYWNlKHRhcmdldCkKICAgIGZpbmFsbHk6CiAgICAgICAgdGVtcG9yYXJ5LnVubGluayhtaXNzaW5nX29rPVRydWUpCgoKZGVmIHJlYWQocGF0aCwgZGVmYXVsdD1Ob25lKToKICAgIHJldHVybiBkZWNvZGUocGF0aC5yZWFkX2J5dGVzKCkpIGlmIHBhdGguZXhpc3RzKCkgZWxzZSBkZWZhdWx0CgoKZGVmIHJ1bnRpbWVfZmluZ2VycHJpbnQobWFuaWZlc3QpOgogICAgcmV0dXJuIGhhc2hsaWIuc2hhMjU2KGNhbm9uaWNhbCh7a2V5OiBtYW5pZmVzdC5nZXQoa2V5LCBbXSkgZm9yIGtleSBpbiAoJ3J1bnRpbWVzJywgJ3B5dGhvbldoZWVscycpfSkpLmhleGRpZ2VzdCgpCgoKZGVmIHNhZmVfcGF0aChuYW1lKToKICAgIGlmIChub3QgaXNpbnN0YW5jZShuYW1lLCBzdHIpIG9yIG5vdCBuYW1lIG9yIGxlbihuYW1lKSA+IDIzMCBvciAnXFwnIGluIG5hbWUKICAgICAgICAgICAgb3IgJzonIGluIG5hbWUgb3IgJ1x4MDAnIGluIG5hbWUgb3IgbmFtZS5zdGFydHN3aXRoKCcvJykKICAgICAgICAgICAgb3IgYW55KG9yZChjKSA8IDMyIGZvciBjIGluIG5hbWUpKToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5paH5Lu26Lev5b6E5peg5pWI44CCJykKICAgIHBhcnRzID0gbmFtZS5zcGxpdCgnLycpCiAgICBpZiBhbnkocCBpbiAoJycsICcuJywgJy4uJykgb3IgcC5lbmRzd2l0aCgoJyAnLCAnLicpKQogICAgICAgICAgIG9yIHJlLmZ1bGxtYXRjaChyJyg/aSkoY29ufHBybnxhdXh8bnVsfGNvbVsxLTldfGxwdFsxLTldKSg/OlwuLiopPycsIHApIGZvciBwIGluIHBhcnRzKToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5paH5Lu26Lev5b6E5peg5pWI44CCJykKICAgIHJldHVybiBQdXJlUG9zaXhQYXRoKG5hbWUpCgoKZGVmIHByb2dyYW1fcGF0aChuYW1lKToKICAgIHBhdGggPSBzYWZlX3BhdGgobmFtZSkKICAgIGlmIG5hbWUgPT0gS0VZIG9yIG5hbWUuc3RhcnRzd2l0aCgnZGVwbG95L2Jvb3RzdHJhcC8nKToKICAgICAgICByZXR1cm4gRmFsc2UgICMgaW1tdXRhYmxlIHRydXN0IGFuY2hvciwgZXZlbiBpbiBhIGNvcnJlY3RseSBzaWduZWQgcmVsZWFzZQogICAgcmV0dXJuIChuYW1lID09ICdhcHAvYmFja2VuZC5qYXInIG9yIG5hbWUuc3RhcnRzd2l0aCgnYXBwL3dlYi8nKQogICAgICAgICAgICBvciBuYW1lID09ICdiYWNrZW5kL3NyYy9tYWluL3Jlc291cmNlcy9hbmFseXNpcy1ydW50aW1lLXBvbGljeS5qc29uJwogICAgICAgICAgICBvciBuYW1lLnN0YXJ0c3dpdGgoJ3NjcmlwdHMvJykgYW5kIHBhdGguc3VmZml4ID09ICcucHknCiAgICAgICAgICAgIG9yIG5hbWUuc3RhcnRzd2l0aCgnYWkvbmFvLXF1YWxpdHkvJykKICAgICAgICAgICAgb3IgbmFtZS5zdGFydHN3aXRoKCdkZXBsb3kvb2ZmbGluZS8nKSBhbmQgcGF0aC5zdWZmaXggaW4gKCcucHknLCAnLnBzMScsICcubWQnKSkKCgpkZWYgZGVzdGluYXRpb24ocm9vdCwgbmFtZSk6CiAgICBzYWZlX3BhdGgobmFtZSkKICAgIHRhcmdldCA9IHJvb3Quam9pbnBhdGgoKm5hbWUuc3BsaXQoJy8nKSkKICAgIGZvciBwYXRoIGluICh0YXJnZXQsICp0YXJnZXQucGFyZW50cyk6CiAgICAgICAgaWYgcGF0aCA9PSByb290OgogICAgICAgICAgICBicmVhawogICAgICAgIGlmIHBhdGguaXNfc3ltbGluaygpOgogICAgICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw55uu5b2V5YyF5ZCr56ym5Y+36ZO+5o6l77yM5bey5YGc5q2i5pu05paw44CCJykKICAgICAgICBpZiBvcy5uYW1lID09ICdudCcgYW5kIHBhdGguZXhpc3RzKCkgYW5kIGdldGF0dHIocGF0aC5sc3RhdCgpLCAnc3RfZmlsZV9hdHRyaWJ1dGVzJywgMCkgJiBzdGF0LkZJTEVfQVRUUklCVVRFX1JFUEFSU0VfUE9JTlQ6CiAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDnm67lvZXljIXlkKvph43op6PmnpDngrnvvIzlt7LlgZzmraLmm7TmlrDjgIInKQogICAgaWYgbm90IHRhcmdldC5yZXNvbHZlKCkuaXNfcmVsYXRpdmVfdG8ocm9vdC5yZXNvbHZlKCkpOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmlofku7botoXlh7rnqIvluo/nm67lvZXjgIInKQogICAgcmV0dXJuIHRhcmdldAoKCmRlZiBlbnRyeV9tYXAoZW50cmllcywgcHJvZ3JhbV9vbmx5PUZhbHNlKToKICAgIGlmIG5vdCBpc2luc3RhbmNlKGVudHJpZXMsIGxpc3QpIG9yIGxlbihlbnRyaWVzKSA+IDUwMDAwOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmlofku7bmuIXljZXml6DmlYjjgIInKQogICAgcmVzdWx0LCBmb2xkZWQgPSB7fSwgc2V0KCkKICAgIGZvciBlbnRyeSBpbiBlbnRyaWVzOgogICAgICAgIGlmIG5vdCBpc2luc3RhbmNlKGVudHJ5LCBkaWN0KToKICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOaWh+S7tua4heWNleaXoOaViOOAgicpCiAgICAgICAgbmFtZSA9IGVudHJ5LmdldCgncGF0aCcpCiAgICAgICAgc2FmZV9wYXRoKG5hbWUpCiAgICAgICAgaWYgbmFtZS5jYXNlZm9sZCgpIGluIGZvbGRlZDoKICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOaWh+S7tui3r+W+hOmHjeWkjeOAgicpCiAgICAgICAgZm9sZGVkLmFkZChuYW1lLmNhc2Vmb2xkKCkpCiAgICAgICAgaWYgcHJvZ3JhbV9vbmx5IGFuZCBub3QgcHJvZ3JhbV9wYXRoKG5hbWUpOgogICAgICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5YyF5ZCr5LiN5YWB6K645pu/5o2i55qE5paH5Lu244CCJykKICAgICAgICBpZiBlbnRyeS5nZXQoJ3N5bWxpbmsnKSBpcyBub3QgTm9uZSBvciBub3QgU0hBLmZ1bGxtYXRjaChzdHIoZW50cnkuZ2V0KCdzaGEyNTYnLCAnJykpKToKICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOaWh+S7tuagoemqjOS/oeaBr+aXoOaViOOAgicpCiAgICAgICAgcmVzdWx0W25hbWVdID0gZW50cnkKICAgIHJldHVybiByZXN1bHQKCgpkZWYgdmFsaWRhdGVfcmVsZWFzZShlbnZlbG9wZSwgcm9vdD1ST09UKToKICAgIHRyeToKICAgICAgICByYXcgPSBiYXNlNjQuYjY0ZGVjb2RlKGVudmVsb3BlWydwYXlsb2FkJ10sIHZhbGlkYXRlPVRydWUpCiAgICAgICAgc2lnbmF0dXJlID0gYmFzZTY0LmI2NGRlY29kZShlbnZlbG9wZVsnc2lnbmF0dXJlJ10sIHZhbGlkYXRlPVRydWUpCiAgICAgICAga2V5ID0gbG9hZF9wZW1fcHVibGljX2tleShkZXN0aW5hdGlvbihyb290LCBLRVkpLnJlYWRfYnl0ZXMoKSkKICAgICAgICBpZiBub3QgaXNpbnN0YW5jZShrZXksIEVkMjU1MTlQdWJsaWNLZXkpOgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCdrZXkgdHlwZScpCiAgICAgICAga2V5LnZlcmlmeShzaWduYXR1cmUsIHJhdykKICAgICAgICByZWxlYXNlID0gZGVjb2RlKHJhdykKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXJyb3I6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOetvuWQjemqjOivgeWksei0peOAgicpIGZyb20gZXJyb3IKICAgIGN1cnJlbnQgPSByZWFkKHJvb3QgLyAnbWFuaWZlc3QuanNvbicpCiAgICBpZiByZWxlYXNlLmdldCgnc2NoZW1hJykgIT0gMSBvciByZWxlYXNlLmdldCgncGxhdGZvcm0nKSAhPSAnd2luZG93cy14NjQnOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDljIXkuI3pgILnlKjkuo7mraTns7vnu5/jgIInKQogICAgaWYgbm90IGlzaW5zdGFuY2UocmVsZWFzZS5nZXQoJ3ZlcnNpb24nKSwgc3RyKSBvciBub3QgcmUuZnVsbG1hdGNoKHInW0EtWmEtejAtOV1bQS1aYS16MC05Ll8tXXswLDc5fScsIHJlbGVhc2VbJ3ZlcnNpb24nXSk6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOeJiOacrOaXoOaViOOAgicpCiAgICBpZiB0eXBlKHJlbGVhc2UuZ2V0KCdzZXF1ZW5jZScpKSBpcyBub3QgaW50IG9yIHJlbGVhc2VbJ3NlcXVlbmNlJ10gPD0gMDoKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw54mI5pys5bqP5Y+35peg5pWI44CCJykKICAgIGlmIHJlbGVhc2UuZ2V0KCdydW50aW1lRmluZ2VycHJpbnQnKSAhPSBydW50aW1lX2ZpbmdlcnByaW50KGN1cnJlbnQpOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfov5DooYznjq/looPniYjmnKzkuI3lkIzvvIzor7fkvb/nlKjlrozmlbTlronoo4XljIXmm7TmlrDjgIInKQogICAgYXJjaGl2ZSA9IHJlbGVhc2UuZ2V0KCdhcmNoaXZlJywge30pCiAgICBpZiAobm90IHJlLmZ1bGxtYXRjaChyJ3F1YWxpdHktdXBkYXRlLVtBLVphLXowLTldW0EtWmEtejAtOS5fLV17MCwxMDB9XC56aXAnLCBzdHIoYXJjaGl2ZS5nZXQoJ2ZpbGUnLCAnJykpKQogICAgICAgICAgICBvciB0eXBlKGFyY2hpdmUuZ2V0KCdzaXplJykpIGlzIG5vdCBpbnQgb3Igbm90IDAgPCBhcmNoaXZlWydzaXplJ10gPD0gTUFYX0FSQ0hJVkUKICAgICAgICAgICAgb3Igbm90IFNIQS5mdWxsbWF0Y2goc3RyKGFyY2hpdmUuZ2V0KCdzaGEyNTYnLCAnJykpKSk6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOS4i+i9veS/oeaBr+aXoOaViOOAgicpCiAgICBmaWxlcyA9IGVudHJ5X21hcChyZWxlYXNlLmdldCgnZmlsZXMnKSwgVHJ1ZSkKICAgIGlmIG5vdCBmaWxlcyBvciBhbnkodHlwZShlLmdldCgnc2l6ZScpKSBpcyBub3QgaW50IG9yIGVbJ3NpemUnXSA8IDAgZm9yIGUgaW4gZmlsZXMudmFsdWVzKCkpIG9yIHN1bShlWydzaXplJ10gZm9yIGUgaW4gZmlsZXMudmFsdWVzKCkpID4gTUFYX0VYUEFOREVEOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmlofku7blpKflsI/ml6DmlYjjgIInKQogICAgaWYgJ2J1bmRsZU1hbmlmZXN0JyBpbiByZWxlYXNlOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmuIXljZXkuI3og73ljIXlkKvpg6jnvbLmlbDmja7jgIInKQogICAgb2xkX2ZpbGVzID0gZW50cnlfbWFwKGN1cnJlbnQuZ2V0KCdmaWxlcycpKQogICAgcmVxdWlyZWQgPSB7J2FwcC9iYWNrZW5kLmphcicsICdkZXBsb3kvb2ZmbGluZS9jb250cm9sLnB5JywgJ2RlcGxveS9vZmZsaW5lL3VwZGF0ZV9tYW5hZ2VyLnB5JywKICAgICAgICAgICAgICAgICdiYWNrZW5kL3NyYy9tYWluL3Jlc291cmNlcy9hbmFseXNpcy1ydW50aW1lLXBvbGljeS5qc29uJ30KICAgIGlmIG5vdCByZXF1aXJlZC5pc3N1YnNldChmaWxlcykgb3Igbm90IGFueShuYW1lLnN0YXJ0c3dpdGgoJ2FwcC93ZWIvJykgZm9yIG5hbWUgaW4gZmlsZXMpOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDnvLrlsJHlv4XopoHnqIvluo/mlofku7bjgIInKQogICAgaWYgS0VZIG5vdCBpbiBvbGRfZmlsZXMgb3IgZGlnZXN0KGRlc3RpbmF0aW9uKHJvb3QsIEtFWSkpICE9IG9sZF9maWxlc1tLRVldWydzaGEyNTYnXToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5L+h5Lu75YWs6ZKl5qCh6aqM5aSx6LSl44CCJykKICAgICMg5Y+q5o6i5rWL5pys5qyh5Lya5aSH5Lu944CB5Yig6Zmk5oiW5pu/5o2i55qE56iL5bqP6Lev5b6E77yb6L+Q6KGM546v5aKD5L+d5oyB5Y6f5qC344CCCiAgICAjIOWujOaVtOa4heWNleS7jemAkOmhuemqjOivgei3r+W+hOivreazleOAgemHjeWkjemhueOAgeaRmOimgeS7peWPiui/kOihjOeOr+Wig+aMh+e6ueOAggogICAgb2xkX3Byb2dyYW0gPSB7bmFtZSBmb3IgbmFtZSBpbiBvbGRfZmlsZXMgaWYgcHJvZ3JhbV9wYXRoKG5hbWUpfQogICAgZm9yIG5hbWUgaW4gb2xkX3Byb2dyYW0gfCBzZXQoZmlsZXMpOgogICAgICAgIGRlc3RpbmF0aW9uKHJvb3QsIG5hbWUpCiAgICByZXR1cm4gcmVsZWFzZQoKCmNsYXNzIE5vUmVkaXJlY3QoSFRUUFJlZGlyZWN0SGFuZGxlcik6CiAgICBkZWYgcmVkaXJlY3RfcmVxdWVzdChzZWxmLCByZXEsIGZwLCBjb2RlLCBtc2csIGhlYWRlcnMsIG5ld3VybCk6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOacjeWKoeWZqOi/lOWbnuS6huS4jeWFgeiuuOeahOi3s+i9rOOAgicpCgoKZGVmIGRvd25sb2FkKHVybCwgdGFyZ2V0PU5vbmUsIGV4cGVjdGVkX3NpemU9Tm9uZSwgZXhwZWN0ZWRfaGFzaD1Ob25lKToKICAgIGlmIHVybCAhPSBMQVRFU1QgYW5kIG5vdCAodXJsLnN0YXJ0c3dpdGgoQkFTRSkgYW5kIHJlLmZ1bGxtYXRjaChyJ3F1YWxpdHktdXBkYXRlLVtBLVphLXowLTldW0EtWmEtejAtOS5fLV17MCwxMDB9XC56aXAnLCB1cmxbbGVuKEJBU0UpOl0pKToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5LiL6L295Zyw5Z2A5peg5pWI44CCJykKICAgIGxpbWl0ID0gZXhwZWN0ZWRfc2l6ZSBpZiBleHBlY3RlZF9zaXplIGlzIG5vdCBOb25lIGVsc2UgTUFYX01BTklGRVNUCiAgICBoYXNoZXIsIGNvdW50LCBjaHVua3MgPSBoYXNobGliLnNoYTI1NigpLCAwLCBbXQogICAgd2l0aCBidWlsZF9vcGVuZXIoUHJveHlIYW5kbGVyKHt9KSwgTm9SZWRpcmVjdCgpKS5vcGVuKHVybCwgdGltZW91dD0zMCkgYXMgcmVzcG9uc2U6CiAgICAgICAgaWYgcmVzcG9uc2UuZ2V0dXJsKCkgIT0gdXJsIG9yIHJlc3BvbnNlLnN0YXR1cyAhPSAyMDA6CiAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmnI3liqHlmajlk43lupTml6DmlYjjgIInKQogICAgICAgIHdpdGggdGFyZ2V0Lm9wZW4oJ3hiJykgaWYgdGFyZ2V0IGlzIG5vdCBOb25lIGVsc2UgY29udGV4dGxpYi5udWxsY29udGV4dCgpIGFzIHN0cmVhbToKICAgICAgICAgICAgd2hpbGUgYmxvY2sgOj0gcmVzcG9uc2UucmVhZCgxMDI0ICogMTAyNCk6CiAgICAgICAgICAgICAgICBjb3VudCArPSBsZW4oYmxvY2spCiAgICAgICAgICAgICAgICBpZiBjb3VudCA+IGxpbWl0OgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDkuIvovb3lpKflsI/kuI3nrKblkIjmuIXljZXjgIInKQogICAgICAgICAgICAgICAgaGFzaGVyLnVwZGF0ZShibG9jaykKICAgICAgICAgICAgICAgIGlmIHN0cmVhbSBpcyBub3QgTm9uZToKICAgICAgICAgICAgICAgICAgICBzdHJlYW0ud3JpdGUoYmxvY2spCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNodW5rcy5hcHBlbmQoYmxvY2spCiAgICBpZiBleHBlY3RlZF9zaXplIGlzIG5vdCBOb25lIGFuZCAoY291bnQgIT0gZXhwZWN0ZWRfc2l6ZSBvciBoYXNoZXIuaGV4ZGlnZXN0KCkgIT0gZXhwZWN0ZWRfaGFzaCk6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOS4i+i9veagoemqjOWksei0peOAgicpCiAgICByZXR1cm4gYicnLmpvaW4oY2h1bmtzKQoKCmRlZiB1bnBhY2soYXJjaGl2ZSwgc3RhZ2UsIHJlbGVhc2UpOgogICAgZW50cmllcyA9IHtlWydwYXRoJ106IGUgZm9yIGUgaW4gcmVsZWFzZVsnZmlsZXMnXX0KICAgIHdpdGggemlwZmlsZS5aaXBGaWxlKGFyY2hpdmUpIGFzIHNvdXJjZToKICAgICAgICBzZWVuID0gc2V0KCkKICAgICAgICBmb3IgbWVtYmVyIGluIHNvdXJjZS5pbmZvbGlzdCgpOgogICAgICAgICAgICBuYW1lID0gbWVtYmVyLmZpbGVuYW1lCiAgICAgICAgICAgIGlmIG5hbWUgbm90IGluIGVudHJpZXMgb3IgbmFtZSBpbiBzZWVuIG9yIG1lbWJlci5pc19kaXIoKSBvciBtZW1iZXIuZmxhZ19iaXRzICYgMToKICAgICAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDljovnvKnljIXmlofku7bmuIXljZXkuI3nrKbjgIInKQogICAgICAgICAgICBzZWVuLmFkZChuYW1lKQogICAgICAgICAgICBpZiBzdGF0LlNfSVNMTksobWVtYmVyLmV4dGVybmFsX2F0dHIgPj4gMTYpIG9yIG1lbWJlci5maWxlX3NpemUgIT0gZW50cmllc1tuYW1lXVsnc2l6ZSddOgogICAgICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOWOi+e8qeWMheaWh+S7tuexu+Wei+aIluWkp+Wwj+S4jeespuOAgicpCiAgICAgICAgICAgIHRhcmdldCA9IGRlc3RpbmF0aW9uKHN0YWdlLCBuYW1lKQogICAgICAgICAgICB0YXJnZXQucGFyZW50Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKICAgICAgICAgICAgY291bnQgPSAwCiAgICAgICAgICAgIHdpdGggc291cmNlLm9wZW4obWVtYmVyKSBhcyByZWFkZXIsIHRhcmdldC5vcGVuKCd4YicpIGFzIHdyaXRlcjoKICAgICAgICAgICAgICAgIHdoaWxlIGJsb2NrIDo9IHJlYWRlci5yZWFkKDEwMjQgKiAxMDI0KToKICAgICAgICAgICAgICAgICAgICBjb3VudCArPSBsZW4oYmxvY2spCiAgICAgICAgICAgICAgICAgICAgaWYgY291bnQgPiBlbnRyaWVzW25hbWVdWydzaXplJ106CiAgICAgICAgICAgICAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDop6PljovlpKflsI/kuI3nrKbjgIInKQogICAgICAgICAgICAgICAgICAgIHdyaXRlci53cml0ZShibG9jaykKICAgICAgICAgICAgaWYgZGlnZXN0KHRhcmdldCkgIT0gZW50cmllc1tuYW1lXVsnc2hhMjU2J106CiAgICAgICAgICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw6Kej5Y6L5qCh6aqM5aSx6LSl44CCJykKICAgICAgICBpZiBzZWVuICE9IHNldChlbnRyaWVzKToKICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOWOi+e8qeWMhee8uuWwkeaWh+S7tuOAgicpCgoKZGVmIGVuYWJsZWQocm9vdD1ST09UKToKICAgIHJldHVybiBvcy5uYW1lID09ICdudCcgYW5kIG9zLmVudmlyb24uZ2V0KCdRVUFMSVRZX1VQREFURV9FTkFCTEVEJykgPT0gJ3RydWUnIGFuZCAocm9vdCAvICdydW50aW1lcy9weXRob24vcHl0aG9uLmV4ZScpLmlzX2ZpbGUoKSBhbmQgKHJvb3QgLyAnbWFuaWZlc3QuanNvbicpLmlzX2ZpbGUoKQoKCmRlZiBzdGF0dXMocm9vdD1ST09UKToKICAgIG1hbmlmZXN0ID0gcmVhZChyb290IC8gJ21hbmlmZXN0Lmpzb24nLCB7fSkKICAgIHZhbHVlID0gcmVhZChyb290IC8gJy5ydW50aW1lL3VwZGF0ZS9zdGF0dXMuanNvbicsIHt9KQogICAgYWxsb3dlZCA9IHsnc3RhdGUnLCAnbWVzc2FnZScsICdjaGVja2VkQXQnLCAndmVyc2lvbicsICdhdmFpbGFibGUnLCAnc3RhcnRlZEF0JywgJ2ZpbmlzaGVkQXQnfQogICAgcmVzdWx0ID0ge2s6IHYgZm9yIGssIHYgaW4gdmFsdWUuaXRlbXMoKSBpZiBrIGluIGFsbG93ZWR9CiAgICByZXN1bHQudXBkYXRlKGVuYWJsZWQ9ZW5hYmxlZChyb290KSwgY3VycmVudFZlcnNpb249bWFuaWZlc3QuZ2V0KCd1cGRhdGVWZXJzaW9uJywgbWFuaWZlc3QuZ2V0KCdidWlsdEF0VXRjJywgJ+W9k+WJjeeJiOacrCcpKSkKICAgIHJlc3VsdC5zZXRkZWZhdWx0KCdzdGF0ZScsICdpZGxlJykKICAgIGlmIHJlc3VsdFsnc3RhdGUnXSBpbiBBQ1RJVkUgYW5kIG5vdCBsaXZlKHJlYWQocm9vdCAvICcucnVudGltZS91cGRhdGUvd29ya2VyLmpzb24nLCB7fSkpOgogICAgICAgIHBlbmRpbmcgPSByZWFkKHJvb3QgLyAnLnJ1bnRpbWUvdXBkYXRlL3JlcXVlc3QuanNvbicsIHt9KQogICAgICAgIGpvdXJuYWwgPSByZWFkKHJvb3QgLyAnLnJ1bnRpbWUvdXBkYXRlL2pvdXJuYWwuanNvbicsIHt9KQogICAgICAgIGlmIHRpbWUudGltZSgpIC0gcGVuZGluZy5nZXQoJ2NyZWF0ZWQnLCAwKSA+IDEyMDoKICAgICAgICAgICAgcmVzdWx0LnVwZGF0ZShzdGF0ZT0nZmFpbGVkJywgYXZhaWxhYmxlPUZhbHNlLCBtZXNzYWdlPSfmm7TmlrDov5vnqIvlt7LkuK3mlq3vvIzor7fph43mlrDov5DooYzova/ku7blkK/liqjlhaXlj6PmgaLlpI3jgIInCiAgICAgICAgICAgICAgICAgICAgICAgICAgaWYgam91cm5hbC5nZXQoJ3BoYXNlJykgbm90IGluIChOb25lLCAnY29tcGxldGUnLCAncm9sbGVkQmFjaycpIGVsc2UgJ+abtOaWsOi/m+eoi+W3sue7k+adn++8jOivt+mHjeaWsOajgOafpeabtOaWsOOAgicpCiAgICBpZiBub3QgcmVzdWx0WydlbmFibGVkJ106CiAgICAgICAgcmVzdWx0WydtZXNzYWdlJ10gPSAn5q2k6YOo572y5LiN5pSv5oyB6L2v5Lu25YaF5pu05paw44CCJwogICAgcmV0dXJuIHJlc3VsdAoKCmRlZiB3cml0ZV9zdGF0dXMoc3RhdGUsIG1lc3NhZ2UsIHJvb3Q9Uk9PVCwgKipleHRyYSk6CiAgICBkZXN0aW5hdGlvbihyb290LCAnLnJ1bnRpbWUvdXBkYXRlL3N0YXR1cy5qc29uJykKICAgIHZhbHVlID0gcmVhZChyb290IC8gJy5ydW50aW1lL3VwZGF0ZS9zdGF0dXMuanNvbicsIHt9KQogICAgdmFsdWUudXBkYXRlKGV4dHJhLCBzdGF0ZT1zdGF0ZSwgbWVzc2FnZT1tZXNzYWdlKQogICAgYXRvbWljKHJvb3QgLyAnLnJ1bnRpbWUvdXBkYXRlL3N0YXR1cy5qc29uJywgdmFsdWUpCiAgICByZXR1cm4gc3RhdHVzKHJvb3QpCgoKZGVmIGNoZWNrKHJvb3Q9Uk9PVCk6CiAgICByZWxlYXNlID0gdmFsaWRhdGVfcmVsZWFzZShkZWNvZGUoZG93bmxvYWQoTEFURVNUKSksIHJvb3QpCiAgICBhdmFpbGFibGUgPSByZWxlYXNlWydzZXF1ZW5jZSddID4gcmVhZChyb290IC8gJ21hbmlmZXN0Lmpzb24nKS5nZXQoJ3VwZGF0ZVNlcXVlbmNlJywgMCkKICAgIGF0b21pYyhyb290IC8gJy5ydW50aW1lL3VwZGF0ZS9jaGVja2VkLmpzb24nLCByZWxlYXNlKQogICAgcmV0dXJuIHdyaXRlX3N0YXR1cygnYXZhaWxhYmxlJyBpZiBhdmFpbGFibGUgZWxzZSAnY3VycmVudCcsICflj5HnjrDmlrDniYjmnKzjgIInIGlmIGF2YWlsYWJsZSBlbHNlICflt7LmmK/mnIDmlrDniYjmnKzjgIInLCByb290LAogICAgICAgICAgICAgICAgICAgICAgICB2ZXJzaW9uPXJlbGVhc2VbJ3ZlcnNpb24nXSwgYXZhaWxhYmxlPWF2YWlsYWJsZSwgY2hlY2tlZEF0PWRhdGV0aW1lLm5vdyh0aW1lem9uZS51dGMpLmlzb2Zvcm1hdCgpKQoKCmRlZiBwcm9jZXNzX29wdGlvbnMoKToKICAgIHJldHVybiB7J2NyZWF0aW9uZmxhZ3MnOiBzdWJwcm9jZXNzLkRFVEFDSEVEX1BST0NFU1MgfCBzdWJwcm9jZXNzLkNSRUFURV9ORVdfUFJPQ0VTU19HUk9VUCB8IHN1YnByb2Nlc3MuQ1JFQVRFX0JSRUFLQVdBWV9GUk9NX0pPQn0gaWYgb3MubmFtZSA9PSAnbnQnIGVsc2UgeydzdGFydF9uZXdfc2Vzc2lvbic6IFRydWV9CgoKQGNvbnRleHRsaWIuY29udGV4dG1hbmFnZXIKZGVmIGV4Y2x1c2l2ZV91cGRhdGUocm9vdD1ST09UKToKICAgIGRpcmVjdG9yeSA9IHJvb3QgLyAnLnJ1bnRpbWUvdXBkYXRlJwogICAgZGVzdGluYXRpb24ocm9vdCwgJy5ydW50aW1lL3VwZGF0ZScpCiAgICBkaXJlY3RvcnkubWtkaXIocGFyZW50cz1UcnVlLCBleGlzdF9vaz1UcnVlKQogICAgaWYgbGl2ZShyZWFkKGRpcmVjdG9yeSAvICd3b3JrZXIuanNvbicsIHt9KSk6CiAgICAgICAgcmFpc2UgQnVzeVVwZGF0ZSgn5bey5pyJ5pu05paw5oiW5oGi5aSN5q2j5Zyo6L+b6KGM44CCJykKICAgIHN5cy5wYXRoLmluc2VydCgwLCBzdHIocm9vdCAvICdzY3JpcHRzJykpCiAgICBmcm9tIHJ1bnRpbWVfcGxhdGZvcm0gaW1wb3J0IGZjbnRsCiAgICB3aXRoIChkaXJlY3RvcnkgLyAnd29ya2VyLmxvY2snKS5vcGVuKCdhK2InKSBhcyBsb2NrOgogICAgICAgIHRyeToKICAgICAgICAgICAgZmNudGwuZmxvY2sobG9jay5maWxlbm8oKSwgZmNudGwuTE9DS19FWCB8IGZjbnRsLkxPQ0tfTkIpCiAgICAgICAgZXhjZXB0IChCbG9ja2luZ0lPRXJyb3IsIE9TRXJyb3IpOgogICAgICAgICAgICByYWlzZSBCdXN5VXBkYXRlKCflt7LmnInmm7TmlrDmiJbmgaLlpI3mraPlnKjov5vooYzjgIInKSBmcm9tIE5vbmUKICAgICAgICBpZGVudGl0eSA9IHsncGlkJzogb3MuZ2V0cGlkKCksICdjcmVhdGVkJzogcHN1dGlsLlByb2Nlc3MoKS5jcmVhdGVfdGltZSgpfQogICAgICAgIGF0b21pYyhkaXJlY3RvcnkgLyAnd29ya2VyLmpzb24nLCBpZGVudGl0eSkKICAgICAgICB0cnk6CiAgICAgICAgICAgIHlpZWxkCiAgICAgICAgZmluYWxseToKICAgICAgICAgICAgaWYgcmVhZChkaXJlY3RvcnkgLyAnd29ya2VyLmpzb24nLCB7fSkgPT0gaWRlbnRpdHk6CiAgICAgICAgICAgICAgICAoZGlyZWN0b3J5IC8gJ3dvcmtlci5qc29uJykudW5saW5rKG1pc3Npbmdfb2s9VHJ1ZSkKCgpkZWYgbGF1bmNoKHJvb3Q9Uk9PVCk6CiAgICB3aXRoIGV4Y2x1c2l2ZV91cGRhdGUocm9vdCk6CiAgICAgICAgcmV0dXJuIGxhdW5jaF9sb2NrZWQocm9vdCkKCgpkZWYgbGF1bmNoX2xvY2tlZChyb290PVJPT1QpOgogICAgZGlyZWN0b3J5ID0gcm9vdCAvICcucnVudGltZS91cGRhdGUnCiAgICBkZXN0aW5hdGlvbihyb290LCAnLnJ1bnRpbWUvdXBkYXRlJykKICAgIGRpcmVjdG9yeS5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCiAgICAjIFRoaXMgc2hvcnQgbGF1bmNoZXIgTVVTVCBleGl0IGJlZm9yZSBpdHMgd29ya2VyIG1heSBzdG9wIEphdmEuIFRoZSB3b3JrZXIKICAgICMgdmVyaWZpZXMgbGF1bmNoZXIgZGlzYXBwZWFyYW5jZTsgREVUQUNIRUQgYWxvbmUgZG9lcyBub3QgcmVtb3ZlIGFuY2VzdHJ5LgogICAgcGVuZGluZyA9IHJlYWQoZGlyZWN0b3J5IC8gJ3JlcXVlc3QuanNvbicsIHt9KQogICAgaWYgcGVuZGluZyBhbmQgdGltZS50aW1lKCkgLSBwZW5kaW5nLmdldCgnY3JlYXRlZCcsIDApIDwgMTIwOgogICAgICAgIHJhaXNlIEJ1c3lVcGRhdGUoJ+W3suacieabtOaWsOato+WcqOWQr+WKqOOAgicpCiAgICBjaGVjayhyb290KSAgIyByZWZldGNoIGFuZCB2ZXJpZnk7IGRvIG5vdCB0cnVzdCBicm93c2VyIG9yIGNhY2hlZCBKU09OCiAgICBpZiBub3Qgc3RhdHVzKHJvb3QpLmdldCgnYXZhaWxhYmxlJyk6CiAgICAgICAgcmV0dXJuIHN0YXR1cyhyb290KQogICAgdG9rZW4gPSBzZWNyZXRzLnRva2VuX2hleCgxNikKICAgIGF0b21pYyhkaXJlY3RvcnkgLyAncmVxdWVzdC5qc29uJywgeyd0b2tlbic6IHRva2VuLCAnY3JlYXRlZCc6IHRpbWUudGltZSgpfSkKICAgIHdyaXRlX3N0YXR1cygncXVldWVkJywgJ+abtOaWsOW3suW8gOWni++8jOWujOaIkOWQjuWwhuiHquWKqOmHjeaWsOi/nuaOpeOAgicsIHJvb3QsIHN0YXJ0ZWRBdD1kYXRldGltZS5ub3codGltZXpvbmUudXRjKS5pc29mb3JtYXQoKSwgYXZhaWxhYmxlPUZhbHNlKQogICAgdHJ5OgogICAgICAgIHdpdGggKGRpcmVjdG9yeSAvICd3b3JrZXIubG9nJykub3BlbignYWInKSBhcyBsb2c6CiAgICAgICAgICAgIHN1YnByb2Nlc3MuUG9wZW4oW3N5cy5leGVjdXRhYmxlLCBzdHIoUGF0aChfX2ZpbGVfXykucmVzb2x2ZSgpKSwgJ3dvcmtlcicsICctLWxhdW5jaGVyJywgc3RyKG9zLmdldHBpZCgpKSwgJy0tdG9rZW4nLCB0b2tlbl0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY3dkPXJvb3QsIGVudj1vcy5lbnZpcm9uLmNvcHkoKSwgc3RkaW49c3VicHJvY2Vzcy5ERVZOVUxMLCBzdGRvdXQ9bG9nLCBzdGRlcnI9bG9nLCAqKnByb2Nlc3Nfb3B0aW9ucygpKQogICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAoZGlyZWN0b3J5IC8gJ3JlcXVlc3QuanNvbicpLnVubGluayhtaXNzaW5nX29rPVRydWUpCiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+eLrOeri+abtOaWsOi/m+eoi+aXoOazleWQr+WKqO+8jOacjeWKoeS7jeWcqOi/kOihjOOAgicpIGZyb20gTm9uZQogICAgcmV0dXJuIHN0YXR1cyhyb290KQoKCmRlZiBsaXZlKGluZm8pOgogICAgdHJ5OgogICAgICAgIHByb2Nlc3MgPSBwc3V0aWwuUHJvY2VzcyhpbmZvWydwaWQnXSkKICAgICAgICByZXR1cm4gcHJvY2Vzcy5pc19ydW5uaW5nKCkgYW5kIHByb2Nlc3Muc3RhdHVzKCkgIT0gcHN1dGlsLlNUQVRVU19aT01CSUUgYW5kIGFicyhwcm9jZXNzLmNyZWF0ZV90aW1lKCkgLSBpbmZvWydjcmVhdGVkJ10pIDwgLjAxCiAgICBleGNlcHQgKHBzdXRpbC5FcnJvciwgS2V5RXJyb3IsIFR5cGVFcnJvcik6CiAgICAgICAgcmV0dXJuIEZhbHNlCgoKZGVmIGNvbnRyb2xsZXIocm9vdCwgYWN0aW9uLCBkYXRhYmFzZV9vbmx5PUZhbHNlLCB0aW1lb3V0PTExMDApOgogICAgcHl0aG9uID0gcm9vdCAvICdydW50aW1lcy9weXRob24nIC8gKCdweXRob24uZXhlJyBpZiBvcy5uYW1lID09ICdudCcgZWxzZSAnYmluL3B5dGhvbjMnKQogICAgY29tbWFuZCA9IFtzdHIocHl0aG9uKSwgc3RyKHJvb3QgLyAnZGVwbG95L29mZmxpbmUvY29udHJvbC5weScpLCBhY3Rpb25dCiAgICBpZiBkYXRhYmFzZV9vbmx5OgogICAgICAgIGNvbW1hbmQgKz0gWyctLWRhdGFiYXNlLW9ubHknXQogICAgaWYgYWN0aW9uID09ICdzZXJ2ZSc6CiAgICAgICAgd2l0aCAocm9vdCAvICcucnVudGltZS91cGRhdGUvc2VydmljZS5sb2cnKS5vcGVuKCdhYicpIGFzIGxvZzoKICAgICAgICAgICAgcmV0dXJuIHN1YnByb2Nlc3MuUG9wZW4oY29tbWFuZCwgY3dkPXJvb3QsIGVudj1vcy5lbnZpcm9uLmNvcHkoKSwgc3RkaW49c3VicHJvY2Vzcy5ERVZOVUxMLCBzdGRvdXQ9bG9nLCBzdGRlcnI9bG9nLCAqKnByb2Nlc3Nfb3B0aW9ucygpKQogICAgcmVzdWx0ID0gc3VicHJvY2Vzcy5ydW4oY29tbWFuZCwgY3dkPXJvb3QsIGVudj1vcy5lbnZpcm9uLmNvcHkoKSwgc3RkaW49c3VicHJvY2Vzcy5ERVZOVUxMLCBzdGRvdXQ9c3VicHJvY2Vzcy5QSVBFLCBzdGRlcnI9c3VicHJvY2Vzcy5QSVBFLCB0aW1lb3V0PXRpbWVvdXQpCiAgICBpZiByZXN1bHQucmV0dXJuY29kZToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pyN5Yqh5ZCv5YGc5aSx6LSl77yM6K+35p+l55yL5pys5py65pu05paw5pel5b+X44CCJykKICAgIHJldHVybiByZXN1bHQKCgpkZWYgd2FpdF9yZWFkeShyb290LCBwcm9jZXNzLCBkYXRhYmFzZV9vbmx5PUZhbHNlKToKICAgIGRlYWRsaW5lID0gdGltZS5tb25vdG9uaWMoKSArICgxODAgaWYgZGF0YWJhc2Vfb25seSBlbHNlIDEwMDApCiAgICB3aGlsZSB0aW1lLm1vbm90b25pYygpIDwgZGVhZGxpbmU6CiAgICAgICAgc3RhdGUgPSByZWFkKHJvb3QgLyAnLnJ1bnRpbWUvc2VydmljZS5qc29uJywge30pCiAgICAgICAgaWYgc3RhdGUuZ2V0KCdyZWFkeScpIGFuZCBzdGF0ZS5nZXQoJ2RhdGFiYXNlT25seScsIEZhbHNlKSA9PSBkYXRhYmFzZV9vbmx5IGFuZCBzdGF0ZS5nZXQoJ3N1cGVydmlzb3InLCB7fSkuZ2V0KCdwaWQnKSA9PSBwcm9jZXNzLnBpZDoKICAgICAgICAgICAgcmV0dXJuCiAgICAgICAgaWYgcHJvY2Vzcy5wb2xsKCkgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDlkI7mnI3liqHlkK/liqjlpLHotKXvvIzmraPlnKjmgaLlpI3ml6fniYjmnKzjgIInKQogICAgICAgIHRpbWUuc2xlZXAoLjUpCiAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5ZCO5pyN5Yqh5ZCv5Yqo6LaF5pe277yM5q2j5Zyo5oGi5aSN5pen54mI5pys44CCJykKCgpkZWYgbXlzcWwocm9vdCwgY29uZmlnLCBiaW5hcnk9J215c3FsJywgZGF0YWJhc2U9Tm9uZSwgc3RkaW49Tm9uZSwgc3Rkb3V0PXN1YnByb2Nlc3MuUElQRSk6CiAgICBleGUgPSAnLmV4ZScgaWYgb3MubmFtZSA9PSAnbnQnIGVsc2UgJycKICAgIGNvbW1hbmQgPSBbc3RyKHJvb3QgLyAncnVudGltZXMvbXlzcWwvYmluJyAvIChiaW5hcnkgKyBleGUpKSwgJy0tbm8tZGVmYXVsdHMnLCAnLS1wcm90b2NvbD1UQ1AnLCAnLS1ob3N0PTEyNy4wLjAuMScsCiAgICAgICAgICAgICAgICctLXBvcnQ9JyArIHN0cihjb25maWdbJ215c3FsUG9ydCddKSwgJy0tdXNlcj1yb290JywgJy0tZGVmYXVsdC1jaGFyYWN0ZXItc2V0PXV0ZjhtYjQnXQogICAgaWYgYmluYXJ5ID09ICdteXNxbGR1bXAnOgogICAgICAgIGNvbW1hbmQgKz0gWyctLXNpbmdsZS10cmFuc2FjdGlvbicsICctLXJvdXRpbmVzJywgJy0tZXZlbnRzJywgJy0tdHJpZ2dlcnMnLCAnLS1oZXgtYmxvYicsICctLW5vLXRhYmxlc3BhY2VzJywgJy0tc2V0LWd0aWQtcHVyZ2VkPU9GRicsIGNvbmZpZy5nZXQoJ215c3FsRGF0YWJhc2UnLCAncXVhbGl0eV9wbGF0Zm9ybScpXQogICAgZWxpZiBkYXRhYmFzZToKICAgICAgICBjb21tYW5kLmFwcGVuZChkYXRhYmFzZSkKICAgIGlmIGJpbmFyeSA9PSAnbXlzcWwnOgogICAgICAgIGNvbW1hbmQuYXBwZW5kKCctLWNvbm5lY3QtdGltZW91dD0xMCcpCiAgICBlbnYgPSBvcy5lbnZpcm9uLmNvcHkoKQogICAgZW52WydNWVNRTF9QV0QnXSA9IGNvbmZpZ1sncm9vdFBhc3N3b3JkJ10KICAgIHJlc3VsdCA9IHN1YnByb2Nlc3MucnVuKGNvbW1hbmQsIGN3ZD1yb290LCBlbnY9ZW52LCBzdGRpbj1zdGRpbiwgc3Rkb3V0PXN0ZG91dCwgc3RkZXJyPXN1YnByb2Nlc3MuUElQRSwgdGltZW91dD0zNjAwKQogICAgaWYgcmVzdWx0LnJldHVybmNvZGU6CiAgICAgICAgZXJyb3JfbG9nID0gZGVzdGluYXRpb24ocm9vdCwgJy5ydW50aW1lL3VwZGF0ZS9kYXRhYmFzZS1lcnJvci5sb2cnKQogICAgICAgIHdpdGggZXJyb3JfbG9nLm9wZW4oJ2FiJykgYXMgbG9nOgogICAgICAgICAgICBsb2cud3JpdGUoKGJpbmFyeSArICcgZmFpbGVkXG4nKS5lbmNvZGUoKSArIHJlc3VsdC5zdGRlcnJbLTE2Mzg0Ol0gKyBiJ1xuJykKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pWw5o2u5bqT5aSH5Lu95oiW5oGi5aSN5aSx6LSl77yM5bey5L+d55WZ5pys5py65aSH5Lu95ZKM5Y6f5pWw5o2u5bqT44CCJykKCgpkZWYgc3FsKHJvb3QsIGNvbmZpZywgcXVlcnkpOgogICAgIyBzdGRpbiBtdXN0IGJlIGEgcmVhbCBmaWxlIG9uIFdpbmRvd3M7IGRvIG5vdCBwbGFjZSBwYXNzd29yZHMgaW4gYXJndW1lbnRzLgogICAgcGF0aCA9IHJvb3QgLyAnLnJ1bnRpbWUvdXBkYXRlL3F1ZXJ5LnNxbCcKICAgIHRyeToKICAgICAgICBwYXRoLndyaXRlX3RleHQocXVlcnksIGVuY29kaW5nPSd1dGYtOCcpCiAgICAgICAgd2l0aCBwYXRoLm9wZW4oJ3JiJykgYXMgc291cmNlOgogICAgICAgICAgICBteXNxbChyb290LCBjb25maWcsIHN0ZGluPXNvdXJjZSkKICAgIGZpbmFsbHk6CiAgICAgICAgcGF0aC51bmxpbmsobWlzc2luZ19vaz1UcnVlKQoKCmRlZiBiYWNrdXBfcHJvZ3JhbShyb290LCBiYWNrdXAsIHJlbGVhc2UpOgogICAgY3VycmVudCA9IHJlYWQocm9vdCAvICdtYW5pZmVzdC5qc29uJykKICAgIG5hbWVzID0gc29ydGVkKHtlWydwYXRoJ10gZm9yIGUgaW4gY3VycmVudFsnZmlsZXMnXSBpZiBwcm9ncmFtX3BhdGgoZVsncGF0aCddKX0gfCB7ZVsncGF0aCddIGZvciBlIGluIHJlbGVhc2VbJ2ZpbGVzJ119KQogICAgcHJlc2VudCA9IFtdCiAgICBmb3IgbmFtZSBpbiBuYW1lczoKICAgICAgICBzb3VyY2UgPSBkZXN0aW5hdGlvbihyb290LCBuYW1lKQogICAgICAgIGlmIHNvdXJjZS5leGlzdHMoKToKICAgICAgICAgICAgdGFyZ2V0ID0gZGVzdGluYXRpb24oYmFja3VwIC8gJ3Byb2dyYW0nLCBuYW1lKQogICAgICAgICAgICB0YXJnZXQucGFyZW50Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKICAgICAgICAgICAgY29weV9hdG9taWMoc291cmNlLCB0YXJnZXQpCiAgICAgICAgICAgIHByZXNlbnQuYXBwZW5kKG5hbWUpCiAgICBjb3B5X2F0b21pYyhyb290IC8gJ21hbmlmZXN0Lmpzb24nLCBiYWNrdXAgLyAnbWFuaWZlc3QuanNvbicpCiAgICAjIEFsbCBwcml2YXRlIGNvbmZpZ3VyYXRpb24gKEFJIGtleXMsIGRlcGxveW1lbnQgY3JlZGVudGlhbHMgYW5kIGJvb3RzdHJhcAogICAgIyBmbGFncykgaXMgYmFja2VkIHVwIGxvY2FsbHksIGJ1dCBuZXZlciBiZWNvbWVzIGEgcmVsZWFzZSBwYXlsb2FkLgogICAgIyDlpI3liLbml7blnKjlhpnlj6Xmn4TkuIrliLfnm5jvvJtXaW5kb3dzIOS4jeaUr+aMgeWcqOWPquivu+WPpeafhOS4iuaJp+ihjCBfY29tbWl044CCCiAgICBzaHV0aWwuY29weXRyZWUocm9vdCAvICcucnVudGltZScsIGJhY2t1cCAvICdwcml2YXRlJywgaWdub3JlPXNodXRpbC5pZ25vcmVfcGF0dGVybnMoJ3VwZGF0ZScsICcqLmxvY2snLCAnKi5sb2cnLCAnc2VydmljZS5qc29uJywgJ3N0b3AtcmVxdWVzdC5qc29uJyksIHN5bWxpbmtzPUZhbHNlLCBjb3B5X2Z1bmN0aW9uPWNvcHlfYXRvbWljKQogICAgYXRvbWljKGJhY2t1cCAvICdwcm9ncmFtLmpzb24nLCB7J2FsbCc6IG5hbWVzLCAncHJlc2VudCc6IHByZXNlbnR9KQoKCmRlZiByZXBsYWNlX3Byb2dyYW0ocm9vdCwgc3RhZ2UsIGJhY2t1cCwgcmVsZWFzZSk6CiAgICBjdXJyZW50ID0gcmVhZChyb290IC8gJ21hbmlmZXN0Lmpzb24nKQogICAgbmFtZXMgPSByZWFkKGJhY2t1cCAvICdwcm9ncmFtLmpzb24nKVsnYWxsJ10KICAgIHdhbnRlZCA9IHtlWydwYXRoJ10gZm9yIGUgaW4gcmVsZWFzZVsnZmlsZXMnXX0KICAgIGZvciBuYW1lIGluIG5hbWVzOgogICAgICAgIHRhcmdldCA9IGRlc3RpbmF0aW9uKHJvb3QsIG5hbWUpCiAgICAgICAgaWYgbmFtZSBpbiB3YW50ZWQ6CiAgICAgICAgICAgIGNvcHlfYXRvbWljKHN0YWdlIC8gbmFtZSwgdGFyZ2V0KQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHRhcmdldC51bmxpbmsobWlzc2luZ19vaz1UcnVlKQogICAgY3VycmVudFsnZmlsZXMnXSA9IChbZSBmb3IgZSBpbiBjdXJyZW50WydmaWxlcyddIGlmIG5vdCBwcm9ncmFtX3BhdGgoZVsncGF0aCddKV0KICAgICAgICAgICAgICAgICAgICAgICAgKyBbeydwYXRoJzogZVsncGF0aCddLCAnc2hhMjU2JzogZVsnc2hhMjU2J119IGZvciBlIGluIHJlbGVhc2VbJ2ZpbGVzJ11dKQogICAgY3VycmVudC51cGRhdGUodXBkYXRlU2VxdWVuY2U9cmVsZWFzZVsnc2VxdWVuY2UnXSwgdXBkYXRlVmVyc2lvbj1yZWxlYXNlWyd2ZXJzaW9uJ10pCiAgICBhdG9taWMocm9vdCAvICdtYW5pZmVzdC5qc29uJywgY3VycmVudCkKCgpkZWYgcmVzdG9yZV9wcm9ncmFtKHJvb3QsIGJhY2t1cCk6CiAgICBuYW1lcyA9IHJlYWQoYmFja3VwIC8gJ3Byb2dyYW0uanNvbicpCiAgICBmb3IgbmFtZSBpbiBuYW1lc1snYWxsJ106CiAgICAgICAgdGFyZ2V0ID0gZGVzdGluYXRpb24ocm9vdCwgbmFtZSkKICAgICAgICBpZiBuYW1lIGluIG5hbWVzWydwcmVzZW50J106CiAgICAgICAgICAgIGNvcHlfYXRvbWljKGJhY2t1cCAvICdwcm9ncmFtJyAvIG5hbWUsIHRhcmdldCkKICAgICAgICBlbHNlOgogICAgICAgICAgICB0YXJnZXQudW5saW5rKG1pc3Npbmdfb2s9VHJ1ZSkKICAgIGNvcHlfYXRvbWljKGJhY2t1cCAvICdtYW5pZmVzdC5qc29uJywgZGVzdGluYXRpb24ocm9vdCwgJ21hbmlmZXN0Lmpzb24nKSkKICAgIGZvciBzb3VyY2UgaW4gKGJhY2t1cCAvICdwcml2YXRlJykucmdsb2IoJyonKToKICAgICAgICBpZiBzb3VyY2UuaXNfZmlsZSgpOgogICAgICAgICAgICByZWxhdGl2ZSA9IHNvdXJjZS5yZWxhdGl2ZV90byhiYWNrdXAgLyAncHJpdmF0ZScpLmFzX3Bvc2l4KCkKICAgICAgICAgICAgY29weV9hdG9taWMoc291cmNlLCBkZXN0aW5hdGlvbihyb290LCAnLnJ1bnRpbWUvJyArIHJlbGF0aXZlKSkKCgpkZWYgcm9sbGJhY2socm9vdCwgYmFja3VwLCBkYXRhYmFzZV9iYWNrZWRfdXApOgogICAgd3JpdGVfc3RhdHVzKCdyb2xsaW5nQmFjaycsICfmm7TmlrDmnKrlrozmiJDvvIzmraPlnKjmgaLlpI3ljp/niYjmnKzjgIInLCByb290KQogICAgY29udHJvbGxlcihyb290LCAnc3RvcCcsIHRpbWVvdXQ9MTgwKQogICAgcmVzdG9yZV9wcm9ncmFtKHJvb3QsIGJhY2t1cCkKICAgIGlmIGRhdGFiYXNlX2JhY2tlZF91cDoKICAgICAgICBjb25maWcgPSByZWFkKHJvb3QgLyAnLnJ1bnRpbWUvZGVwbG95bWVudC5qc29uJykKICAgICAgICBwcm9jZXNzID0gY29udHJvbGxlcihyb290LCAnc2VydmUnLCBUcnVlKQogICAgICAgIHdhaXRfcmVhZHkocm9vdCwgcHJvY2VzcywgVHJ1ZSkKICAgICAgICBkYXRhYmFzZSA9ICdxdWFsaXR5X3Jlc3RvcmVfJyArIHNlY3JldHMudG9rZW5faGV4KDEyKQogICAgICAgIHNxbChyb290LCBjb25maWcsIGYnQ1JFQVRFIERBVEFCQVNFIGB7ZGF0YWJhc2V9YCBDSEFSQUNURVIgU0VUIHV0ZjhtYjQgQ09MTEFURSB1dGY4bWI0XzA5MDBfYWlfY2k7JykKICAgICAgICB3aXRoIChiYWNrdXAgLyAnZGF0YWJhc2Uuc3FsJykub3BlbigncmInKSBhcyBzb3VyY2U6CiAgICAgICAgICAgIG15c3FsKHJvb3QsIGNvbmZpZywgZGF0YWJhc2U9ZGF0YWJhc2UsIHN0ZGluPXNvdXJjZSkKICAgICAgICBzcWwocm9vdCwgY29uZmlnLCBmIkdSQU5UIEFMTCBQUklWSUxFR0VTIE9OIGB7ZGF0YWJhc2V9YC4qIFRPICdxdWFsaXR5X2FwcCdAJzEyNy4wLjAuMSc7IikKICAgICAgICBjb250cm9sbGVyKHJvb3QsICdzdG9wJywgdGltZW91dD0xODApCiAgICAgICAgY29uZmlnWydteXNxbERhdGFiYXNlJ10gPSBkYXRhYmFzZQogICAgICAgIGF0b21pYyhyb290IC8gJy5ydW50aW1lL2RlcGxveW1lbnQuanNvbicsIGNvbmZpZykKICAgICAgICByZWNlaXB0X3BhdGggPSByb290IC8gJy5ydW50aW1lL21pZ3JhdGlvbi1yZWNlaXB0Lmpzb24nCiAgICAgICAgcmVjZWlwdCA9IHJlYWQocmVjZWlwdF9wYXRoLCB7fSkKICAgICAgICBpZiByZWNlaXB0IGFuZCByZWNlaXB0LmdldCgnbWlncmF0aW9uSWQnKSA9PSBjb25maWcuZ2V0KCdyZXN0b3JlZEZyb21NaWdyYXRpb24nKToKICAgICAgICAgICAgcmVjZWlwdFsnZGF0YWJhc2UnXSA9IGRhdGFiYXNlCiAgICAgICAgICAgIGF0b21pYyhyZWNlaXB0X3BhdGgsIHJlY2VpcHQpCiAgICBwcm9jZXNzID0gY29udHJvbGxlcihyb290LCAnc2VydmUnKQogICAgd2FpdF9yZWFkeShyb290LCBwcm9jZXNzKQoKCmRlZiByZWNvcmRfZmFpbHVyZShyb290LCBzdGFnZSwgZXJyb3IsIG5hbWU9J2ZhaWx1cmUuanNvbicpOgogICAgIyDku4XorrDlvZXpmLbmrrXjgIHlvILluLjnsbvlnovjgIHns7vnu5/plJnor6/noIHlkozku6PnoIHkvY3nva7vvIzkuI3orrDlvZXmraPmlofjgIHlh63mja7miJblvILluLjlj4LmlbDjgIIKICAgIHJlY2VpcHQgPSB7J3N0YWdlJzogc3RhZ2UsICdlcnJvclR5cGUnOiB0eXBlKGVycm9yKS5fX25hbWVfXywKICAgICAgICAgICAgICAgJ29jY3VycmVkQXQnOiBkYXRldGltZS5ub3codGltZXpvbmUudXRjKS5pc29mb3JtYXQoKSwKICAgICAgICAgICAgICAgJ2ZyYW1lcyc6IFt7J2ZpbGUnOiBQYXRoKGZyYW1lLmZfY29kZS5jb19maWxlbmFtZSkubmFtZSwgJ2Z1bmN0aW9uJzogZnJhbWUuZl9jb2RlLmNvX25hbWUsICdsaW5lJzogbGluZX0KICAgICAgICAgICAgICAgICAgICAgICAgICBmb3IgZnJhbWUsIGxpbmUgaW4gdHJhY2ViYWNrLndhbGtfdGIoZXJyb3IuX190cmFjZWJhY2tfXyldfQogICAgZm9yIGtleSBpbiAoJ2Vycm5vJywgJ3dpbmVycm9yJyk6CiAgICAgICAgdmFsdWUgPSBnZXRhdHRyKGVycm9yLCBrZXksIE5vbmUpCiAgICAgICAgaWYgdHlwZSh2YWx1ZSkgaXMgaW50OgogICAgICAgICAgICByZWNlaXB0W2tleV0gPSB2YWx1ZQogICAgdHJ5OgogICAgICAgIGF0b21pYyhyb290IC8gJy5ydW50aW1lL3VwZGF0ZScgLyBuYW1lLCByZWNlaXB0KQogICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICBwYXNzICAjIOiviuaWreWGmeWFpeWksei0peS4jeiDvemYu+atouWOn+acieWbnumAgOa1geeoi+OAggoKCmRlZiB3b3JrZXIobGF1bmNoZXIsIHRva2VuLCByb290PVJPT1QpOgogICAgZGlyZWN0b3J5ID0gcm9vdCAvICcucnVudGltZS91cGRhdGUnCiAgICBmb3IgXyBpbiByYW5nZSgzMDApOgogICAgICAgIGlmIG5vdCBwc3V0aWwucGlkX2V4aXN0cyhsYXVuY2hlcik6CiAgICAgICAgICAgIGJyZWFrCiAgICAgICAgdGltZS5zbGVlcCguMSkKICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOWQr+WKqOWZqOacqumAgOWHuu+8jOW3suWuieWFqOWBnOatouOAgicpCiAgICBpZiBhbnkoJ2phdmEnIGluIHAubmFtZSgpLmxvd2VyKCkgZm9yIHAgaW4gcHN1dGlsLlByb2Nlc3MoKS5wYXJlbnRzKCkpOgogICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDov5vnqIvmnKrni6znq4vlkK/liqjvvIzlt7LlronlhajlgZzmraLjgIInKQogICAgaWYgcmVhZChkaXJlY3RvcnkgLyAncmVxdWVzdC5qc29uJywge30pLmdldCgndG9rZW4nKSAhPSB0b2tlbjoKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw6K+35rGC5bey5aSx5pWI44CCJykKICAgIHdpdGggZXhjbHVzaXZlX3VwZGF0ZShyb290KToKICAgICAgICBiYWNrdXAgPSBkaXJlY3RvcnkgLyAoJ2JhY2t1cC0nICsgZGF0ZXRpbWUubm93KHRpbWV6b25lLnV0Yykuc3RyZnRpbWUoJyVZJW0lZFQlSCVNJVNaJykgKyAnLScgKyBzZWNyZXRzLnRva2VuX2hleCg0KSkKICAgICAgICBzdG9wcGVkID0gYmFja2VkX3VwID0gcHJvZ3JhbV9zYXZlZCA9IGNoYW5nZWQgPSBGYWxzZQogICAgICAgIGZhaWx1cmVfc3RhZ2UgPSAndmFsaWRhdGVSZWxlYXNlJwogICAgICAgIHRyeToKICAgICAgICAgICAgZW52ZWxvcGUgPSBkZWNvZGUoZG93bmxvYWQoTEFURVNUKSkKICAgICAgICAgICAgcmVsZWFzZSA9IHZhbGlkYXRlX3JlbGVhc2UoZW52ZWxvcGUsIHJvb3QpCiAgICAgICAgICAgIGlmIHJlbGVhc2VbJ3NlcXVlbmNlJ10gPD0gcmVhZChyb290IC8gJ21hbmlmZXN0Lmpzb24nKS5nZXQoJ3VwZGF0ZVNlcXVlbmNlJywgMCk6CiAgICAgICAgICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5q2k5pu05paw54mI5pys5bey5a6J6KOF44CCJykKICAgICAgICAgICAgYmFja3VwLm1rZGlyKCkKICAgICAgICAgICAgYXJjaGl2ZSwgc3RhZ2UgPSBiYWNrdXAgLyAndXBkYXRlLnppcCcsIGJhY2t1cCAvICdzdGFnZScKICAgICAgICAgICAgZXhwYW5kZWQgPSBzdW0oZW50cnlbJ3NpemUnXSBmb3IgZW50cnkgaW4gcmVsZWFzZVsnZmlsZXMnXSkKICAgICAgICAgICAgIyBLZWVwIGVub3VnaCBoZWFkcm9vbSBmb3IgZXh0cmFjdGlvbiwgb2xkIHByb2dyYW0gZmlsZXMgYW5kIGEKICAgICAgICAgICAgIyBsb2dpY2FsIGR1bXA7IHRoZSBkdW1wIGl0c2VsZiBtdXN0IGNvbXBsZXRlIGJlZm9yZSByZXBsYWNlbWVudC4KICAgICAgICAgICAgZGJfc2l6ZSA9IHN1bShwLnN0YXQoKS5zdF9zaXplIGZvciBwIGluIChyb290IC8gJ3J1bnRpbWVkYXRhL215c3FsJykucmdsb2IoJyonKSBpZiBwLmlzX2ZpbGUoKSkKICAgICAgICAgICAgb2xkX3Byb2dyYW1fc2l6ZSA9IHN1bSgocm9vdCAvIGVudHJ5WydwYXRoJ10pLnN0YXQoKS5zdF9zaXplIGZvciBlbnRyeSBpbiByZWFkKHJvb3QgLyAnbWFuaWZlc3QuanNvbicpWydmaWxlcyddIGlmIHByb2dyYW1fcGF0aChlbnRyeVsncGF0aCddKSBhbmQgKHJvb3QgLyBlbnRyeVsncGF0aCddKS5pc19maWxlKCkpCiAgICAgICAgICAgIHByaXZhdGVfc2l6ZSA9IHN1bShwLnN0YXQoKS5zdF9zaXplIGZvciBwIGluIChyb290IC8gJy5ydW50aW1lJykucmdsb2IoJyonKSBpZiBwLmlzX2ZpbGUoKSBhbmQgbm90IHAuaXNfcmVsYXRpdmVfdG8oZGlyZWN0b3J5KSkKICAgICAgICAgICAgaWYgc2h1dGlsLmRpc2tfdXNhZ2Uocm9vdCkuZnJlZSA8IHJlbGVhc2VbJ2FyY2hpdmUnXVsnc2l6ZSddICsgZXhwYW5kZWQgKyBvbGRfcHJvZ3JhbV9zaXplICsgcHJpdmF0ZV9zaXplICsgZGJfc2l6ZSAqIDMgKyA1MTIgKiAxMDI0KioyOgogICAgICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+ejgeebmOWJqeS9meepuumXtOS4jei2s++8jOivt+mHiuaUvuepuumXtOWQjumHjeivleOAgicpCiAgICAgICAgICAgIHdyaXRlX3N0YXR1cygnZG93bmxvYWRpbmcnLCAn5q2j5Zyo5LiL6L295bm25qCh6aqM5pu05paw44CCJywgcm9vdCkKICAgICAgICAgICAgaW5mbyA9IHJlbGVhc2VbJ2FyY2hpdmUnXQogICAgICAgICAgICBmYWlsdXJlX3N0YWdlID0gJ2Rvd25sb2FkQXJjaGl2ZScKICAgICAgICAgICAgZG93bmxvYWQoQkFTRSArIGluZm9bJ2ZpbGUnXSwgYXJjaGl2ZSwgaW5mb1snc2l6ZSddLCBpbmZvWydzaGEyNTYnXSkKICAgICAgICAgICAgZmFpbHVyZV9zdGFnZSA9ICd1bnBhY2tBcmNoaXZlJwogICAgICAgICAgICB1bnBhY2soYXJjaGl2ZSwgc3RhZ2UsIHJlbGVhc2UpCiAgICAgICAgICAgICMgUmV2YWxpZGF0ZSBpbW1lZGlhdGVseSBiZWZvcmUgYW55IHNlcnZpY2Ugb3IgZGlzayBtdXRhdGlvbi4KICAgICAgICAgICAgdmFsaWRhdGVfcmVsZWFzZShlbnZlbG9wZSwgcm9vdCkKICAgICAgICAgICAgd3JpdGVfc3RhdHVzKCdiYWNraW5nVXAnLCAn5q2j5Zyo5aSH5Lu977yM5pyN5Yqh5Y2z5bCG6YeN5paw5ZCv5Yqo44CCJywgcm9vdCkKICAgICAgICAgICAgYXRvbWljKGRpcmVjdG9yeSAvICdqb3VybmFsLmpzb24nLCB7J2JhY2t1cCc6IGJhY2t1cC5uYW1lLCAncGhhc2UnOiAnc3RvcHBpbmcnLCAnZGF0YWJhc2VCYWNrZWRVcCc6IEZhbHNlfSkKICAgICAgICAgICAgZmFpbHVyZV9zdGFnZSA9ICdzdG9wU2VydmljZScKICAgICAgICAgICAgY29udHJvbGxlcihyb290LCAnc3RvcCcsIHRpbWVvdXQ9MTgwKQogICAgICAgICAgICBzdG9wcGVkID0gVHJ1ZQogICAgICAgICAgICBmYWlsdXJlX3N0YWdlID0gJ2JhY2t1cFByb2dyYW0nCiAgICAgICAgICAgIGJhY2t1cF9wcm9ncmFtKHJvb3QsIGJhY2t1cCwgcmVsZWFzZSkKICAgICAgICAgICAgcHJvZ3JhbV9zYXZlZCA9IFRydWUKICAgICAgICAgICAgY29uZmlnID0gcmVhZChyb290IC8gJy5ydW50aW1lL2RlcGxveW1lbnQuanNvbicpCiAgICAgICAgICAgIGZhaWx1cmVfc3RhZ2UgPSAnc3RhcnRCYWNrdXBEYXRhYmFzZScKICAgICAgICAgICAgZGF0YWJhc2UgPSBjb250cm9sbGVyKHJvb3QsICdzZXJ2ZScsIFRydWUpCiAgICAgICAgICAgIHdhaXRfcmVhZHkocm9vdCwgZGF0YWJhc2UsIFRydWUpCiAgICAgICAgICAgIGZhaWx1cmVfc3RhZ2UgPSAnYmFja3VwRGF0YWJhc2UnCiAgICAgICAgICAgIHdpdGggKGJhY2t1cCAvICdkYXRhYmFzZS5zcWwnKS5vcGVuKCd4YicpIGFzIG91dHB1dDoKICAgICAgICAgICAgICAgIG15c3FsKHJvb3QsIGNvbmZpZywgJ215c3FsZHVtcCcsIHN0ZG91dD1vdXRwdXQpCiAgICAgICAgICAgICAgICBvdXRwdXQuZmx1c2goKQogICAgICAgICAgICAgICAgb3MuZnN5bmMob3V0cHV0LmZpbGVubygpKQogICAgICAgICAgICBpZiAoYmFja3VwIC8gJ2RhdGFiYXNlLnNxbCcpLnN0YXQoKS5zdF9zaXplIDwgMTAwOgogICAgICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+aVsOaNruW6k+Wkh+S7veS4uuepuu+8jOW3suWBnOatouabtOaWsOOAgicpCiAgICAgICAgICAgIGJhY2tlZF91cCA9IFRydWUKICAgICAgICAgICAgY29udHJvbGxlcihyb290LCAnc3RvcCcsIHRpbWVvdXQ9MTgwKQogICAgICAgICAgICAjIFRoZSBhY3R1YWwgZHVtcCBjYW4gZXhjZWVkIHRoZSBzb3VyY2UncyBwaHlzaWNhbCBzdG9yYWdlLiBEbyBub3QKICAgICAgICAgICAgIyByZXBsYWNlIGFueXRoaW5nIHVubGVzcyBhIHNlcGFyYXRlIHJvbGxiYWNrIGRhdGFiYXNlIHN0aWxsIGZpdHMuCiAgICAgICAgICAgIHJlcXVpcmVkX3JlY292ZXJ5ID0gbWF4KGRiX3NpemUsIChiYWNrdXAgLyAnZGF0YWJhc2Uuc3FsJykuc3RhdCgpLnN0X3NpemUgKiAzKSArIDUxMiAqIDEwMjQqKjIKICAgICAgICAgICAgaWYgc2h1dGlsLmRpc2tfdXNhZ2Uocm9vdCkuZnJlZSA8IHJlcXVpcmVkX3JlY292ZXJ5OgogICAgICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+Wkh+S7veW3suWujOaIkO+8jOS9huWJqeS9meejgeebmOepuumXtOS4jei2s+S7peWuieWFqOWbnumAgO+8jOW3suWBnOatouabtOaWsOOAgicpCiAgICAgICAgICAgIGF0b21pYyhkaXJlY3RvcnkgLyAnam91cm5hbC5qc29uJywgeydiYWNrdXAnOiBiYWNrdXAubmFtZSwgJ3BoYXNlJzogJ2luc3RhbGxpbmcnLCAnZGF0YWJhc2VCYWNrZWRVcCc6IFRydWUsCiAgICAgICAgICAgICAgICAgICAnYmFja3VwTWFuaWZlc3RTaGEyNTYnOiBkaWdlc3QoYmFja3VwIC8gJ21hbmlmZXN0Lmpzb24nKSwKICAgICAgICAgICAgICAgICAgICdwcm9ncmFtSW52ZW50b3J5U2hhMjU2JzogZGlnZXN0KGJhY2t1cCAvICdwcm9ncmFtLmpzb24nKX0pCiAgICAgICAgICAgIHdyaXRlX3N0YXR1cygnaW5zdGFsbGluZycsICfmraPlnKjlronoo4Xmm7TmlrDjgIInLCByb290KQogICAgICAgICAgICBjaGFuZ2VkID0gVHJ1ZQogICAgICAgICAgICBmYWlsdXJlX3N0YWdlID0gJ3JlcGxhY2VQcm9ncmFtJwogICAgICAgICAgICByZXBsYWNlX3Byb2dyYW0ocm9vdCwgc3RhZ2UsIGJhY2t1cCwgcmVsZWFzZSkKICAgICAgICAgICAgZmFpbHVyZV9zdGFnZSA9ICd2ZXJpZnlQcm9ncmFtJwogICAgICAgICAgICBjb250cm9sbGVyKHJvb3QsICd2ZXJpZnknLCB0aW1lb3V0PTE4MCkKICAgICAgICAgICAgd3JpdGVfc3RhdHVzKCdyZXN0YXJ0aW5nJywgJ+ato+WcqOmHjeaWsOWQr+WKqO+8jOivt+eojeWAmeOAgicsIHJvb3QpCiAgICAgICAgICAgIGZhaWx1cmVfc3RhZ2UgPSAncmVzdGFydFNlcnZpY2UnCiAgICAgICAgICAgIHByb2Nlc3MgPSBjb250cm9sbGVyKHJvb3QsICdzZXJ2ZScpCiAgICAgICAgICAgIHdhaXRfcmVhZHkocm9vdCwgcHJvY2VzcykKICAgICAgICAgICAgYXRvbWljKGRpcmVjdG9yeSAvICdqb3VybmFsLmpzb24nLCB7J2JhY2t1cCc6IGJhY2t1cC5uYW1lLCAncGhhc2UnOiAnY29tcGxldGUnLCAnZGF0YWJhc2VCYWNrZWRVcCc6IFRydWV9KQogICAgICAgICAgICB3cml0ZV9zdGF0dXMoJ2NvbXBsZXRlZCcsICfmm7TmlrDlrozmiJDjgIInLCByb290LCB2ZXJzaW9uPXJlbGVhc2VbJ3ZlcnNpb24nXSwgZmluaXNoZWRBdD1kYXRldGltZS5ub3codGltZXpvbmUudXRjKS5pc29mb3JtYXQoKSwgYXZhaWxhYmxlPUZhbHNlKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXJyb3I6CiAgICAgICAgICAgIHJlY29yZF9mYWlsdXJlKHJvb3QsIGZhaWx1cmVfc3RhZ2UsIGVycm9yKQogICAgICAgICAgICBtZXNzYWdlID0gc3RyKGVycm9yKSBpZiBpc2luc3RhbmNlKGVycm9yLCBVcGRhdGVFcnJvcikgZWxzZSAn5pu05paw5pyq5a6M5oiQ77yM6K+35p+l55yL5pys5py65pu05paw5pel5b+X44CCJwogICAgICAgICAgICBpZiBzdG9wcGVkOgogICAgICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgICAgIGlmIHByb2dyYW1fc2F2ZWQgYW5kIGNoYW5nZWQ6CiAgICAgICAgICAgICAgICAgICAgICAgIHJvbGxiYWNrKHJvb3QsIGJhY2t1cCwgYmFja2VkX3VwKQogICAgICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRyb2xsZXIocm9vdCwgJ3N0b3AnLCB0aW1lb3V0PTE4MCkKICAgICAgICAgICAgICAgICAgICAgICAgcHJvY2VzcyA9IGNvbnRyb2xsZXIocm9vdCwgJ3NlcnZlJykKICAgICAgICAgICAgICAgICAgICAgICAgd2FpdF9yZWFkeShyb290LCBwcm9jZXNzKQogICAgICAgICAgICAgICAgICAgIGF0b21pYyhkaXJlY3RvcnkgLyAnam91cm5hbC5qc29uJywgeydiYWNrdXAnOiBiYWNrdXAubmFtZSwgJ3BoYXNlJzogJ3JvbGxlZEJhY2snLCAnZGF0YWJhc2VCYWNrZWRVcCc6IGJhY2tlZF91cH0pCiAgICAgICAgICAgICAgICAgICAgbWVzc2FnZSArPSAnIOW3suaBouWkjeWOn+eJiOacrOOAgicKICAgICAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgcm9sbGJhY2tfZXJyb3I6CiAgICAgICAgICAgICAgICAgICAgcmVjb3JkX2ZhaWx1cmUocm9vdCwgJ3JvbGxiYWNrJywgcm9sbGJhY2tfZXJyb3IsICdyb2xsYmFjay1mYWlsdXJlLmpzb24nKQogICAgICAgICAgICAgICAgICAgIG1lc3NhZ2UgKz0gJyDoh6rliqjmgaLlpI3mnKrlrozmiJDvvIzor7fogZTns7vnrqHnkIblkZjkvb/nlKjmnKzmnLrmm7TmlrDlpIfku73mgaLlpI3jgIInCiAgICAgICAgICAgIHdyaXRlX3N0YXR1cygnZmFpbGVkJywgbWVzc2FnZSwgcm9vdCwgYXZhaWxhYmxlPUZhbHNlLCBmaW5pc2hlZEF0PWRhdGV0aW1lLm5vdyh0aW1lem9uZS51dGMpLmlzb2Zvcm1hdCgpKQogICAgICAgIGZpbmFsbHk6CiAgICAgICAgICAgIChkaXJlY3RvcnkgLyAncmVxdWVzdC5qc29uJykudW5saW5rKG1pc3Npbmdfb2s9VHJ1ZSkKCgpkZWYgcmVjb3Zlcihyb290PVJPT1QpOgogICAgd2l0aCBleGNsdXNpdmVfdXBkYXRlKHJvb3QpOgogICAgICAgIHJldHVybiByZWNvdmVyX2xvY2tlZChyb290KQoKCmRlZiByZWNvdmVyX2xvY2tlZChyb290PVJPT1QpOgogICAgZGlyZWN0b3J5ID0gcm9vdCAvICcucnVudGltZS91cGRhdGUnCiAgICBqb3VybmFsID0gcmVhZChkaXJlY3RvcnkgLyAnam91cm5hbC5qc29uJywge30pCiAgICBpZiBqb3VybmFsLmdldCgncGhhc2UnKSBpbiAoTm9uZSwgJ2NvbXBsZXRlJywgJ3JvbGxlZEJhY2snKToKICAgICAgICByZXR1cm4gc3RhdHVzKHJvb3QpCiAgICBuYW1lID0gam91cm5hbC5nZXQoJ2JhY2t1cCcsICcnKQogICAgaWYgbm90IHJlLmZ1bGxtYXRjaChyJ2JhY2t1cC1bMC05XXs4fVRbMC05XXs2fVotW2EtZjAtOV17OH0nLCBuYW1lKToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5pu05paw5oGi5aSN6K6w5b2V5peg5pWI44CCJykKICAgIGJhY2t1cCA9IGRpcmVjdG9yeSAvIG5hbWUKICAgIGlmIGpvdXJuYWwuZ2V0KCdwaGFzZScpID09ICdpbnN0YWxsaW5nJzoKICAgICAgICBpZiAoZGlnZXN0KGJhY2t1cCAvICdtYW5pZmVzdC5qc29uJykgIT0gam91cm5hbC5nZXQoJ2JhY2t1cE1hbmlmZXN0U2hhMjU2JykKICAgICAgICAgICAgICAgIG9yIGRpZ2VzdChiYWNrdXAgLyAncHJvZ3JhbS5qc29uJykgIT0gam91cm5hbC5nZXQoJ3Byb2dyYW1JbnZlbnRvcnlTaGEyNTYnKSk6CiAgICAgICAgICAgIHJhaXNlIFVwZGF0ZUVycm9yKCfmm7TmlrDmgaLlpI3lpIfku73moKHpqozlpLHotKXvvIzlt7Lkv53nlZnnjrDlnLrjgIInKQogICAgICAgIHJvbGxiYWNrKHJvb3QsIGJhY2t1cCwgam91cm5hbC5nZXQoJ2RhdGFiYXNlQmFja2VkVXAnKSBpcyBUcnVlKQogICAgZWxzZToKICAgICAgICBjb250cm9sbGVyKHJvb3QsICdzdG9wJywgdGltZW91dD0xODApCiAgICAgICAgcHJvY2VzcyA9IGNvbnRyb2xsZXIocm9vdCwgJ3NlcnZlJykKICAgICAgICB3YWl0X3JlYWR5KHJvb3QsIHByb2Nlc3MpCiAgICBhdG9taWMoZGlyZWN0b3J5IC8gJ2pvdXJuYWwuanNvbicsIHsqKmpvdXJuYWwsICdwaGFzZSc6ICdyb2xsZWRCYWNrJ30pCiAgICByZXR1cm4gd3JpdGVfc3RhdHVzKCdmYWlsZWQnLCAn5LiK5qyh5pu05paw5Lit5pat77yM5bey5oGi5aSN5Y6f54mI5pys44CCJywgcm9vdCwgYXZhaWxhYmxlPUZhbHNlKQoKCmRlZiBtYWluKCk6CiAgICBwYXJzZXIgPSBhcmdwYXJzZS5Bcmd1bWVudFBhcnNlcihkZXNjcmlwdGlvbj1fX2RvY19fKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgnYWN0aW9uJywgY2hvaWNlcz0oJ3N0YXR1cycsICdjaGVjaycsICdsYXVuY2gnLCAnd29ya2VyJywgJ3JlY292ZXInKSkKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoJy0tbGF1bmNoZXInLCB0eXBlPWludCkKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoJy0tdG9rZW4nKQogICAgYXJncyA9IHBhcnNlci5wYXJzZV9hcmdzKCkKICAgIGlmIGFyZ3MuYWN0aW9uID09ICdzdGF0dXMnOgogICAgICAgIHJldHVybiBzdGF0dXMoKQogICAgaWYgbm90IGVuYWJsZWQoKToKICAgICAgICByYWlzZSBVcGRhdGVFcnJvcign5q2k6YOo572y5LiN5pSv5oyB6L2v5Lu25YaF5pu05paw44CCJykKICAgIGlmIGFyZ3MuYWN0aW9uID09ICd3b3JrZXInOgogICAgICAgIGlmIG5vdCBhcmdzLmxhdW5jaGVyIG9yIG5vdCByZS5mdWxsbWF0Y2gocidbYS1mMC05XXszMn0nLCBhcmdzLnRva2VuIG9yICcnKToKICAgICAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+abtOaWsOivt+axguaXoOaViOOAgicpCiAgICAgICAgd29ya2VyKGFyZ3MubGF1bmNoZXIsIGFyZ3MudG9rZW4pCiAgICAgICAgcmV0dXJuIHN0YXR1cygpCiAgICBpZiBhcmdzLmFjdGlvbiA9PSAncmVjb3Zlcic6CiAgICAgICAgcmV0dXJuIHJlY292ZXIoKQogICAgam91cm5hbCA9IHJlYWQoUk9PVCAvICcucnVudGltZS91cGRhdGUvam91cm5hbC5qc29uJywge30pCiAgICBpZiBqb3VybmFsLmdldCgncGhhc2UnKSBub3QgaW4gKE5vbmUsICdjb21wbGV0ZScsICdyb2xsZWRCYWNrJyk6CiAgICAgICAgcmFpc2UgVXBkYXRlRXJyb3IoJ+S4iuasoeabtOaWsOS4reaWre+8jOivt+mHjeaWsOi/kOihjOi9r+S7tuWQr+WKqOWFpeWPo+aBouWkjeOAgicpCiAgICBpZiBsaXZlKHJlYWQoUk9PVCAvICcucnVudGltZS91cGRhdGUvd29ya2VyLmpzb24nLCB7fSkpOgogICAgICAgIHJhaXNlIEJ1c3lVcGRhdGUoJ+W3suacieabtOaWsOato+WcqOi/m+ihjOOAgicpCiAgICByZXR1cm4gY2hlY2soKSBpZiBhcmdzLmFjdGlvbiA9PSAnY2hlY2snIGVsc2UgbGF1bmNoKCkKCgppZiBfX25hbWVfXyA9PSAnX19tYWluX18nOgogICAgdHJ5OgogICAgICAgIHByaW50KGpzb24uZHVtcHMobWFpbigpLCBlbnN1cmVfYXNjaWk9RmFsc2UpKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICBtZXNzYWdlID0gc3RyKGVycm9yKSBpZiBpc2luc3RhbmNlKGVycm9yLCBVcGRhdGVFcnJvcikgZWxzZSAn5pu05paw5pyN5Yqh5pqC5LiN5Y+v55So77yM6K+356iN5ZCO6YeN6K+V44CCJwogICAgICAgIGlmIGVuYWJsZWQoKSBhbmQgbm90IGlzaW5zdGFuY2UoZXJyb3IsIEJ1c3lVcGRhdGUpIGFuZCBzeXMuYXJndlsxOjJdICE9IFsnc3RhdHVzJ10gYW5kIG5vdCBsaXZlKHJlYWQoUk9PVCAvICcucnVudGltZS91cGRhdGUvd29ya2VyLmpzb24nLCB7fSkpOgogICAgICAgICAgICB3cml0ZV9zdGF0dXMoJ2ZhaWxlZCcsIG1lc3NhZ2UpCiAgICAgICAgcHJpbnQoanNvbi5kdW1wcyh7J2Vycm9yJzogbWVzc2FnZX0sIGVuc3VyZV9hc2NpaT1GYWxzZSkpCiAgICAgICAgcmFpc2UgU3lzdGVtRXhpdCgxKQo=') +MANAGER = 'deploy/offline/update_manager.py' +CONTROL = 'deploy/offline/control.py' +MANAGER_SOURCE = Path(__file__).with_name('update_manager.py') +CONTROL_SOURCE = Path(__file__).with_name('control.py') -def sha(data): +# 仅接受已交付的历史源码。当前修复版本摘要在运行时由内嵌源码计算。 +OLD_MANAGER_SHA256 = frozenset({ + '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b', + 'b91cb7fec10e183d2904d86f032ff0ffcf7f19f7cb27aecd2d803d3448685482', + 'a4bd05789f559fede0bed083ce61123307d6b5cf52fa638b6bde1fa112a0348a', +}) +OLD_CONTROL_SHA256 = frozenset({ + 'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1', +}) +JOURNAL_SAFE_PHASES = frozenset({'complete', 'rolledBack'}) +JOURNAL_ACTIVE_PHASES = frozenset({ + 'stopping', 'backingUp', 'installing', 'restarting', 'rollingBack', +}) +MODULE_MARKER = b'\r\n# QUALITY_UPDATE_CHECK_REPAIR_MODULE\r\n' +PAYLOAD_MARKER = b'\r\n# QUALITY_UPDATE_CHECK_REPAIR_PAYLOAD\r\n' + + +def sha(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def local(root, name): - path = root / name +def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError('duplicate JSON key') + value[key] = item + return value + + +def local(root: Path, name: str) -> Path: + path = root.joinpath(*name.split('/')) if path.is_symlink() or not path.resolve().is_relative_to(root): raise RuntimeError('修复路径异常,已停止。') return path -def atomic(path, data): - fd, name = tempfile.mkstemp(prefix=path.name + '.repair-', suffix='.tmp', dir=path.parent) +def atomic(path: Path, data: bytes) -> None: + fd, temporary_name = tempfile.mkstemp( + prefix=path.name + '.firstaid-', suffix='.tmp', dir=path.parent, + ) + temporary = Path(temporary_name) try: with os.fdopen(fd, 'wb') as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) - os.replace(name, path) + os.replace(temporary, path) finally: - Path(name).unlink(missing_ok=True) + temporary.unlink(missing_ok=True) @contextmanager -def update_lock(root): +def update_lock(root: Path): root = Path(root).resolve() - private = local(root, '.runtime') - private.mkdir(exist_ok=True) - local(root, '.runtime/update').mkdir(exist_ok=True) - with local(root, '.runtime/update/worker.lock').open('a+b') as stream: + directory = local(root, '.runtime/update') + directory.mkdir(parents=True, exist_ok=True) + directory = local(root, '.runtime/update') + lock_path = local(root, '.runtime/update/worker.lock') + with lock_path.open('a+b') as stream: if os.name == 'nt': import msvcrt + # The updater locks byte 0 with msvcrt.locking as well. stream.seek(0) try: msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) except OSError: - raise RuntimeError('更新任务仍在运行,请等待结束后再运行本修复。') from None + raise RuntimeError('更新或恢复任务仍在运行,请等待结束后再运行本修复。') from None try: yield finally: @@ -62,60 +92,251 @@ def update_lock(root): import fcntl try: fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - raise RuntimeError('更新任务仍在运行,请等待结束后再运行本修复。') from None - yield + except OSError: + raise RuntimeError('更新或恢复任务仍在运行,请等待结束后再运行本修复。') from None + try: + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) -def repair(root): +def windows_process_active(pid: int) -> bool: + """只读查询 Windows PID;不存在返回 False,权限或 API 问题 fail closed。""" + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + kernel32.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD) + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.GetExitCodeProcess.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)) + kernel32.GetExitCodeProcess.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + kernel32.CloseHandle.restype = wintypes.BOOL + + process_query_limited_information = 0x1000 + error_invalid_parameter = 87 + still_active = 259 + handle = kernel32.OpenProcess(process_query_limited_information, False, pid) + if not handle: + error = ctypes.get_last_error() + if error == error_invalid_parameter: + return False + raise RuntimeError('无法只读确认 Windows 更新工作进程,未修改文件。') + + try: + exit_code = wintypes.DWORD() + if not kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)): + raise RuntimeError('无法读取 Windows 更新工作进程状态,未修改文件。') + return exit_code.value == still_active + finally: + if not kernel32.CloseHandle(handle): + raise RuntimeError('无法关闭 Windows 进程查询句柄,未修改文件。') + + +def worker_active(root: Path) -> bool: + worker_path = local(root, '.runtime/update/worker.json') + if not worker_path.exists(): + return False + try: + value = json.loads(worker_path.read_text(encoding='utf-8'), object_pairs_hook=unique_object) + except (OSError, ValueError, UnicodeError): + raise RuntimeError('更新工作记录无法识别,未修改文件。') from None + pid = value.get('pid') if isinstance(value, dict) else None + if type(pid) is not int or pid <= 0: + raise RuntimeError('更新工作记录无法识别,未修改文件。') + if os.name == 'nt': + return windows_process_active(pid) + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as error: + if getattr(error, 'winerror', None) == 87: + return False + raise RuntimeError('无法确认更新工作进程状态,未修改文件。') from None + return True + + +def check_journal(root: Path) -> None: + journal_path = local(root, '.runtime/update/journal.json') + if not journal_path.exists(): + return + try: + journal = json.loads(journal_path.read_text(encoding='utf-8'), object_pairs_hook=unique_object) + except (OSError, ValueError, UnicodeError): + raise RuntimeError('更新恢复记录无法识别,未修改文件。') from None + phase = journal.get('phase') if isinstance(journal, dict) else None + if isinstance(phase, str) and phase in JOURNAL_SAFE_PHASES: + return + if isinstance(phase, str) and phase in JOURNAL_ACTIVE_PHASES: + raise RuntimeError('上次更新或恢复尚未结束,请先使用原启动入口恢复。') + raise RuntimeError('更新恢复记录状态未知,未修改文件。') + + +def read_hardened_sources() -> tuple[bytes, bytes]: + for path in (MANAGER_SOURCE, CONTROL_SOURCE): + if path.is_symlink() or not path.is_file(): + raise RuntimeError('当前修复源码不完整,无法生成 first-aid 入口。') + return MANAGER_SOURCE.read_bytes(), CONTROL_SOURCE.read_bytes() + + +def check_source_state(actual: str, recorded: str, old_hashes: frozenset[str], target: str, label: str) -> None: + accepted = old_hashes | {target} + if actual not in accepted or recorded not in accepted: + raise RuntimeError(label + '版本与本修复不匹配,未覆盖现有程序。') + # 只允许原版到目标版之间的中断混合,不接纳两个互不相同的旧版本。 + if actual != recorded and target not in (actual, recorded): + raise RuntimeError(label + '源码与清单摘要不一致,未覆盖现有程序。') + + +def make_backup(root: Path, before: dict[str, bytes]) -> Path: + backup_parent = local(root, '.runtime/update-check-repair-backups') + backup_parent.mkdir(parents=True, exist_ok=True, mode=0o700) + backup_parent = local(root, '.runtime/update-check-repair-backups') + temporary = Path(tempfile.mkdtemp(prefix='.firstaid-', dir=backup_parent)) + final = backup_parent / (datetime.now().strftime('%Y%m%d-%H%M%S') + '-' + uuid.uuid4().hex[:8]) + try: + for name, data in before.items(): + target = temporary / name + with target.open('xb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, final) + except BaseException: + for child in temporary.iterdir(): + child.unlink(missing_ok=True) + temporary.rmdir() + raise + return final + + +def repair(root: Path, manager_payload: bytes | None = None, + control_payload: bytes | None = None) -> dict: root = Path(root).resolve() + if manager_payload is None or control_payload is None: + manager_payload, control_payload = read_hardened_sources() + if not isinstance(manager_payload, bytes) or not isinstance(control_payload, bytes): + raise TypeError('修复源码必须为字节。') + manifest_path = local(root, 'manifest.json') - script = local(root, SCRIPT) - if not manifest_path.is_file() or not script.is_file() or not local(root, 'deploy/bootstrap/windows-start.ps1').is_file(): + manager_path = local(root, MANAGER) + control_path = local(root, CONTROL) + if (not manifest_path.is_file() or not manager_path.is_file() or not control_path.is_file() + or not local(root, 'deploy/bootstrap/windows-start.ps1').is_file()): raise RuntimeError('请将本文件放到原 QualityCenter 目录,与“一键部署启动.cmd”并列后运行。') + with update_lock(root): - if local(root, '.runtime/update/worker.json').exists(): - raise RuntimeError('存在更新或恢复任务,请等待任务结束;不要在更新中运行修复。') - journal = local(root, '.runtime/update/journal.json') - if journal.exists() and json.loads(journal.read_text(encoding='utf-8')).get('phase') not in (None, 'complete', 'rolledBack'): - raise RuntimeError('上次更新尚未恢复完成,请先使用原启动入口恢复。') - before_manifest, before_script = manifest_path.read_bytes(), script.read_bytes() - manifest = json.loads(before_manifest.decode('utf-8-sig')) - if manifest.get('platform') != 'windows': - raise RuntimeError('本修复仅适用于 Windows 离线包。') - entries = manifest['files'] - if len({entry['path'] for entry in entries}) != len(entries): - raise RuntimeError('文件清单存在重复路径,未修改。') - selected = [entry for entry in entries if entry['path'] == SCRIPT] - actual = sha(before_script) - if len(selected) != 1 or selected[0].get('sha256') not in {OLD_SHA, PREVIOUS_SHA, NEW_SHA} or actual not in {OLD_SHA, PREVIOUS_SHA, NEW_SHA}: - raise RuntimeError('更新检查脚本版本与本修复不匹配,未覆盖现有程序。') - if actual == NEW_SHA and selected[0]['sha256'] == NEW_SHA: - return {'alreadyRepaired': True, 'changedFiles': 0} - after_script = PATCHED_SCRIPT - if sha(after_script) != NEW_SHA: - raise RuntimeError('修复内容校验失败,未修改。') - selected[0]['sha256'] = NEW_SHA - after_manifest = (json.dumps(manifest, ensure_ascii=False, indent=2) + '\n').encode('utf-8') - backup = local(root, '.runtime/update-check-repair-backups') / (datetime.now().strftime('%Y%m%d-%H%M%S') + '-' + uuid.uuid4().hex[:8]) - backup.mkdir(parents=True, mode=0o700) - (backup / 'manifest.json').write_bytes(before_manifest) - (backup / 'update_manager.py').write_bytes(before_script) - if manifest_path.read_bytes() != before_manifest or script.read_bytes() != before_script: - raise RuntimeError('文件正被其他进程更新,未执行修复。') + if worker_active(root): + raise RuntimeError('更新或恢复工作进程仍在运行,请等待结束后再运行本修复。') + check_journal(root) + + before = { + 'manifest.json': manifest_path.read_bytes(), + 'update_manager.py': manager_path.read_bytes(), + 'control.py': control_path.read_bytes(), + } try: - atomic(script, after_script) + manifest = json.loads(before['manifest.json'].decode('utf-8-sig'), object_pairs_hook=unique_object) + except (ValueError, UnicodeError): + raise RuntimeError('Windows 文件清单无法识别,未修改文件。') from None + if not isinstance(manifest, dict) or manifest.get('platform') != 'windows': + raise RuntimeError('本修复仅适用于 Windows 离线包。') + entries = manifest.get('files') + if not isinstance(entries, list) or any(not isinstance(entry, dict) for entry in entries): + raise RuntimeError('Windows 文件清单结构无法识别,未修改文件。') + names = [entry.get('path') for entry in entries] + if any(not isinstance(name, str) for name in names) or len(set(names)) != len(names): + raise RuntimeError('文件清单存在重复路径,未修改。') + + targets = { + MANAGER: (manager_path, before['update_manager.py'], manager_payload, + OLD_MANAGER_SHA256, '更新检查脚本'), + CONTROL: (control_path, before['control.py'], control_payload, + OLD_CONTROL_SHA256, '服务控制脚本'), + } + selected = {} + for name, (path, current, payload, old_hashes, label) in targets.items(): + rows = [entry for entry in entries if entry.get('path') == name] + if len(rows) != 1 or not isinstance(rows[0].get('sha256'), str): + raise RuntimeError('文件清单缺少唯一的' + label + '摘要,未修改文件。') + target_hash = sha(payload) + check_source_state(sha(current), rows[0]['sha256'], old_hashes, target_hash, label) + selected[name] = (rows[0], target_hash) + + if (before['update_manager.py'] == manager_payload + and before['control.py'] == control_payload + and all(row['sha256'] == digest for row, digest in selected.values())): + return {'alreadyRepaired': True, 'changedFiles': 0} + + for name, (path, current, payload, _old_hashes, _label) in targets.items(): + if local(root, name).read_bytes() != current: + raise RuntimeError('程序文件正被其他进程更新,未执行修复。') + if manifest_path.read_bytes() != before['manifest.json']: + raise RuntimeError('文件清单正被其他进程更新,未执行修复。') + + for name, (row, digest) in selected.items(): + row['sha256'] = digest + after_manifest = (json.dumps(manifest, ensure_ascii=False, indent=2) + '\n').encode('utf-8') + backup = make_backup(root, before) + try: + atomic(manager_path, manager_payload) + atomic(control_path, control_payload) atomic(manifest_path, after_manifest) - except BaseException: - atomic(script, before_script) - atomic(manifest_path, before_manifest) + except BaseException as original_error: + rollback_errors = [] + for path, data in ((manager_path, before['update_manager.py']), + (control_path, before['control.py']), + (manifest_path, before['manifest.json'])): + try: + atomic(path, data) + except BaseException as rollback_error: + rollback_errors.append(str(rollback_error)) + if rollback_errors: + raise RuntimeError('first-aid 写入失败,自动回滚未全部完成;原备份位于 ' + + str(backup.relative_to(root))) from original_error raise - return {'changedFiles': 2, 'backup': str(backup.relative_to(root)), 'updateCheckRepaired': True} + return { + 'changedFiles': 3, + 'backup': str(backup.relative_to(root)), + 'updateCheckRepaired': True, + 'controlRecoveryRepaired': True, + } -def command_file(): - header = '@echo off\nsetlocal\nchcp 65001 >nul\nset "PYTHONUTF8=1"\nset "PYTHONIOENCODING=utf-8"\nset "PYTHONDONTWRITEBYTECODE=1"\nif not exist "%~dp0runtimes\\python\\python.exe" (\n echo 请将本文件放入原 QualityCenter 文件夹,与一键部署启动放在一起。\n pause\n exit /b 1\n)\n"%~dp0runtimes\\python\\python.exe" -I -S -B -X utf8 -c "import sys;from pathlib import Path;p=Path(sys.argv[1]);s=p.read_text(encoding=\'utf-8\').split(chr(10)+\'# QUALITY_UPDATE_CHECK_REPAIR_PYTHON\'+chr(10),1)[1];n={\'__name__\':\'quality_update_check_repair\'};exec(compile(s,str(p),\'exec\'),n);print(n[\'repair\'](p.parent))" "%~f0"\nif errorlevel 1 (\n echo 修复未完成,请保留上方报错;数据库及私有配置未覆盖。\n pause\n exit /b 1\n)\necho 更新检查和备份兼容问题已修复,无需停止服务或重新导入数据库。\necho 返回浏览器:任务与监控 → 系统更新 → 检查更新 → 更新并重启。\npause\nexit /b\n# QUALITY_UPDATE_CHECK_REPAIR_PYTHON\n' - return header.replace('\n', '\r\n').encode('utf-8') + Path(__file__).read_bytes() +def command_file() -> bytes: + """生成单个可双击 CMD;源码字节以内嵌 Python 字节字面量保存。""" + manager_payload, control_payload = read_hardened_sources() + header = r'''@echo off +setlocal +chcp 65001 >nul +set "PYTHONUTF8=1" +set "PYTHONIOENCODING=utf-8" +set "PYTHONDONTWRITEBYTECODE=1" +if not exist "%~dp0runtimes\python\python.exe" ( + echo 请将本文件放到原 QualityCenter 目录,与一键部署启动放在一起。 + pause + exit /b 1 +) +"%~dp0runtimes\python\python.exe" -I -S -B -X utf8 -c "import sys;from pathlib import Path;p=Path(sys.argv[1]);b=p.read_bytes();m=bytes((13,10))+b'# QUALITY_UPDATE_CHECK_REPAIR_MODULE'+bytes((13,10));t=b.rsplit(m,1)[1];q=bytes((13,10))+b'# QUALITY_UPDATE_CHECK_REPAIR_PAYLOAD'+bytes((13,10));s,d=t.rsplit(q,1);n={'__name__':'quality_firstaid','__file__':str(p)};exec(compile(s.decode('utf-8'),str(p),'exec'),n);v={};exec(compile(d.decode('ascii'),str(p),'exec'),v);import json;print(json.dumps(n['repair'](p.parent,v['PATCHED_MANAGER'],v['PATCHED_CONTROL']),ensure_ascii=False))" "%~f0" +if errorlevel 1 ( + echo 修复未完成,请保留上方报错;文件和恢复备份已按修复结果保留。 + pause + exit /b 1 +) +echo 更新检查和中断恢复兼容代码已修复;服务、数据、版本信息和更新缓存未启动或导入。 +pause +exit /b +''' + output = header.replace('\n', '\r\n').encode('utf-8') + output += MODULE_MARKER + Path(__file__).read_bytes() + payload = ( + 'PATCHED_MANAGER = ' + repr(manager_payload) + '\n' + + 'PATCHED_CONTROL = ' + repr(control_payload) + '\n' + ).encode('ascii') + return output + PAYLOAD_MARKER + payload if __name__ == '__main__': diff --git a/deploy/offline/test_update_bootstrap_hardened.py b/deploy/offline/test_update_bootstrap_hardened.py new file mode 100644 index 0000000..6424ff6 --- /dev/null +++ b/deploy/offline/test_update_bootstrap_hardened.py @@ -0,0 +1,68 @@ +import types +import unittest +from pathlib import Path +from unittest.mock import patch +import test_update_manager as fixtures +import update_bootstrap as bootstrap + + +class HardenedBootstrapTests(unittest.TestCase): + def test_immutable_bootstrap_restores_hardened_executor_and_reuses_durable_database(self): + f = fixtures.UpdateTests() + f.setUp() + self.addCleanup(f.doCleanups) + root, u = f.root, fixtures.update + # Firstaid preinstalls both executors before the old-version backup is made. + for name in ('control.py', 'update_manager.py'): + relative = 'deploy/offline/' + name + (root / relative).write_bytes((Path(__file__).parent / name).read_bytes()) + manifest = u.read(root / 'manifest.json') + for entry in manifest['files']: + if entry['path'] in ('deploy/offline/control.py', 'deploy/offline/update_manager.py'): + entry['sha256'] = u.digest(root / entry['path']) + u.atomic(root / 'manifest.json', manifest) + backup = root / '.runtime/update/backup-20261105T170000Z-abcdef12' + backup.mkdir(parents=True) + u.backup_program(root, backup, f.release) + (backup / 'database.sql').write_bytes(b'-- restore snapshot\n' + b'SELECT 1;\n' * 30) + u.atomic(root / '.runtime/update/journal.json', { + 'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True, + 'databaseMayHaveChanged': True, + 'backupManifestSha256': u.digest(backup / 'manifest.json'), + 'programInventorySha256': u.digest(backup / 'program.json')}) + imports, attempts = [], [] + fail_once = True + real_spec = bootstrap.importlib.util.spec_from_file_location + def patched_spec(name, path): + spec = real_spec(name, path) + execute = spec.loader.exec_module + def load(module): + nonlocal fail_once + execute(module) + module.controller = lambda *a, **kw: types.SimpleNamespace(pid=1) + module.mysql = lambda *a, **kw: imports.append(kw['database']) + module.sql = lambda root, cfg, query: attempts.append(query) + def ready(root, process, database_only=False): + nonlocal fail_once + if not database_only and fail_once: + fail_once = False + raise module.UpdateError('simulated service restart failure') + module.wait_ready = ready + spec.loader.exec_module = load + return spec + with patch.object(bootstrap.importlib.util, 'spec_from_file_location', side_effect=patched_spec): + with self.assertRaisesRegex(RuntimeError, 'restart failure'): + bootstrap.recover(root) + chosen = u.read(root / '.runtime/deployment.json')['mysqlDatabase'] + (root / 'deploy/offline/update_manager.py').write_bytes(b'broken new updater syntax') + bootstrap.recover(root) + self.assertEqual(1, len(imports)) + self.assertEqual(1, sum('CREATE DATABASE' in query for query in attempts)) + self.assertEqual(chosen, u.read(root / '.runtime/deployment.json')['mysqlDatabase']) + self.assertEqual((Path(__file__).parent / 'update_manager.py').read_bytes(), (root / 'deploy/offline/update_manager.py').read_bytes()) + self.assertEqual('rolledBack', u.read(root / '.runtime/update/journal.json')['phase']) + self.assertEqual({'recovered': False}, bootstrap.recover(root)) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_update_environment.py b/deploy/offline/test_update_environment.py new file mode 100644 index 0000000..70b4815 --- /dev/null +++ b/deploy/offline/test_update_environment.py @@ -0,0 +1,299 @@ +"""Read-only runtime verification and safe-stop regression tests.""" +import hashlib +import importlib.util +import json +import platform +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import control + + +class ControlEnvironmentTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.original = { + key: getattr(control, key) + for key in ('ROOT', 'PRIVATE', 'DATA', 'CONFIG', 'STATE', 'STOP', 'IS_WINDOWS', + 'EXE', 'PYTHON', 'JAVA', 'MYSQL', 'MYSQLD', 'ADMIN') + } + control.ROOT = self.root + control.PRIVATE = self.root / '.runtime' + control.DATA = self.root / 'runtimedata' + control.CONFIG = control.PRIVATE / 'deployment.json' + control.STATE = control.PRIVATE / 'service.json' + control.STOP = control.PRIVATE / 'stop-request.json' + control.IS_WINDOWS = False + control.EXE = '' + control.PYTHON = self.root / 'runtimes/python/bin/python3' + control.JAVA = self.root / 'runtimes/java/bin/java' + control.MYSQL = self.root / 'runtimes/mysql/bin/mysql' + control.MYSQLD = self.root / 'runtimes/mysql/bin/mysqld' + control.ADMIN = self.root / 'runtimes/mysql/bin/mysqladmin' + self.addCleanup(self.restore_constants) + self.files = { + 'scripts/app_probe.py': b'print("probe")', + 'app/web/index.html': b'', + 'runtimes/python/bin/python3': b'python', + 'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'good source', + 'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'old timestamp cache', + } + for name, content in self.files.items(): + self.write(name, content) + entries = [{'path': name, 'sha256': hashlib.sha256(content).hexdigest()} + for name, content in self.files.items()] + self.manifest = {'platform': platform.system().lower(), 'files': entries} + (self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8') + + def restore_constants(self): + for key, value in self.original.items(): + setattr(control, key, value) + + def write(self, name, content): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + def successful_probe(self, command, **kwargs): + return type('Result', (), {'returncode': 0})() + + def test_verify_runtime_is_read_only_and_skips_only_source_verified_pyc(self): + self.write('runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc', + b'bytecode changed by Windows') + with patch.object(control.platform, 'machine', return_value='x86_64'), \ + patch.object(control.subprocess, 'run', side_effect=self.successful_probe): + control.verify_runtime() + self.assertFalse(control.CONFIG.exists()) + self.assertFalse(control.DATA.exists()) + + def test_pyc_is_not_ignored_when_its_manifest_source_hash_fails(self): + self.write('runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py', b'corrupted source') + with patch.object(control.platform, 'machine', return_value='x86_64'), \ + patch.object(control.subprocess, 'run', side_effect=self.successful_probe) as run: + with self.assertRaisesRegex(RuntimeError, '对应源码缺失或校验失败'): + control.verify_runtime() + run.assert_not_called() + + def test_legacy_bundle_verification_needs_only_control_gateway_and_runtime_platform(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + offline = root / 'deploy/offline' + offline.mkdir(parents=True) + scripts = root / 'scripts' + scripts.mkdir() + (offline / 'control.py').write_bytes(Path(control.__file__).read_bytes()) + (offline / 'gateway.py').write_text('def make_server(*args, **kwargs): return None\n', encoding='utf-8') + (scripts / 'runtime_platform.py').write_text('# minimal legacy runtime platform module\n', encoding='utf-8') + files = { + 'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'legacy source', + 'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'legacy cache', + } + entries = [] + for name, content in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()}) + (root / 'manifest.json').write_text(json.dumps({ + 'platform': platform.system().lower(), 'files': entries, + }), encoding='utf-8') + self.assertFalse((offline / 'repair_windows_cache.py').exists()) + + original_path = list(sys.path) + isolated_path = [] + for item in original_path: + resolved = Path(item or '.').resolve() + if resolved != HERE: + isolated_path.append(item) + with patch.object(sys, 'path', [str(offline), *isolated_path]), patch.dict(sys.modules): + for module_name in ('gateway', 'runtime_platform', 'repair_windows_cache'): + sys.modules.pop(module_name, None) + spec = importlib.util.spec_from_file_location('control_legacy_fixture', offline / 'control.py') + isolated_control = importlib.util.module_from_spec(spec) + spec.loader.exec_module(isolated_control) + with patch.object(isolated_control.platform, 'machine', return_value='x86_64'), \ + patch.object(isolated_control.subprocess, 'run', side_effect=self.successful_probe): + isolated_control.verify_runtime() + self.assertNotIn('repair_windows_cache', sys.modules) + self.assertFalse((root / '.runtime/deployment.json').exists()) + self.assertFalse((root / 'runtimedata').exists()) + + def test_program_only_checks_program_files_and_application_probe_only(self): + self.write('runtimes/python/bin/python3', b'changed runtime should not be rehashed') + calls = [] + + def probe(command, **kwargs): + calls.append(command) + return self.successful_probe(command, **kwargs) + + with patch.object(control.platform, 'machine', return_value='x86_64'), \ + patch.object(control.subprocess, 'run', side_effect=probe): + control.verify_runtime(program_only=True) + self.assertEqual(1, len(calls)) + self.assertIn('-c', calls[0]) + self.assertFalse(control.CONFIG.exists()) + self.assertFalse(control.DATA.exists()) + + def test_stop_waits_until_recorded_children_are_gone(self): + supervisor = {'pid': 10} + child = {'pid': 20} + state = {'supervisor': supervisor, 'children': [child]} + ticks = iter([0, 0, .2, .4]) + matches = iter([True, False, False, True, False, False, False]) + with patch.object(control, 'atomic') as atomic, \ + patch.object(control, 'read_state', return_value=state), \ + patch.object(control, 'matches', side_effect=lambda _info: next(matches)), \ + patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \ + patch.object(control.time, 'sleep'), \ + patch('builtins.print') as output: + control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0) + atomic.assert_called_once_with(control.STOP, {'token': 'safe-token'}) + output.assert_called_once_with('服务已停止。') + + def test_wait_for_stop_blocks_when_registered_supervisor_is_uninspectable(self): + supervisor = {'pid': 7301, 'created': 1.0, 'executable': '/service', 'args': []} + state = {'supervisor': supervisor, 'children': [], 'token': 'safe-token'} + control.STATE.parent.mkdir(parents=True, exist_ok=True) + control.STATE.write_text(json.dumps(state), encoding='utf-8') + with patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw( + control.psutil.AccessDenied(pid=pid))), \ + patch.object(control, 'atomic') as request, patch('builtins.print') as output: + with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'): + control.wait_for_stop(supervisor, {'token': state['token']}, timeout_seconds=1) + request.assert_not_called() + output.assert_not_called() + + def test_main_stop_blocks_when_registered_supervisor_is_uninspectable(self): + supervisor = {'pid': 7302, 'created': 1.0, 'executable': '/service', 'args': []} + control.STATE.parent.mkdir(parents=True, exist_ok=True) + control.STATE.write_text(json.dumps({ + 'supervisor': supervisor, 'children': [], 'token': 'safe-token', + }), encoding='utf-8') + with patch.object(sys, 'argv', ['control.py', 'stop']), \ + patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw( + control.psutil.AccessDenied(pid=pid))), \ + patch.object(control, 'atomic') as request, patch('builtins.print') as output: + with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'): + control.main() + request.assert_not_called() + output.assert_not_called() + + def test_main_start_blocks_before_recovery_when_registered_supervisor_is_uninspectable(self): + supervisor = {'pid': 7303, 'created': 1.0, 'executable': '/service', 'args': []} + control.PRIVATE.mkdir(parents=True) + control.DATA.mkdir(parents=True) + control.STATE.write_text(json.dumps({ + 'supervisor': supervisor, 'children': [], 'token': 'safe-token', + }), encoding='utf-8') + updater = self.root / 'deploy/offline/update_manager.py' + updater.parent.mkdir(parents=True) + updater.write_text('# staged updater', encoding='utf-8') + with patch.object(sys, 'argv', ['control.py', 'start']), \ + patch.object(control, 'IS_WINDOWS', True), \ + patch.object(control, 'private_directory') as private_directory, \ + patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw( + control.psutil.AccessDenied(pid=pid))), \ + patch.object(control.subprocess, 'run') as recovery, \ + patch('builtins.print') as output: + with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'): + control.main() + recovery.assert_not_called() + private_directory.assert_not_called() + output.assert_not_called() + + def test_stop_does_not_report_success_when_child_survives_budget(self): + supervisor = {'pid': 10} + child = {'pid': 20} + state = {'supervisor': supervisor, 'children': [child]} + ticks = iter([0, 0, .5, 1.1]) + + def is_running(info): + return info is child + + with patch.object(control, 'atomic'), \ + patch.object(control, 'read_state', return_value=state), \ + patch.object(control, 'matches', side_effect=is_running), \ + patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \ + patch.object(control.time, 'sleep'), \ + patch('builtins.print') as output: + with self.assertRaisesRegex(RuntimeError, '仍在安全关闭'): + control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0) + output.assert_not_called() + + def test_stop_has_three_hundred_second_internal_budget(self): + self.assertEqual(300, control.STOP_TIMEOUT_SECONDS) + + def test_unknown_process_state_is_not_treated_as_exit_or_signalled(self): + class Uninspectable: + def is_running(self): + return True + + def status(self): + raise control.psutil.AccessDenied(pid=4321) + + def terminate(self): + raise AssertionError('unknown process must not be terminated') + + def kill(self): + raise AssertionError('unknown process must not be killed') + + process = Uninspectable() + with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]): + survivors, unverified = control.terminate_observed([(process, 1.0, None)], timeout=0) + self.assertEqual([], survivors) + self.assertTrue(unverified) + + def test_registered_backend_and_mysql_are_retained_when_identity_is_unknown(self): + backend = {'pid': 8101, 'created': 1.0, 'executable': '/java', 'args': ['-jar'] , 'name': 'backend'} + mysql = {'pid': 8102, 'created': 2.0, 'executable': '/mysqld', 'args': ['--port=53306'], 'name': 'mysql'} + + def denied(pid): + raise control.psutil.AccessDenied(pid=pid) + + with patch.object(control.psutil, 'Process', side_effect=denied): + state = {} + retained, unknown = control.retain_registered_children(state, [backend, mysql]) + backend_confirmed, backend_unknown = control.registered_named([backend, mysql], 'backend') + mysql_confirmed, mysql_unknown = control.registered_named([backend, mysql], 'mysql') + self.assertEqual([backend, mysql], retained) + self.assertTrue(unknown) + self.assertEqual([backend, mysql], state['children']) + self.assertTrue(state['unknownChildren']) + self.assertEqual([], backend_confirmed) + self.assertEqual([backend], backend_unknown) + self.assertEqual([], mysql_confirmed) + self.assertEqual([mysql], mysql_unknown) + + def test_no_such_process_is_confirmed_gone(self): + class Gone: + def is_running(self): + raise control.psutil.NoSuchProcess(pid=4321) + + with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]): + survivors, unverified = control.terminate_observed([(Gone(), 1.0, None)], timeout=0) + self.assertEqual([], survivors) + self.assertFalse(unverified) + + def test_malformed_registered_identity_is_unknown(self): + self.assertIsNone(control.matches({'pid': 4321, 'created': float('nan'), + 'executable': '/service', 'args': []})) + + def test_verify_runtime_cli_bypasses_running_service_shortcut(self): + with patch.object(sys, 'argv', ['control.py', 'verify-runtime', '--program-only']), \ + patch.object(control.platform, 'machine', return_value='x86_64'), \ + patch.object(control, 'verify_runtime', return_value={}) as verify, \ + patch.object(control, 'private_directory') as private_directory: + control.main() + verify.assert_called_once_with(program_only=True) + private_directory.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_update_hardening.py b/deploy/offline/test_update_hardening.py new file mode 100644 index 0000000..41a9457 --- /dev/null +++ b/deploy/offline/test_update_hardening.py @@ -0,0 +1,229 @@ +import json +import zipfile +from pathlib import Path +import types +import unittest +from unittest.mock import patch +import test_update_manager as fixtures +update = fixtures.update + + +class HardeningTests(unittest.TestCase): + def setUp(self): + self.f = fixtures.UpdateTests() + self.f.setUp() + self.addCleanup(self.f.doCleanups) + self.root = self.f.root + + def backup(self): + b = self.root / '.runtime/update/backup-20261105T120000Z-abcdef12' + b.mkdir(parents=True) + update.backup_program(self.root, b, self.f.release) + (b / 'database.sql').write_bytes(b'-- database snapshot\n' + b'SELECT 1;\n' * 30) + update.atomic(self.root / '.runtime/update/journal.json', { + 'backup': b.name, 'phase': 'installing', 'databaseBackedUp': True, + 'databaseMayHaveChanged': True, + 'backupManifestSha256': update.digest(b / 'manifest.json'), + 'programInventorySha256': update.digest(b / 'program.json')}) + return b + + def test_completed_status_failure_never_rolls_back_committed_program(self): + original = update.write_status + def status(state, message, root=update.ROOT, **extra): + if state == 'completed': + raise PermissionError('status write failure') + return original(state, message, root, **extra) + with patch.object(update, 'write_status', side_effect=status): + calls, restores, _ = self.f.worker_harness() + self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence']) + self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase']) + self.assertFalse(restores) + self.assertEqual('completed', update.status(self.root)['state']) + + def test_partially_completed_stop_attempt_restores_old_service(self): + failed = False + def hook(root, action, database_only, timeout, program_only): + nonlocal failed + if action == 'stop' and not failed: + failed = True + raise update.UpdateError('service still closing') + calls, restores, _ = self.f.worker_harness(control_hook=hook) + self.assertIn(('serve', False), calls) + self.assertFalse(restores) + self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json')) + self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase']) + + def test_runtime_failure_occurs_before_stop_or_replacement(self): + def hook(root, action, database_only, timeout, program_only): + if action == 'verify-runtime' and not program_only: + raise update.UpdateError('runtime preflight failed') + calls, _, _ = self.f.worker_harness(control_hook=hook) + self.assertNotIn(('stop', False), calls) + self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json')) + self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state']) + + def test_program_verification_failure_does_not_restore_unchanged_database(self): + calls, imports, queries = self.f.worker_harness(fail_verify=True) + self.assertFalse(imports) + self.assertFalse(any('CREATE DATABASE' in q for q in queries)) + self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json')) + + def test_durable_database_is_reused_after_startup_failure_and_bootstrap_replay(self): + b = self.backup() + imports, queries = [], [] + fail = True + def ready(root, process, database_only=False): + nonlocal fail + if not database_only and fail: + fail = False + raise update.UpdateError('temporary startup failure') + with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \ + patch.object(update, 'wait_ready', side_effect=ready), \ + patch.object(update, 'mysql', side_effect=lambda *args, **kwargs: imports.append(kwargs['database'])), \ + patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)): + with self.assertRaises(update.UpdateError): + update.rollback(self.root, b, True) + selected = update.read(self.root / '.runtime/deployment.json')['mysqlDatabase'] + update.restore_program(self.root, b) # immutable bootstrap replays old program/private snapshot + update.recover_locked(self.root) + self.assertEqual(1, len(imports)) + self.assertEqual(1, sum('CREATE DATABASE' in q for q in queries)) + self.assertEqual(selected, update.read(self.root / '.runtime/deployment.json')['mysqlDatabase']) + + def test_recovery_progress_rejects_changed_dump_before_any_service_mutation(self): + b = self.backup() + fail = True + def ready(root, process, database_only=False): + if not database_only: + raise update.UpdateError('temporary startup failure') + with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \ + patch.object(update, 'wait_ready', side_effect=ready), \ + patch.object(update, 'mysql'), patch.object(update, 'sql'): + with self.assertRaises(update.UpdateError): + update.rollback(self.root, b, True) + (b / 'database.sql').write_bytes(b'altered dump') + with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError): + update.recover_locked(self.root) + controller.assert_not_called() + + def test_old_terminal_status_cannot_override_new_operation(self): + update.atomic(self.root / '.runtime/update/journal.json', { + 'phase': 'rolledBack', 'operationId': 'old-operation'}) + update.write_status('downloading', 'downloading new attempt', self.root, operationId='new-operation') + with patch.object(update, 'live', return_value=True): + self.assertEqual('downloading', update.status(self.root)['state']) + + def test_uncertain_complete_write_never_triggers_rollback(self): + original = update.atomic + def atomic(path, value): + original(path, value) + if path.name == 'journal.json' and value.get('phase') == 'complete': + raise OSError('write receipt lost after durable commit') + with patch.object(update, 'atomic', side_effect=atomic): + _, imports, _ = self.f.worker_harness() + self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence']) + self.assertFalse(imports) + journal = update.read(self.root / '.runtime/update/journal.json') + self.assertEqual('complete', journal['phase']) + self.assertIn('backupManifestSha256', journal) + + def test_interrupted_import_attempts_are_bounded_without_dropping_databases(self): + b = self.backup() + imports, queries = [], [] + def mysql(*args, **kwargs): + imports.append(kwargs['database']) + raise update.UpdateError('simulated partial import') + with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \ + patch.object(update, 'wait_ready'), patch.object(update, 'mysql', side_effect=mysql), \ + patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)): + for _ in range(update.MAX_RECOVERY_ATTEMPTS + 1): + with self.assertRaises(update.UpdateError): + update.recover_locked(self.root) + self.assertEqual(update.MAX_RECOVERY_ATTEMPTS, len(imports)) + self.assertEqual(len(imports), len(set(imports))) + self.assertFalse(any('DROP' in q for q in queries)) + self.assertNotIn(self.f.config['mysqlDatabase'], imports) + + def test_database_is_not_durable_until_clean_stop_succeeds(self): + b = self.backup() + stops = 0 + def controller(root, action, database_only=False, **kwargs): + nonlocal stops + if action == 'stop': + stops += 1 + if stops == 2: + raise update.UpdateError('database still closing') + return types.SimpleNamespace(pid=1) + with patch.object(update, 'controller', side_effect=controller), \ + patch.object(update, 'wait_ready'), patch.object(update, 'mysql'), patch.object(update, 'sql'): + with self.assertRaises(update.UpdateError): + update.rollback(self.root, b, True) + self.assertEqual('importing', update.read(self.root / '.runtime/update/recovery.json')['stage']) + + def test_recovery_cannot_be_redirected_to_original_database(self): + b = self.backup() + binding = update.recovery_binding(b) + update.atomic(self.root / '.runtime/update/recovery.json', { + **binding, 'attempt': 1, 'stage': 'databaseDurable', 'database': self.f.config['mysqlDatabase']}) + with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError): + update.recover_locked(self.root) + controller.assert_not_called() + + def test_backup_preflight_failure_keeps_live_service_and_program(self): + with patch.object(update, 'preflight_backup', side_effect=PermissionError('backup path not writable')): + calls, _, _ = self.f.worker_harness() + self.assertNotIn(('stop', False), calls) + self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json')) + + def test_staged_protocol_blocks_old_or_mixed_package_before_stop(self): + stage = self.root / '.runtime/staged-protocol' + scripts = stage / 'deploy/offline' + scripts.mkdir(parents=True) + (stage / 'app').mkdir() + (scripts / 'update_manager.py').write_text('UPDATE_PROTOCOL = 2\n') + (scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n') + with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar: + jar.writestr('BOOT-INF/classes/update-protocol.json', '{"protocol":2}') + update.preflight_programs(stage) + (scripts / 'control.py').write_text('OLD_CONTROLLER = True\n') + with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'): + update.preflight_programs(stage) + (scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n') + with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar: + jar.writestr('old-backend', 'old protocol') + with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'): + update.preflight_programs(stage) + + + def test_actual_worker_rejects_incompatible_signed_payload_before_stopping(self): + calls, imports, _ = self.f.worker_harness(program_hook=update.preflight_programs) + self.assertNotIn(('stop', False), calls) + self.assertFalse(imports) + self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json')) + self.assertIn('更新包', update.read(self.root / '.runtime/update/status.json')['message']) + + def test_legacy_backend_cannot_be_left_in_active_maintenance_after_final_status_failure(self): + original = update.write_status + def status(state, message, root=update.ROOT, **extra): + if state == 'failed': + raise PermissionError('final UI state blocked') + return original(state, message, root, **extra) + with patch.object(update, 'write_status', side_effect=status): + self.f.worker_harness(fail_start=True) + self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase']) + raw = update.read(self.root / '.runtime/update/status.json')['state'] + self.assertNotIn(raw, {'backingUp', 'installing', 'restarting', 'rollingBack'}) + + def test_legacy_terminal_recovery_persists_status_reconciliation(self): + directory = self.root / '.runtime/update' + update.atomic(directory / 'journal.json', {'phase': 'rolledBack', 'operationId': 'same'}) + update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'}) + update.recover_locked(self.root) + self.assertEqual('failed', update.read(directory / 'status.json')['state']) + update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'}) + with patch.object(update, 'write_status', side_effect=PermissionError('status blocked')), self.assertRaises(PermissionError): + update.recover_locked(self.root) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_update_manager.py b/deploy/offline/test_update_manager.py index e5ddf4e..de99420 100644 --- a/deploy/offline/test_update_manager.py +++ b/deploy/offline/test_update_manager.py @@ -194,13 +194,14 @@ class UpdateTests(unittest.TestCase): self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes()) - def worker_harness(self, fail_verify=False): + def worker_harness(self, fail_verify=False, control_hook=None, fail_start=False, program_hook=None): directory = self.root / '.runtime/update' update.atomic(directory / 'request.json', {'token': 'a' * 32}) archive = self.archive() envelope = self.envelope() original_download = update.download calls, restores, queries = [], [], [] + failed_start = False def fake_download(url, target=None, *args): if target: @@ -208,9 +209,15 @@ class UpdateTests(unittest.TestCase): return b'' return json.dumps(envelope).encode() - def fake_control(root, action, database_only=False, timeout=0): + def fake_control(root, action, database_only=False, timeout=0, program_only=False): + nonlocal failed_start calls.append((action, database_only)) - if action == 'verify' and fail_verify: + if fail_start and action == 'serve' and not database_only and not failed_start: + failed_start = True + raise update.UpdateError('模拟新版启动失败。') + if control_hook: + control_hook(root, action, database_only, timeout, program_only) + if action in ('verify', 'verify-runtime') and fail_verify and (program_only or action == 'verify'): raise update.UpdateError('模拟新版启动校验失败。') return types.SimpleNamespace(pid=1234) @@ -226,6 +233,7 @@ class UpdateTests(unittest.TestCase): patch.object(update.psutil, 'Process', return_value=process), \ patch.object(update, 'download', side_effect=fake_download), \ patch.object(update, 'controller', side_effect=fake_control), \ + patch.object(update, 'preflight_programs', side_effect=program_hook), \ patch.object(update, 'wait_ready'), \ patch.object(update, 'mysql', side_effect=fake_mysql), \ patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)): @@ -265,7 +273,7 @@ class UpdateTests(unittest.TestCase): def test_complete_update_orchestration_with_isolated_database_backup(self): calls, restores, queries = self.worker_harness() - self.assertEqual([('stop', False), ('serve', True), ('stop', False), ('verify', False), ('serve', False)], calls) + self.assertEqual([('verify-runtime', False), ('stop', False), ('serve', True), ('stop', False), ('verify-runtime', False), ('serve', False)], calls) self.assertFalse(restores) self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state']) self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase']) @@ -276,7 +284,7 @@ class UpdateTests(unittest.TestCase): update.atomic(self.root / '.runtime/deployment.json', self.config) update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration', 'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64}) - calls, restores, queries = self.worker_harness(fail_verify=True) + calls, restores, queries = self.worker_harness(fail_start=True) self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) config = update.read(self.root / '.runtime/deployment.json') self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$') diff --git a/deploy/offline/test_windows_cache_repair.py b/deploy/offline/test_windows_cache_repair.py index 466533a..996424f 100644 --- a/deploy/offline/test_windows_cache_repair.py +++ b/deploy/offline/test_windows_cache_repair.py @@ -54,10 +54,9 @@ class WindowsCacheRepairTest(unittest.TestCase): patch.object(control.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)): control.verify() - def test_actual_startup_hash_check_fails_on_regenerated_cache_then_passes_after_repair(self): + def test_startup_accepts_regenerated_cache_only_when_source_hash_is_valid(self): (self.root / self.cache).write_bytes(b'new-bytecode-created-on-windows') - with self.assertRaisesRegex(RuntimeError, '__future__.*pyc'): - self.verify() + self.verify() # 合法源码允许运行库重建缓存;原位修复仍可清除旧缓存清单。 result = fix.repair(self.root) self.assertEqual(result['cacheEntriesRemoved'], 1) self.verify() diff --git a/deploy/offline/test_windows_update_check_repair.py b/deploy/offline/test_windows_update_check_repair.py index 4b75c54..acabe4a 100644 --- a/deploy/offline/test_windows_update_check_repair.py +++ b/deploy/offline/test_windows_update_check_repair.py @@ -1,127 +1,253 @@ import hashlib +import ctypes import json +import os from pathlib import Path import subprocess import sys import tempfile import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import repair_windows_update_check as fix -ORIGINAL_SHA = fix.OLD_SHA class UpdateCheckRepairTest(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory(prefix='quality source path repair ') + self.temp = tempfile.TemporaryDirectory(prefix='quality firstaid repair ') self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) / '中文 QualityCenter & files' self.root.mkdir() - self.original = b'original source fixture\n' - self.old_sha = hashlib.sha256(self.original).hexdigest() - self.guard = patch.object(fix, 'OLD_SHA', self.old_sha) - self.guard.start() - self.addCleanup(self.guard.stop) - for name, value in {fix.SCRIPT: self.original, - 'deploy/bootstrap/windows-start.ps1': b'original bootstrap', - 'migration/database.sql.gz': b'existing migration payload', - '.runtime/deployment.json': b'original private configuration', - 'runtimedata/mysql/retained-database': b'imported database data'}.items(): + self.old_manager = b'known old updater fixture\n' + self.old_control = b'known old controller fixture\n' + self.old_manager_sha = hashlib.sha256(self.old_manager).hexdigest() + self.old_control_sha = hashlib.sha256(self.old_control).hexdigest() + self.reviewed_manager_hashes = fix.OLD_MANAGER_SHA256 + self.reviewed_control_hashes = fix.OLD_CONTROL_SHA256 + self.manager_guard = patch.object(fix, 'OLD_MANAGER_SHA256', frozenset({self.old_manager_sha})) + self.control_guard = patch.object(fix, 'OLD_CONTROL_SHA256', frozenset({self.old_control_sha})) + self.manager_guard.start() + self.control_guard.start() + self.addCleanup(self.manager_guard.stop) + self.addCleanup(self.control_guard.stop) + self.manager_payload, self.control_payload = fix.read_hardened_sources() + for name, value in { + fix.MANAGER: self.old_manager, + fix.CONTROL: self.old_control, + 'deploy/bootstrap/windows-start.ps1': b'original bootstrap', + 'migration/database.sql.gz': b'existing migration payload', + '.runtime/deployment.json': b'original private configuration', + 'runtimedata/mysql/retained-database': b'imported database data', + }.items(): target = self.root / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(value) - self.manifest = {'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1, - 'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True}, - 'files': [{'path': fix.SCRIPT, 'sha256': self.old_sha}, - {'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'}]} + self.manifest = { + 'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1, + 'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True}, + 'files': [ + {'path': fix.MANAGER, 'sha256': self.old_manager_sha}, + {'path': fix.CONTROL, 'sha256': self.old_control_sha}, + {'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'}, + ], + } self.manifest_path = self.root / 'manifest.json' self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8') - def test_backup_manifest_and_script_only_preserving_database_and_configuration(self): + def test_backups_both_programs_and_manifest_and_preserves_private_data(self): private = {name: (self.root / name).read_bytes() for name in - ['migration/database.sql.gz', '.runtime/deployment.json', 'runtimedata/mysql/retained-database']} - result = fix.repair(self.root) + ['migration/database.sql.gz', '.runtime/deployment.json', + 'runtimedata/mysql/retained-database']} + result = fix.repair(self.root, self.manager_payload, self.control_payload) self.assertTrue(result['updateCheckRepaired']) - self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT) + self.assertTrue(result['controlRecoveryRepaired']) + self.assertEqual(result['changedFiles'], 3) + self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.manager_payload) + self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload) expected = json.loads(json.dumps(self.manifest)) - expected['files'][0]['sha256'] = fix.NEW_SHA + expected['files'][0]['sha256'] = hashlib.sha256(self.manager_payload).hexdigest() + expected['files'][1]['sha256'] = hashlib.sha256(self.control_payload).hexdigest() self.assertEqual(json.loads(self.manifest_path.read_text()), expected) backup = self.root / result['backup'] - self.assertEqual((backup / 'update_manager.py').read_bytes(), self.original) + self.assertEqual((backup / 'update_manager.py').read_bytes(), self.old_manager) + self.assertEqual((backup / 'control.py').read_bytes(), self.old_control) self.assertEqual(json.loads((backup / 'manifest.json').read_text()), self.manifest) for name, value in private.items(): self.assertEqual((self.root / name).read_bytes(), value) - self.assertTrue(fix.repair(self.root)['alreadyRepaired']) + self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['alreadyRepaired']) - def test_modified_script_or_manifest_is_not_approved_as_a_repair(self): - before = self.manifest_path.read_bytes() - (self.root / fix.SCRIPT).write_bytes(b'unrecognized source') + def test_unknown_program_or_manifest_digest_is_never_overwritten(self): + original_manifest = self.manifest_path.read_bytes() + (self.root / fix.MANAGER).write_bytes(b'unrecognized updater') with self.assertRaisesRegex(RuntimeError, '版本'): - fix.repair(self.root) - self.assertEqual(self.manifest_path.read_bytes(), before) + fix.repair(self.root, self.manager_payload, self.control_payload) + self.assertEqual(self.manifest_path.read_bytes(), original_manifest) + (self.root / fix.MANAGER).write_bytes(self.old_manager) + value = json.loads(original_manifest) + value['files'][0]['sha256'] = '0' * 64 + self.manifest_path.write_text(json.dumps(value), encoding='utf-8') + with self.assertRaisesRegex(RuntimeError, '版本'): + fix.repair(self.root, self.manager_payload, self.control_payload) + self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager) - def test_failed_second_write_restores_original_program_and_manifest(self): - before = self.manifest_path.read_bytes() + def test_failed_manifest_write_rolls_back_both_programs_and_manifest(self): + before = {fix.MANAGER: (self.root / fix.MANAGER).read_bytes(), + fix.CONTROL: (self.root / fix.CONTROL).read_bytes(), + 'manifest.json': self.manifest_path.read_bytes()} original_atomic = fix.atomic calls = [] + def failing(path, data): calls.append(path) - if len(calls) == 2: + if len(calls) == 3: raise OSError('simulated manifest write failure') original_atomic(path, data) + with patch.object(fix, 'atomic', side_effect=failing), self.assertRaises(OSError): - fix.repair(self.root) - self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original) - self.assertEqual(self.manifest_path.read_bytes(), before) + fix.repair(self.root, self.manager_payload, self.control_payload) + self.assertEqual((self.root / fix.MANAGER).read_bytes(), before[fix.MANAGER]) + self.assertEqual((self.root / fix.CONTROL).read_bytes(), before[fix.CONTROL]) + self.assertEqual(self.manifest_path.read_bytes(), before['manifest.json']) + self.assertEqual(len(list((self.root / '.runtime/update-check-repair-backups').iterdir())), 1) - def test_retry_completes_an_interrupted_script_manifest_pair(self): - (self.root / fix.SCRIPT).write_bytes(fix.PATCHED_SCRIPT) - fix.repair(self.root) - self.assertEqual(json.loads(self.manifest_path.read_text())['files'][0]['sha256'], fix.NEW_SHA) + def test_retry_completes_old_new_mix_and_accepts_rolled_back_journal(self): + (self.root / fix.MANAGER).write_bytes(self.manager_payload) + journal = self.root / '.runtime/update/journal.json' + journal.parent.mkdir(parents=True) + journal.write_text(json.dumps({'phase': 'rolledBack'}), encoding='utf-8') + result = fix.repair(self.root, self.manager_payload, self.control_payload) + self.assertTrue(result['updateCheckRepaired']) + self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload) + manifest = json.loads(self.manifest_path.read_text()) + self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(self.manager_payload).hexdigest()) + self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(self.control_payload).hexdigest()) - def test_cmd_payload_runs_with_bundled_standard_library_without_service_restart(self): - # 真实原包脚本由交付包集成验证;此单元测试使用受控旧源码摘要。 - command = self.root / '修复更新检查.cmd' - body = fix.command_file().decode('utf-8').replace(ORIGINAL_SHA, self.old_sha) - command.write_bytes(body.encode('utf-8')) - code = body.split(' -c "', 1)[1].split('" "%~f0"', 1)[0] - result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT) - self.assertNotIn('powershell.exe', body) - self.assertIn('无需停止服务或重新导入数据库', body) - - def test_active_update_lock_blocks_patching(self): - before = self.manifest_path.read_bytes() - with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'): - fix.repair(self.root) - self.assertEqual(self.manifest_path.read_bytes(), before) - - def test_worker_or_unfinished_journal_blocks_repair(self): - for name, data in [('worker.json', {}), ('journal.json', {'phase': 'installing'})]: - path = self.root / '.runtime/update' / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data)) + def test_active_worker_and_unknown_or_unfinished_journal_block_repair(self): + worker = self.root / '.runtime/update/worker.json' + worker.parent.mkdir(parents=True) + worker.write_text(json.dumps({'pid': __import__('os').getpid()}), encoding='utf-8') + with self.assertRaisesRegex(RuntimeError, '进程'): + fix.repair(self.root, self.manager_payload, self.control_payload) + worker.unlink() + journal = worker.parent / 'journal.json' + for value in ({'phase': 'installing'}, {'phase': 'futurePhase'}, {'unknown': True}): + journal.write_text(json.dumps(value), encoding='utf-8') with self.assertRaises(RuntimeError): - fix.repair(self.root) - self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original) - path.unlink() + fix.repair(self.root, self.manager_payload, self.control_payload) + self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager) + journal.unlink() - def test_previous_check_repair_version_can_upgrade_without_losing_manifest_fields(self): - prior = b'previous optimized checker fixture' - digest = hashlib.sha256(prior).hexdigest() - (self.root / fix.SCRIPT).write_bytes(prior) + def test_windows_worker_probe_is_read_only_for_missing_and_live_pids(self): + worker = self.root / '.runtime/update/worker.json' + worker.parent.mkdir(parents=True) + worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8') + + missing_kernel = Mock() + missing_kernel.OpenProcess = Mock(return_value=None) + missing_kernel.GetExitCodeProcess = Mock() + missing_kernel.CloseHandle = Mock() + with (patch.object(fix.os, 'name', 'nt'), + patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill, + patch('ctypes.WinDLL', return_value=missing_kernel, create=True), + patch('ctypes.get_last_error', return_value=87, create=True)): + self.assertFalse(fix.worker_active(self.root.resolve())) + kill.assert_not_called() + missing_kernel.GetExitCodeProcess.assert_not_called() + missing_kernel.CloseHandle.assert_not_called() + missing_kernel.TerminateProcess.assert_not_called() + missing_kernel.OpenProcess.assert_called_once_with(0x1000, False, 424242) + + denied_kernel = Mock() + denied_kernel.OpenProcess = Mock(return_value=None) + denied_kernel.GetExitCodeProcess = Mock() + denied_kernel.CloseHandle = Mock() + with (patch.object(fix.os, 'name', 'nt'), + patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill, + patch('ctypes.WinDLL', return_value=denied_kernel, create=True), + patch('ctypes.get_last_error', return_value=5, create=True)): + with self.assertRaisesRegex(RuntimeError, '无法只读确认'): + fix.worker_active(self.root.resolve()) + kill.assert_not_called() + denied_kernel.TerminateProcess.assert_not_called() + + live_kernel = Mock() + live_kernel.OpenProcess = Mock(return_value=123) + live_kernel.CloseHandle = Mock(return_value=1) + + def set_still_active(_handle, output): + ctypes.cast(output, ctypes.POINTER(ctypes.c_ulong)).contents.value = 259 + return 1 + + live_kernel.GetExitCodeProcess = Mock(side_effect=set_still_active) + with (patch.object(fix.os, 'name', 'nt'), + patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill, + patch('ctypes.WinDLL', return_value=live_kernel, create=True)): + self.assertTrue(fix.worker_active(self.root.resolve())) + kill.assert_not_called() + live_kernel.CloseHandle.assert_called_once_with(123) + live_kernel.TerminateProcess.assert_not_called() + + def test_stale_worker_pid_allows_repair_and_live_reused_pid_fails_closed(self): + worker = self.root / '.runtime/update/worker.json' + worker.parent.mkdir(parents=True) + worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8') + with patch.object(fix.os, 'kill', side_effect=ProcessLookupError): + self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['updateCheckRepaired']) + + # A live process with a stale/reused PID is treated as active; no identity match is required. + (self.root / fix.MANAGER).write_bytes(self.old_manager) + (self.root / fix.CONTROL).write_bytes(self.old_control) + self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8') + with patch.object(fix.os, 'kill', return_value=None): + with self.assertRaisesRegex(RuntimeError, '仍在运行'): + fix.repair(self.root, self.manager_payload, self.control_payload) + + def test_update_lock_blocks_firstaid(self): + with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'): + fix.repair(self.root, self.manager_payload, self.control_payload) + + def test_self_contained_cmd_uses_bundled_python_and_latest_source_bytes(self): + manager, control = fix.read_hardened_sources() + # This models a crash after source replacement but before manifest update. + (self.root / fix.MANAGER).write_bytes(manager) + (self.root / fix.CONTROL).write_bytes(control) value = json.loads(self.manifest_path.read_text()) - value['files'][0]['sha256'] = digest - self.manifest_path.write_text(json.dumps(value)) - with patch.object(fix, 'PREVIOUS_SHA', digest): - self.assertTrue(fix.repair(self.root)['updateCheckRepaired']) - self.assertEqual(fix.PATCHED_SCRIPT, (self.root / fix.SCRIPT).read_bytes()) + value['files'][0]['sha256'] = '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b' + value['files'][1]['sha256'] = 'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1' + self.manifest_path.write_text(json.dumps(value), encoding='utf-8') + command = self.root / '兼容旧安装修复.cmd' + body = fix.command_file() + command.write_bytes(body) + module, payload = body.rsplit(fix.MODULE_MARKER, 1)[1].rsplit(fix.PAYLOAD_MARKER, 1) + embedded = {} + exec(compile(payload.decode('ascii'), str(command), 'exec'), embedded) + self.assertEqual(embedded['PATCHED_MANAGER'], manager) + self.assertEqual(embedded['PATCHED_CONTROL'], control) + code = body.decode('utf-8').split(' -c "', 1)[1].split('" "%~f0"', 1)[0] + result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.root / fix.MANAGER).read_bytes(), manager) + self.assertEqual((self.root / fix.CONTROL).read_bytes(), control) + manifest = json.loads(self.manifest_path.read_text()) + self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(manager).hexdigest()) + self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(control).hexdigest()) + command_header = body.split(fix.MODULE_MARKER, 1)[0].decode('utf-8') + self.assertNotIn('powershell.exe', command_header) + self.assertNotIn('windows-start.ps1', command_header) + self.assertIn(' -I -S -B ', command_header) - def test_embedded_program_matches_reviewed_source(self): - self.assertEqual(fix.PATCHED_SCRIPT, (HERE / 'update_manager.py').read_bytes()) + def test_historical_allowlist_matches_known_release_inputs(self): + self.assertEqual(self.reviewed_manager_hashes, frozenset({ + '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b', + 'b91cb7fec10e183d2904d86f032ff0ffcf7f19f7cb27aecd2d803d3448685482', + 'a4bd05789f559fede0bed083ce61123307d6b5cf52fa638b6bde1fa112a0348a', + })) + self.assertEqual(self.reviewed_control_hashes, frozenset({ + 'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1', + })) if __name__ == '__main__': diff --git a/deploy/offline/update_manager.py b/deploy/offline/update_manager.py index d8b7e98..67e692a 100644 --- a/deploy/offline/update_manager.py +++ b/deploy/offline/update_manager.py @@ -8,8 +8,10 @@ snapshot; private data and runtimes are never update destinations. from __future__ import annotations import argparse +import ast import base64 import contextlib +import fnmatch from datetime import datetime, timezone import hashlib import json @@ -30,6 +32,7 @@ import psutil from cryptography.hazmat.primitives.serialization import load_pem_public_key from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +UPDATE_PROTOCOL = 2 ROOT = Path(__file__).resolve().parents[2] BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/' LATEST = BASE + 'latest.json' @@ -267,11 +270,25 @@ def enabled(root=ROOT): def status(root=ROOT): manifest = read(root / 'manifest.json', {}) - value = read(root / '.runtime/update/status.json', {}) + try: + value = read(root / '.runtime/update/status.json', {}) + except (OSError, ValueError): + value = {} + if not isinstance(value, dict): + value = {} allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'} result = {k: v for k, v in value.items() if k in allowed} result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本'))) result.setdefault('state', 'idle') + journal = read(root / '.runtime/update/journal.json', {}) + same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId') + if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE + and manifest.get('updateSequence') == journal.get('sequence') + and manifest.get('updateVersion') == journal.get('version')): + result.update(state='completed', message='更新完成。', available=False, + version=journal['version'], finishedAt=journal['finishedAt']) + elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE: + result.update(state='failed', message='更新未完成,已恢复原版本。', available=False) if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})): pending = read(root / '.runtime/update/request.json', {}) journal = read(root / '.runtime/update/journal.json', {}) @@ -323,7 +340,10 @@ def exclusive_update(root=ROOT): yield finally: if read(directory / 'worker.json', {}) == identity: - (directory / 'worker.json').unlink(missing_ok=True) + try: + (directory / 'worker.json').unlink(missing_ok=True) + except OSError as error: + record_failure(root, 'cleanupWorker', error) def launch(root=ROOT): @@ -345,7 +365,7 @@ def launch_locked(root=ROOT): return status(root) token = secrets.token_hex(16) atomic(directory / 'request.json', {'token': token, 'created': time.time()}) - write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False) + write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token) try: with (directory / 'worker.log').open('ab') as log: subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token], @@ -364,11 +384,13 @@ def live(info): return False -def controller(root, action, database_only=False, timeout=1100): +def controller(root, action, database_only=False, timeout=1100, program_only=False): python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3') command = [str(python), str(root / 'deploy/offline/control.py'), action] if database_only: command += ['--database-only'] + if program_only: + command += ['--program-only'] if action == 'serve': with (root / '.runtime/update/service.log').open('ab') as log: return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options()) @@ -471,29 +493,186 @@ def restore_program(root, backup): copy_atomic(source, destination(root, '.runtime/' + relative)) +STOP_TIMEOUT = 360 +MAX_RECOVERY_ATTEMPTS = 2 + + +def best_effort_status(state, message, root=ROOT, **extra): + try: + return write_status(state, message, root, **extra) + except Exception as error: + record_failure(root, 'writeStatus', error) + + +def verify_backup(root, backup, journal=None): + journal = journal or read(root / '.runtime/update/journal.json', {}) + if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256') + or digest(backup / 'program.json') != journal.get('programInventorySha256')): + raise UpdateError('更新恢复备份校验失败,已保留现场。') + manifest = entry_map(read(backup / 'manifest.json')['files']) + inventory = read(backup / 'program.json') + names, present = inventory.get('all'), inventory.get('present') + if (not isinstance(names, list) or not isinstance(present, list) + or len(names) != len(set(names)) or not set(present).issubset(names)): + raise UpdateError('更新恢复清单无效。') + for name in names: + if not program_path(name): + raise UpdateError('更新恢复清单包含受保护路径。') + destination(root, name) + if name in present: + source = destination(backup / 'program', name) + if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']: + raise UpdateError('原程序备份缺失或损坏。') + elif name in manifest: + raise UpdateError('原程序备份不完整。') + + +def recovery_binding(backup): + return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'), + 'programInventorySha256': digest(backup / 'program.json'), + 'databaseSqlSha256': digest(backup / 'database.sql')} + + +def recovery_database(binding, attempt): + return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24] + + +def recovery_progress(root, backup): + binding = recovery_binding(backup) + try: + value = read(root / '.runtime/update/recovery.json', {}) + except (OSError, ValueError) as error: + raise UpdateError('恢复进度无法读取,已保留现场。') from error + if not value: + return binding, None + if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items()) + or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS + or value.get('database') != recovery_database(binding, value['attempt']) + or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')): + raise UpdateError('恢复进度与备份不一致,已保留现场。') + return binding, value + + +def prepare_old_backend_restart(root): + # 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态, + # 再启动旧服务;写失败则保留恢复记录,不声称已恢复。 + write_status('queued', '正在启动原版本,完成后自动重新连接。', root) + + +def normalize_terminal_status(root, journal): + value = read(root / '.runtime/update/status.json', {}) + if (not isinstance(value, dict) or value.get('state') not in ACTIVE + or journal.get('operationId') != value.get('operationId')): + return + pending = read(root / '.runtime/update/request.json', {}) + if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120: + return + if journal.get('phase') == 'rolledBack': + write_status('failed', '上次更新中断,已恢复原版本。', root, available=False) + elif journal.get('phase') == 'complete': + manifest = read(root / 'manifest.json') + if manifest.get('updateVersion') == journal.get('version', value.get('version')): + write_status('completed', '更新完成。', root, available=False) + + def rollback(root, backup, database_backed_up): - write_status('rollingBack', '更新未完成,正在恢复原版本。', root) - controller(root, 'stop', timeout=180) + verify_backup(root, backup) + binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None) + best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root) + controller(root, 'stop', timeout=STOP_TIMEOUT) restore_program(root, backup) if database_backed_up: - config = read(root / '.runtime/deployment.json') - process = controller(root, 'serve', True) - wait_ready(root, process, True) - database = 'quality_restore_' + secrets.token_hex(12) - sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') - with (backup / 'database.sql').open('rb') as source: - mysql(root, config, database=database, stdin=source) - sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';") - controller(root, 'stop', timeout=180) - config['mysqlDatabase'] = database + config = read(backup / 'private/deployment.json') + # 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。 + if progress is None or progress['stage'] == 'importing': + attempt = 1 if progress is None else progress['attempt'] + 1 + if attempt > MAX_RECOVERY_ATTEMPTS: + raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。') + reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2 + if shutil.disk_usage(root).free < reserve: + raise UpdateError('空间不足以安全恢复数据库,已保留现场。') + database = recovery_database(binding, attempt) + if database == config.get('mysqlDatabase'): + raise UpdateError('恢复数据库与原库冲突,已停止。') + progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'} + atomic(root / '.runtime/update/recovery.json', progress) + process = controller(root, 'serve', True) + wait_ready(root, process, True) + # 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。 + sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') + with (backup / 'database.sql').open('rb') as source: + mysql(root, config, database=database, stdin=source) + sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';") + controller(root, 'stop', timeout=STOP_TIMEOUT) + progress['stage'] = 'databaseDurable' + atomic(root / '.runtime/update/recovery.json', progress) + config['mysqlDatabase'] = progress['database'] atomic(root / '.runtime/deployment.json', config) receipt_path = root / '.runtime/migration-receipt.json' receipt = read(receipt_path, {}) if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'): - receipt['database'] = database + receipt['database'] = progress['database'] atomic(receipt_path, receipt) + progress['stage'] = 'published' + atomic(root / '.runtime/update/recovery.json', progress) + prepare_old_backend_restart(root) process = controller(root, 'serve') wait_ready(root, process) + if progress: + progress['stage'] = 'ready' + atomic(root / '.runtime/update/recovery.json', progress) + + +def preflight_programs(stage): + # 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。 + for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')): + try: + tree = ast.parse((stage / 'deploy/offline' / name).read_bytes()) + values = [node.value.value for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets) + and isinstance(node.value, ast.Constant) and type(node.value.value) is int] + if values != [UPDATE_PROTOCOL]: + raise ValueError('incompatible recovery protocol') + except (OSError, SyntaxError, ValueError) as error: + raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error + try: + with zipfile.ZipFile(stage / 'app/backend.jar') as jar: + metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json')) + if metadata != {'protocol': UPDATE_PROTOCOL}: + raise ValueError('incompatible backend protocol') + except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error: + raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error + + +def preflight_backup(root, backup, release): + paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path'])) + paths.update(e['path'] for e in release['files']) + targets = [backup / 'program' / n for n in paths] + ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json') + for folder, dirs, files in os.walk(root / '.runtime', followlinks=False): + dirs[:] = [n for n in dirs if n != 'update'] + for n in dirs: + destination(root, (Path(folder) / n).relative_to(root).as_posix()) + for n in files: + if any(fnmatch.fnmatch(n, pattern) for pattern in ignored): + continue + source = Path(folder) / n + destination(root, source.relative_to(root).as_posix()) + with source.open('rb'): + pass + targets.append(backup / 'private' / source.relative_to(root / '.runtime')) + for target in targets: + destination(backup, target.relative_to(backup).as_posix()) + if os.name == 'nt' and len(str(target.absolute())) + 24 > 248: + raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。') + probe = backup / 'write-probe.tmp' + try: + with probe.open('xb') as stream: + stream.write(b'probe') + stream.flush() + os.fsync(stream.fileno()) + finally: + probe.unlink(missing_ok=True) def record_failure(root, stage, error, name='failure.json'): @@ -526,7 +705,7 @@ def worker(launcher, token, root=ROOT): raise UpdateError('更新请求已失效。') with exclusive_update(root): backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4)) - stopped = backed_up = program_saved = changed = False + stopped = backed_up = program_saved = changed = stop_requested = committed = False failure_stage = 'validateRelease' try: envelope = decode(download(LATEST)) @@ -543,18 +722,30 @@ def worker(launcher, token, root=ROOT): private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory)) if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2: raise UpdateError('磁盘剩余空间不足,请释放空间后重试。') - write_status('downloading', '正在下载并校验更新。', root) + write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version']) info = release['archive'] failure_stage = 'downloadArchive' download(BASE + info['file'], archive, info['size'], info['sha256']) failure_stage = 'unpackArchive' unpack(archive, stage, release) + failure_stage = 'programPreflight' + preflight_programs(stage) + failure_stage = 'runtimePreflight' + write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root) + controller(root, 'verify-runtime', timeout=1800) + failure_stage = 'backupPreflight' + preflight_backup(root, backup, release) # Revalidate immediately before any service or disk mutation. validate_release(envelope, root) write_status('backingUp', '正在备份,服务即将重新启动。', root) - atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False}) + if (directory / 'recovery.json').exists(): + copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json') + (directory / 'recovery.json').unlink() + atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False, + 'operationId': token, 'version': release['version'], 'sequence': release['sequence']}) failure_stage = 'stopService' - controller(root, 'stop', timeout=180) + stop_requested = True + controller(root, 'stop', timeout=STOP_TIMEOUT) stopped = True failure_stage = 'backupProgram' backup_program(root, backup, release) @@ -571,13 +762,15 @@ def worker(launcher, token, root=ROOT): if (backup / 'database.sql').stat().st_size < 100: raise UpdateError('数据库备份为空,已停止更新。') backed_up = True - controller(root, 'stop', timeout=180) + controller(root, 'stop', timeout=STOP_TIMEOUT) # The actual dump can exceed the source's physical storage. Do not # replace anything unless a separate rollback database still fits. required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2 if shutil.disk_usage(root).free < required_recovery: raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。') atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True, + 'operationId': token, 'version': release['version'], 'sequence': release['sequence'], + 'databaseMayHaveChanged': False, 'backupManifestSha256': digest(backup / 'manifest.json'), 'programInventorySha256': digest(backup / 'program.json')}) write_status('installing', '正在安装更新。', root) @@ -585,32 +778,46 @@ def worker(launcher, token, root=ROOT): failure_stage = 'replaceProgram' replace_program(root, stage, backup, release) failure_stage = 'verifyProgram' - controller(root, 'verify', timeout=180) + controller(root, 'verify-runtime', timeout=300, program_only=True) write_status('restarting', '正在重新启动,请稍候。', root) failure_stage = 'restartService' + journal = read(directory / 'journal.json') + journal['databaseMayHaveChanged'] = True + atomic(directory / 'journal.json', journal) process = controller(root, 'serve') wait_ready(root, process) - atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True}) - write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False) + journal.update(phase='complete', version=release['version'], sequence=release['sequence'], + finishedAt=datetime.now(timezone.utc).isoformat()) + atomic(directory / 'journal.json', journal) + committed = True + best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False) except Exception as error: record_failure(root, failure_stage, error) message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。' - if stopped: + journal = read(directory / 'journal.json', {}) + if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'): + return # 提交后的展示或清理失败绝不能发起回滚。 + if stop_requested: try: if program_saved and changed: - rollback(root, backup, backed_up) + rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True)) else: - controller(root, 'stop', timeout=180) + controller(root, 'stop', timeout=STOP_TIMEOUT) + prepare_old_backend_restart(root) process = controller(root, 'serve') wait_ready(root, process) - atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up}) + journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up) + atomic(directory / 'journal.json', journal) message += ' 已恢复原版本。' except Exception as rollback_error: record_failure(root, 'rollback', rollback_error, 'rollback-failure.json') message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。' - write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat()) + best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat()) finally: - (directory / 'request.json').unlink(missing_ok=True) + try: + (directory / 'request.json').unlink(missing_ok=True) + except OSError as error: + record_failure(root, 'cleanupRequest', error) def recover(root=ROOT): @@ -622,22 +829,31 @@ def recover_locked(root=ROOT): directory = root / '.runtime/update' journal = read(directory / 'journal.json', {}) if journal.get('phase') in (None, 'complete', 'rolledBack'): + if journal.get('phase') in ('complete', 'rolledBack'): + normalize_terminal_status(root, journal) return status(root) name = journal.get('backup', '') if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name): raise UpdateError('更新恢复记录无效。') backup = directory / name + if journal.get('phase') not in ('installing', 'stopping'): + raise UpdateError('更新恢复阶段无效,已保留现场。') if journal.get('phase') == 'installing': if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256') or digest(backup / 'program.json') != journal.get('programInventorySha256')): raise UpdateError('更新恢复备份校验失败,已保留现场。') - rollback(root, backup, journal.get('databaseBackedUp') is True) + verify_backup(root, backup, journal) + if journal.get('databaseBackedUp') is True: + recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。 + rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True)) else: - controller(root, 'stop', timeout=180) + controller(root, 'stop', timeout=STOP_TIMEOUT) + prepare_old_backend_restart(root) process = controller(root, 'serve') wait_ready(root, process) atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'}) - return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False) + best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False) + return status(root) def main():