diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 0e341c7..d9eb94e 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -47,8 +47,10 @@ python3 deploy/docker/manage.py stop - 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。 - 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。 - 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。 +- 未完成的备份候选另行每 5 分钟检查,超过 24 小时、所有者和已登记子进程确认退出后才回收;该步骤不依赖新备份成功,也不要求数据库可连接。最新备份指针、有效候选、活跃进程、身份不完整的旧记录和未知目录均保留。 - 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。 - Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。 +- AI 原生服务输出由受管捕获进程滚动写入,单段 10MiB、每个服务日志总量约 90MiB,归档最长 7 天。后台清理同时覆盖各用户及会话的日志;正在写入的文件继续保护。日志写入失败会限频重试并持续排空输出,不会为清日志终止正在进行的 AI 请求。已有运行实例继续工作,下次正常创建实例时采用新的日志捕获方式。 - 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。 Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。 diff --git a/deploy/docker/test_docker.py b/deploy/docker/test_docker.py index 9c44edf..9effb80 100644 --- a/deploy/docker/test_docker.py +++ b/deploy/docker/test_docker.py @@ -4,6 +4,7 @@ import os from pathlib import Path import sys import tempfile +import time import unittest from unittest.mock import Mock, patch @@ -24,6 +25,22 @@ import backup_retention as br class ContainerOperationsTests(unittest.TestCase): + def test_database_unavailable_does_not_block_expired_candidate_recovery(self): + worker = module('container_worker', ROOT / 'deploy/docker/worker.py') + with tempfile.TemporaryDirectory() as directory: + root=Path(directory).resolve();folder=root/'output/backups/20261108T010000Z-12345678';folder.mkdir(parents=True) + (folder/'database.sql.gz').write_bytes(b'interrupted dump');br.register_candidate(folder,root) + value=json.loads((folder/'.owner.json').read_text());value.update({'createdAt':time.time()-3*86400,'ownerPid':2000000000,'ownerCreated':1.0}) + br.atomic(folder/'.owner.json',value);os.utime(folder,(time.time()-3*86400,)*2) + br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':'quality_current'}) + br.atomic(root/'.runtime/operations/policy.json',worker.operations.DEFAULTS) + with patch.object(worker,'ROOT',root),patch.object(worker.operations,'snapshot',worker.operations.snapshot), \ + patch('builtins.print') as output, \ + patch.object(backup,'client',side_effect=RuntimeError('Database unavailable')): + with self.assertRaises(RuntimeError):worker.once('monitor') + self.assertFalse(folder.exists()) + self.assertEqual(json.loads(output.call_args.args[0])['incompleteBackupsRemoved'],1) + def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678' diff --git a/deploy/docker/worker.py b/deploy/docker/worker.py index d69504c..746c522 100644 --- a/deploy/docker/worker.py +++ b/deploy/docker/worker.py @@ -39,17 +39,26 @@ def snapshot(root, support=None, now=None): mysql = estimates.get(cfg['mysqlDatabase'], 0) archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups') native = operations.allocated(root / '.runtime/nao-users') - logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations')) - core = mysql + archive + backups + native + logs + log_paths = log_retention.candidates(root, root / '.runtime/operations') + logs = sum(p.stat().st_blocks * 512 for p in log_paths) + native_logs = sum(p.stat().st_blocks * 512 for p in log_paths if p.is_relative_to(root / '.runtime/nao-users')) + core = mysql + archive + backups + max(0, native - native_logs) + logs free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive')) return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo', - 'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native, + 'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': max(0, native - native_logs), 'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private, 'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free, 'overdueTestResources': overdue} def once(role): + # 数据库不可连接时也要能释放中断备份占用,不能放在采样或健康检查之后。 + if not (ROOT / '.runtime/retention/paused').exists(): + try: + result = br.clean_incomplete(ROOT) + if result['incompleteBackupsRemoved']: + print(json.dumps({'event': 'incomplete-backup-cleanup', **result}), flush=True) + except br.Busy: pass if role == 'monitor': operations.snapshot = snapshot return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify) diff --git a/deploy/macos/README.md b/deploy/macos/README.md index 75e3750..0327f31 100644 --- a/deploy/macos/README.md +++ b/deploy/macos/README.md @@ -85,6 +85,8 @@ PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discove 注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。 +未完成备份候选使用独立回收步骤:超过 24 小时且进程身份确认退出后才清理,不等待下一次备份成功;当前备份、有效候选、身份不完整和仍运行的记录保留。AI 用户及会话日志也纳入后台检查;新启动的原生服务通过 `nao_log_stream.py` 捕获输出,单段 10MiB、每服务约 90MiB、归档最长 7 天。捕获进程与原生子进程共享进程组,沿用原有会话身份与停止保护;日志写入异常不会为清理中断 AI 请求。已有实例不因升级被终止,下次正常启动时加载新方式。 + ```sh .runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD" .runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD" diff --git a/deploy/macos/operations.py b/deploy/macos/operations.py index 247f042..bbeb52d 100644 --- a/deploy/macos/operations.py +++ b/deploy/macos/operations.py @@ -47,6 +47,7 @@ def snapshot(root, support=SUPPORT, now=None): output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output')) runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime')) native = allocated(root / '.runtime/nao-users') + native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users')) registered = 0; overdue = 0 for p in resources.registry(root).glob('*.json'): value = resources.load(root, p.stem) @@ -61,10 +62,11 @@ def snapshot(root, support=SUPPORT, now=None): if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p) # Core service data is the same scope as the earlier 11.86GiB measurement. # Software/build/delivery artifacts are visible separately and included in growth checks. - core = max(0, mysql - registered) + archives + backups + log_bytes + native - artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs) + native_data = max(0, native - native_logs) + core = max(0, mysql - registered) + archives + backups + log_bytes + native_data + artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs) return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups, - 'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered, + 'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private, 'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts, 'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue} @@ -143,17 +145,18 @@ def janitor(root, support, policy): if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'} try: with br.lock(root, timeout=0): + result = br.clean_incomplete(root) db = Database(root) try: config = json.loads((root / '.runtime/retention/config.json').read_text()) accepted = br.accepted_backend(root) if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]: - return {'deferred': 'Application version changed'} + return {**result, 'deferred': 'Application version changed'} db.current() - if db.busy(): return {'deferred': 'Application jobs are active'} + if db.busy(): return {**result, 'deferred': 'Application jobs are active'} # Use a short-lived CLI connection so no observer holds a test schema open. import backup - result = resources.cleanup(root, backup.client) + result.update(resources.cleanup(root, backup.client)) result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root) result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br) result.update(log_retention.retain(root, support, diff --git a/deploy/macos/retention.py b/deploy/macos/retention.py index 6d9c031..f814464 100644 --- a/deploy/macos/retention.py +++ b/deploy/macos/retention.py @@ -372,6 +372,7 @@ def run(root, max_batches=30, grace=1800): state=json.loads(state_path.read_text()) if state_path.exists() else {} result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0} with br.lock(root,timeout=0): + result.update(br.clean_incomplete(root)) result['retiredBackupFoldersRemoved']=br.clean_retired(root) result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root) db=Database(root) diff --git a/deploy/macos/test_operations.py b/deploy/macos/test_operations.py index ea82161..58c37d9 100644 --- a/deploy/macos/test_operations.py +++ b/deploy/macos/test_operations.py @@ -21,6 +21,16 @@ ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops) class OperationsTests(unittest.TestCase): + def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d).resolve();native=root/'.runtime/nao-users/actor' + for relative in ['logs/server.log','private/nao.sqlite']: + path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192) + (root/'.runtime/tool.bin').write_bytes(b'x'*16384) + sample=ops.snapshot(root,root/'support') + self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes'])) + self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users')) + def sample(self, **extra): return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB, 'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra} diff --git a/scripts/backup.py b/scripts/backup.py index 3b9bbd2..f1bf1b1 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -369,8 +369,7 @@ def create(): folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8]) folder.mkdir(parents=True, exist_ok=False) folder.chmod(0o700) - backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()), - 'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()}) + backup_retention.register_candidate(folder, ROOT) sql = folder / 'database.sql.gz' command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks', @@ -378,7 +377,9 @@ def create(): process = None try: with tempfile.TemporaryFile() as errors, sql.open('wb') as output: + backup_retention.candidate_child(folder, pending=True) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors) + backup_retention.candidate_child(folder, process.pid) try: with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed: shutil.copyfileobj(process.stdout, compressed) diff --git a/scripts/backup_retention.py b/scripts/backup_retention.py index 22a8c43..59cfde3 100644 --- a/scripts/backup_retention.py +++ b/scripts/backup_retention.py @@ -4,6 +4,7 @@ from datetime import datetime, timezone from functools import wraps import hashlib import json +import math import os from pathlib import Path import re @@ -166,16 +167,9 @@ def _rotate(root, selected): for folder in base.iterdir(): if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue - # A restore-verified older folder is safe to retire under the shared lock. - # Incomplete candidates are kept for 24h and never treated as good backups. + # 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。 older = verified(folder, checksums=False) - try: - owner = json.loads((folder / '.owner.json').read_text()) - owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name - except (OSError, ValueError): - owned = False - stale = owned and time.time() - folder.stat().st_mtime > 86400 - if older is not None or stale: + if older is not None: contents = list(folder.rglob('*')) if any(p.is_symlink() for p in contents): continue @@ -204,6 +198,7 @@ def _rotate(root, selected): retire_folder(root,legacy,'legacy') legacy_info['retired']=True atomic(registration,legacy_info) + clean_incomplete(root) clean_retired(root) return pointer, removed @@ -282,6 +277,75 @@ def backup_ids(root): return {p.name for p in base.iterdir()} if base.exists() else set() +def register_candidate(folder, root): + import psutil + folder = Path(folder); info = folder.stat() + atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2, + 'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(), + 'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(), + 'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False}) + + +def candidate_child(folder, pid=None, pending=False): + import psutil + path = Path(folder) / '.owner.json'; value = json.loads(path.read_text()) + if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed') + value['childPending'] = pending + if pid is not None: + value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()}) + atomic(path, value) + + +def clean_incomplete(root, now=None): + """独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。""" + import psutil + root = Path(root).resolve(); now = time.time() if now is None else now + result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0} + base = root / 'output/backups' + if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root') + def exited(item, pid_key, created_key): + if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or + not isinstance(item.get(created_key), (float, int)) or + not math.isfinite(item[created_key]) or item[created_key] <= 0): return False + try: + process = psutil.Process(item[pid_key]) + return (abs(process.create_time() - item[created_key]) >= .02 or + not process.is_running() or process.status() == psutil.STATUS_ZOMBIE) + except psutil.NoSuchProcess: return True + except psutil.Error: return False + with lock(root, timeout=0): + pointer = root / '.runtime/retention/latest.json' + try: + if pointer.is_symlink(): return result + protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None + except (OSError, ValueError, KeyError, TypeError): return result + for folder in base.iterdir() if base.exists() else []: + if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue + owner = folder / '.owner.json' + if owner.is_symlink(): continue + try: + value = json.loads(owner.read_text()); info = folder.stat() + if not isinstance(value, dict): continue + if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or + value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or + not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue + if verified(folder, checksums=False) is not None: continue + if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or + value.get('childPending') is not False or not isinstance(value.get('children'), list) or + not exited(value, 'ownerPid', 'ownerCreated') or + not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])): + result['incompleteBackupsDeferred'] += 1; continue + contents = list(folder.rglob('*')) + if any(p.is_symlink() for p in contents): continue + size = sum(p.stat().st_size for p in contents if p.is_file()) + if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue + retire_folder(root, folder, 'backup') + result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size + except (OSError, ValueError, TypeError): continue + clean_retired(root) + return result + + def discard_failed_new(root, before): """Delete only this call's registered, incomplete candidate after it has stopped.""" root=Path(root).resolve();base=root/'output/backups' diff --git a/scripts/log_retention.py b/scripts/log_retention.py index 9a62aaf..763bcc6 100644 --- a/scripts/log_retention.py +++ b/scripts/log_retention.py @@ -16,7 +16,17 @@ MAX_AGE = 7 * 86400 def candidates(root, support): root, support = Path(root).resolve(), Path(support).resolve() paths = set() - for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)): + native_logs = [] + native = root / '.runtime/nao-users' + if native.is_dir() and not native.is_symlink(): + for actor in native.iterdir(): + if actor.is_symlink() or not actor.is_dir(): continue + native_logs.append((actor / 'logs', 1)) + sessions = actor / 'sessions' + if sessions.is_dir() and not sessions.is_symlink(): + native_logs.extend((session / 'logs', 1) for session in sessions.iterdir() + if session.is_dir() and not session.is_symlink()) + for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1), *native_logs): if not base.is_dir() or base.is_symlink(): continue for directory, dirs, files in os.walk(base, followlinks=False): diff --git a/scripts/nao_log_stream.py b/scripts/nao_log_stream.py new file mode 100644 index 0000000..778629c --- /dev/null +++ b/scripts/nao_log_stream.py @@ -0,0 +1,135 @@ +"""为 NAO 原生子进程捕获输出,写入有大小和时间限制的私有日志。""" +import argparse +import codecs +from contextlib import contextmanager +import logging +from logging.handlers import RotatingFileHandler +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import time + +SEGMENT_BYTES = 10 * 1024**2 +TOTAL_BYTES = 90 * 1024**2 +MAX_AGE = 7 * 86400 +BACKUPS = 8 + + +def safe_file(path): + if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Unsafe log path') + if path.exists(): + info = path.stat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe log file') + + +@contextmanager +def writer_lock(path): + safe_file(path) + fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe writer lock') + if os.name == 'nt': + import msvcrt + if not info.st_size: os.write(fd, b'0') + os.lseek(fd, 0, 0); msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + finally: os.close(fd) + + +def prune(path, total_bytes=TOTAL_BYTES, now=None): + now = time.time() if now is None else now + archives = [path.with_name(path.name + '.' + str(i)) for i in range(1, BACKUPS + 1)] + for archive in archives: safe_file(archive) + existing = sorted((p for p in archives if p.exists()), key=lambda p: p.stat().st_mtime) + total = (path.stat().st_size if path.exists() else 0) + sum(p.stat().st_size for p in existing) + for archive in existing: + if now - archive.stat().st_mtime > MAX_AGE or total > total_bytes: + total -= archive.stat().st_size; archive.unlink() + + +class PrivateHandler(RotatingFileHandler): + def _open(self): + path = Path(self.baseFilename); safe_file(path) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, 'O_NOFOLLOW', 0), 0o600) + if hasattr(os, 'fchmod'): os.fchmod(fd, 0o600) + else: os.chmod(path, 0o600) + return os.fdopen(fd, 'a', encoding='utf-8', errors='replace') + + def handleError(self, record): + # 交给捕获层限频重试;不能因诊断日志失败终止业务子进程。 + raise RuntimeError('Native log write failed') + + +def capture(path, command, segment_bytes=SEGMENT_BYTES, total_bytes=TOTAL_BYTES): + path = Path(path).absolute() + if path.name not in ('server.log', 'fastapi.log') or path.parent.name != 'logs': raise ValueError('Unexpected log name') + safe_file(path); path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with writer_lock(path.with_name('writer-' + path.stem + '.lock')): + prune(path, total_bytes) + handler = PrivateHandler(path, maxBytes=segment_bytes, backupCount=BACKUPS, encoding='utf-8') + handler.setFormatter(logging.Formatter('%(message)s')); handler.terminator = '' + if path.stat().st_size >= segment_bytes or time.time() - path.stat().st_mtime > MAX_AGE: + handler.doRollover(); prune(path, total_bytes) + logger = logging.Logger('native-output', level=logging.INFO); logger.addHandler(handler) + child = None; stopping = False + def stop(signum, frame): + nonlocal stopping + stopping = True + if child is not None and child.poll() is None: + try: child.terminate() + except OSError: pass + previous = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)} + try: + if stopping: return 0 + child = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if stopping: child.terminate() + decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') + retry_at = 0 + def write(text): + nonlocal retry_at + if time.monotonic() < retry_at: return + try: + logger.info(text); prune(path, total_bytes) + except (OSError, RuntimeError, ValueError): + # 继续排空管道,防止日志空间不足拖住正在进行的 AI 请求。 + retry_at = time.monotonic() + 60 + while chunk := child.stdout.read1(min(64 * 1024, max(1, segment_bytes // 8))): + write(decoder.decode(chunk)) + tail = decoder.decode(b'', final=True) + if tail: write(tail) + code = child.wait() + return code if code >= 0 else 128 - code + finally: + for signum, old in previous.items(): signal.signal(signum, old) + try: handler.close() + except OSError: pass + if child is not None: + if child.poll() is None: + child.terminate() + try: child.wait(timeout=10) + except subprocess.TimeoutExpired: child.kill(); child.wait() + child.stdout.close() + + +def main(): + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--log', type=Path, required=True) + parser.add_argument('command', nargs=argparse.REMAINDER) + args = parser.parse_args(); command = args.command + if command[:1] == ['--']: command = command[1:] + if not command: parser.error('A child command is required') + return capture(args.log, command) + + +if __name__ == '__main__': + try: raise SystemExit(main()) + except Exception: + print('Native log capture failed', file=sys.stderr); raise SystemExit(1) diff --git a/scripts/nao_quality_runtime.py b/scripts/nao_quality_runtime.py index 986c69a..590d867 100644 --- a/scripts/nao_quality_runtime.py +++ b/scripts/nao_quality_runtime.py @@ -991,7 +991,7 @@ def _model_settings(*, include_options=False): def _config_fingerprint(model_base: str, model_key: str, model_name: str) -> str: paths = [TEMPLATES / "RULES.md", TEMPLATES / "agent" / "prompts" / "system.md", - ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py", + ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py", ROOT / "scripts" / "nao_log_stream.py", ROOT / "scripts" / "nao_quality_tools.py", ROOT / "scripts" / "nao_quality_mcp.py", ROOT / "scripts" / "nao_model_settings.py", ROOT / "scripts" / "nao_model_gate.py", ROOT / "scripts" / "nao_evidence_finalizer.py", ROOT / "scripts" / "nao_evidence_report.py", ROOT / "scripts" / "nao_weight_ranking.py", ROOT / "scripts" / "nao_metric_request.py", ROOT / "scripts" / "nao_analysis_coverage.py", ROOT / "scripts" / "nao_combined_ranking.py", @@ -1209,6 +1209,11 @@ def _process_matches(record: dict[str, Any], actor_root: Path) -> bool: if not start or start != record.get("lstart"): return False kind = record.get("kind") + wrapper = str(Path(__file__).with_name("nao_log_stream.py")) + if wrapper in command: + name = "fastapi.log" if kind == "fastapi" else "server.log" + if "--log " + str(actor_root / "logs" / name) not in command: + return False if kind == "fastapi": return str(PYTHON) in command and str(FASTAPI_MAIN) in command if kind == "server": @@ -1668,21 +1673,23 @@ class RuntimeManager: path.touch(mode=0o600, exist_ok=True) path.chmod(0o600) processes: list[dict[str, Any]] = [] - log_path = logs / "server.log" - fd = _open_server_log(log_path, actor_root) + wrapper = str(Path(__file__).with_name("nao_log_stream.py")) + for log_path in (logs / "fastapi.log", logs / "server.log"): + os.close(_open_server_log(log_path, actor_root)) try: - with os.fdopen(fd, "ab") as log: - fastapi = subprocess.Popen( - [str(PYTHON), str(FASTAPI_MAIN)], cwd=project, env=env, - stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True, - ) - processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port)) - server = subprocess.Popen( - [str(NAO_SERVER), "serve", "--port", str(server_port), "--host", "127.0.0.1"], - cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL, - stdout=log, stderr=log, start_new_session=True, - ) - processes.append(_process_record(server.pid, "server", server_port)) + fastapi = subprocess.Popen( + [str(PYTHON), wrapper, "--log", str(logs / "fastapi.log"), "--", str(PYTHON), str(FASTAPI_MAIN)], + cwd=project, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, + ) + processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port)) + server = subprocess.Popen( + [str(PYTHON), wrapper, "--log", str(logs / "server.log"), "--", str(NAO_SERVER), + "serve", "--port", str(server_port), "--host", "127.0.0.1"], + cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, + ) + processes.append(_process_record(server.pid, "server", server_port)) except Exception: for record in processes: if _process_matches(record, actor_root): diff --git a/scripts/test_backup_remediation.py b/scripts/test_backup_remediation.py index dcf6106..c009442 100644 --- a/scripts/test_backup_remediation.py +++ b/scripts/test_backup_remediation.py @@ -3,16 +3,54 @@ import gzip import hashlib import io import json +import os from pathlib import Path import tarfile import tempfile import unittest +from types import SimpleNamespace from unittest.mock import patch import backup class BackupRemediationTest(unittest.TestCase): + @contextlib.contextmanager + def private_verifier_fixture(self, root): + """Exercise verify's Darwin/container path without starting a MySQL server.""" + import backup_verifier + + config = root / '.runtime/retention/verify-fixture/client.cnf' + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n') + registration = root / '.runtime/retention/scratch/fixture.json' + registered = [] + child_pids = [] + + @contextlib.contextmanager + def fake_instance(instance_root): + self.assertEqual(Path(instance_root), root) + yield config + + def fake_scratch_register(instance_root, database, server): + self.assertEqual(Path(instance_root), root) + self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$') + self.assertEqual(server, 'synthetic-server-uuid') + registration.parent.mkdir(parents=True, exist_ok=True) + registration.write_text('{}') + registered.append((database, server)) + return registration + + with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \ + patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \ + patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \ + patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \ + patch.object(backup.backup_retention, 'scratch_child', + side_effect=lambda path, pid: child_pids.append((Path(path), pid))): + yield {'config': config, 'instance': instance, 'registration': registration, + 'registered': registered, 'childPids': child_pids, + 'scratchCreated': scratch_created} + def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -23,18 +61,46 @@ class BackupRemediationTest(unittest.TestCase): blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'" self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n' b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n') + # candidate_child() records process creation time via psutil. + # Reuse this live test process as a synthetic child; no command runs. + self.pid = os.getpid() def wait(self): return 0 def poll(self): return 0 def kill(self): pass + seen_client = [] + + def fake_client(statement, database=None): + seen_client.append((statement, database)) + self.assertEqual(database, 'quality_fixture') + if statement == 'SHOW TABLES;': + return 'binary_fixture\n' + if statement == 'SELECT COUNT(*) FROM `binary_fixture`;': + return '1\n' + self.fail('Unexpected isolated export SQL: ' + statement) + with patch.object(backup, 'ROOT', root), \ patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \ patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \ patch.object(backup, 'verify_references', return_value={'verified': True}), \ patch.object(backup, 'query_evidence', return_value={}), \ + patch.object(backup, 'client', side_effect=fake_client), \ patch.object(backup.subprocess, 'Popen', BinaryDump): folder = backup.create() + self.assertEqual(seen_client, [ + ('SHOW TABLES;', 'quality_fixture'), + ('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'), + ('SHOW TABLES;', 'quality_fixture'), + ('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'), + ]) + owner = json.loads((folder / '.owner.json').read_text()) + self.assertEqual(owner['ownerPid'], os.getpid()) + self.assertIsInstance(owner['ownerCreated'], (int, float)) + self.assertFalse(owner['childPending']) + self.assertEqual(len(owner['children']), 1) + self.assertEqual(owner['children'][0]['pid'], os.getpid()) + self.assertIsInstance(owner['children'][0]['created'], (int, float)) manifest = json.loads((folder / 'manifest.json').read_text()) self.assertEqual(manifest['tableCounts']['binary_fixture'], 1) with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump: @@ -189,6 +255,9 @@ class BackupRemediationTest(unittest.TestCase): state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False} def fake_client(statement, database=None): + self.assertEqual(backup.client_config(), private['config']) + if statement == 'SELECT @@server_uuid;': + return 'synthetic-server-uuid\n' if statement.startswith('CREATE DATABASE'): state['database'] = statement.split('`')[1] return '' @@ -221,6 +290,7 @@ class BackupRemediationTest(unittest.TestCase): class FakeMysqlRestore: def __init__(self, *args, **kwargs): state['restoreCommand'] = list(args[0]) + self.pid = 424242 self.stdin = RecordingInput() self.returncode = 0 def wait(self): @@ -230,7 +300,8 @@ class BackupRemediationTest(unittest.TestCase): def kill(self): self.returncode = -9 - with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ + with self.private_verifier_fixture(root) as private, \ + patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ patch.object(backup, 'client', side_effect=fake_client), \ patch.object(backup.subprocess, 'Popen', FakeMysqlRestore): result = backup.verify(folder) @@ -239,6 +310,12 @@ class BackupRemediationTest(unittest.TestCase): self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n')) self.assertTrue(state['updated']) self.assertTrue(state['dropped']) + self.assertTrue(private['instance'].called) + self.assertTrue(private['registered']) + self.assertEqual(private['childPids'], [(private['registration'], 424242)]) + private['scratchCreated'].assert_called_once_with(private['registration']) + self.assertFalse(private['registration'].exists()) + self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get()) self.assertEqual(state['database'].split('_')[0], 'quality') self.assertFalse(result['migrationReady']) self.assertTrue(result['archiveReferencesRebound']) @@ -502,17 +579,23 @@ class BackupRemediationTest(unittest.TestCase): 'tableCounts': {}, 'queryEvidence': None, 'archiveSourceRoot': str(root / 'old/runtimedata/archive')} (folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64})) - with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ + with self.private_verifier_fixture(root) as private, \ + patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ self.assertRaises(backup.BackupFailure) as checksum: backup.verify(folder) self.assertEqual((checksum.exception.stage, checksum.exception.code), ('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')) + private['instance'].assert_called_once_with(root) + self.assertEqual(private['registered'], []) (folder / 'manifest.json').write_text(json.dumps(base_manifest)) state = {'dropped': False} def fake_client(statement, database=None): + self.assertEqual(backup.client_config(), private['config']) if statement.startswith('CREATE DATABASE'): return '' + if statement == 'SELECT @@server_uuid;': + return 'synthetic-server-uuid\n' if statement.startswith('DROP DATABASE'): state['dropped'] = True return '' @@ -522,6 +605,7 @@ class BackupRemediationTest(unittest.TestCase): class FailedMysqlRestore: def __init__(self, *args, **kwargs): + self.pid = 434343 self.stdin = io.BytesIO() self.returncode = 1 def wait(self): @@ -531,7 +615,8 @@ class BackupRemediationTest(unittest.TestCase): def kill(self): self.returncode = -9 - with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ + with self.private_verifier_fixture(root) as private, \ + patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \ patch.object(backup, 'client', side_effect=fake_client), \ patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \ self.assertRaises(backup.BackupFailure) as restore: @@ -539,6 +624,11 @@ class BackupRemediationTest(unittest.TestCase): self.assertEqual((restore.exception.stage, restore.exception.code), ('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED')) self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up') + private['instance'].assert_called_once_with(root) + self.assertEqual(private['childPids'], [(private['registration'], 434343)]) + private['scratchCreated'].assert_called_once_with(private['registration']) + self.assertFalse(private['registration'].exists()) + self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get()) old_root = (root / 'old/runtimedata/archive').resolve() row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12} diff --git a/scripts/test_nao_log_stream.py b/scripts/test_nao_log_stream.py new file mode 100644 index 0000000..ea50f98 --- /dev/null +++ b/scripts/test_nao_log_stream.py @@ -0,0 +1,88 @@ +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +import nao_log_stream as stream +import log_retention + + +class NativeLogTests(unittest.TestCase): + def test_log_write_failure_does_not_interrupt_child_work(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve(); result = root / 'completed.txt' + command = [sys.executable, '-c', 'import sys,pathlib;sys.stdout.write("x"*300000);pathlib.Path(sys.argv[1]).write_text("done")', str(result)] + with patch.object(stream.PrivateHandler, 'emit', side_effect=OSError('Disk full')): + self.assertEqual(stream.capture(root / 'logs/server.log', command), 0) + self.assertEqual(result.read_text(), 'done') + + def test_real_child_output_rolls_and_stays_bounded(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory).resolve() / 'logs/server.log' + command = [sys.executable, '-c', 'import sys;sys.stdout.write("old line\\n"*6000+"LAST_MARKER\\n");sys.stderr.write("stderr marker\\n")'] + self.assertEqual(stream.capture(path, command, segment_bytes=1024, total_bytes=9000), 0) + files = list(path.parent.glob('server.log*')) + self.assertGreater(len(files), 1); self.assertLessEqual(sum(p.stat().st_size for p in files), 9000) + contents = ''.join(p.read_text() for p in files) + self.assertIn('LAST_MARKER', contents); self.assertIn('stderr marker', contents) + for p in files: self.assertEqual(p.stat().st_mode & 0o077, 0) + + def test_oversized_legacy_log_and_expired_archives_are_bounded_on_start(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory).resolve() / 'logs/server.log'; path.parent.mkdir() + path.write_bytes(b'old' * 10000); old = path.with_name('server.log.8'); old.write_bytes(b'expired') + os.utime(old, (time.time() - 9 * 86400,) * 2) + stream.capture(path, [sys.executable, '-c', 'print("new")'], segment_bytes=1024, total_bytes=9000) + self.assertIn('new', path.read_text()); self.assertFalse(old.exists()) + self.assertLessEqual(sum(p.stat().st_size for p in path.parent.glob('server.log*')), 9000) + + def test_symlink_log_or_archive_is_rejected_before_child_launch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve(); path = root / 'logs/server.log'; path.parent.mkdir() + outside = root / 'data.xlsx'; outside.write_bytes(b'business data') + path.with_name('server.log.1').symlink_to(outside) + with self.assertRaises(ValueError): stream.capture(path, [sys.executable, '-c', 'print("no")']) + self.assertEqual(outside.read_bytes(), b'business data') + + @unittest.skipIf(os.name == 'nt', 'POSIX process-group shutdown contract') + def test_stop_keeps_wrapper_until_term_ignoring_child_can_be_killed_as_a_group(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve(); path = root / 'logs/fastapi.log' + command = [sys.executable, str(Path(stream.__file__).resolve()), '--log', str(path), '--', sys.executable, '-u', '-c', + 'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);print("READY",flush=True);time.sleep(60)'] + process = subprocess.Popen(command, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + deadline = time.monotonic() + 5 + while not path.exists() or 'READY' not in path.read_text(): + if time.monotonic() > deadline: self.fail('Native wrapper did not start') + time.sleep(.02) + import nao_quality_runtime as runtime + with patch.object(runtime, 'PYTHON', Path(sys.executable)), patch.object(runtime, 'FASTAPI_MAIN', Path('-c')): + record=runtime._process_record(process.pid,'fastapi',5283) + self.assertTrue(runtime._process_matches(record,root)) + self.assertFalse(runtime._process_matches(record,root/'another-session')) + os.killpg(process.pid, signal.SIGTERM); time.sleep(.1) + self.assertIsNone(process.poll()) + os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5) + finally: + if process.poll() is None: os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5) + + def test_native_actor_and_session_logs_are_cleaned_but_business_files_and_open_logs_are_kept(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve(); support = root / 'support' + actor = root / '.runtime/nao-users/actor' + paths = [actor / 'logs/server.log', actor / 'sessions/session/logs/fastapi.log.1'] + for path in paths: + path.parent.mkdir(parents=True); path.write_bytes(b'old diagnostics\n'); os.utime(path, (time.time() - 9 * 86400,) * 2) + private = actor / 'sessions/session/private/nao.sqlite'; private.parent.mkdir(); private.write_bytes(b'keep') + self.assertTrue(set(paths).issubset(log_retention.candidates(root, support))) + result = log_retention.retain(root, support, open_paths={str(paths[0])}) + self.assertEqual(result['logsRemoved'], 1); self.assertTrue(paths[0].exists()); self.assertTrue(private.exists()) + + +if __name__ == '__main__': unittest.main() diff --git a/scripts/test_retention_recovery.py b/scripts/test_retention_recovery.py new file mode 100644 index 0000000..0647175 --- /dev/null +++ b/scripts/test_retention_recovery.py @@ -0,0 +1,106 @@ +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')] +import backup +import backup_retention as br +import backup_verifier +import operations +import retention +from test_retention import good_backup + + +class CandidateRecoveryTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(); self.root = Path(self.temporary.name).resolve() + def tearDown(self): self.temporary.cleanup() + + def candidate(self, suffix='22222222', dead=True, age=3 * 86400): + folder = self.root / 'output/backups' / ('20261108T010000Z-' + suffix) + folder.mkdir(parents=True); (folder / 'database.sql.gz').write_bytes(b'partial dump') + br.register_candidate(folder, self.root) + owner = json.loads((folder / '.owner.json').read_text()); owner['createdAt'] -= age + if dead: owner.update({'ownerPid': 2000000000, 'ownerCreated': 1.0}) + br.atomic(folder / '.owner.json', owner); os.utime(folder, (time.time() - age,) * 2) + return folder + + def test_expired_dead_candidate_is_cleaned_without_a_new_backup(self): + current = good_backup(self.root, '11111111'); br.rotate(self.root, current) + orphan = self.candidate() + result = br.clean_incomplete(self.root) + self.assertEqual(result['incompleteBackupsRemoved'], 1) + self.assertGreater(result['incompleteBackupBytesRemoved'], 0) + self.assertFalse(orphan.exists()); self.assertEqual(br.latest(self.root)[0], current) + + def test_live_parent_live_child_pending_and_unknown_records_are_preserved(self): + parent = self.candidate('11111111', dead=False) + child = self.candidate('22222222'); br.candidate_child(parent, os.getpid()) + value = json.loads((parent / '.owner.json').read_text()) + child_owner = json.loads((child / '.owner.json').read_text()); child_owner['children'] = value['children'] + br.atomic(child / '.owner.json', child_owner); os.utime(child, (time.time() - 3 * 86400,) * 2) + pending = self.candidate('33333333'); owner = json.loads((pending / '.owner.json').read_text()) + owner['childPending'] = True; br.atomic(pending / '.owner.json', owner); os.utime(pending, (time.time() - 3 * 86400,) * 2) + unknown = self.candidate('44444444'); owner = json.loads((unknown / '.owner.json').read_text()) + owner.pop('ownerCreated'); br.atomic(unknown / '.owner.json', owner); os.utime(unknown, (time.time() - 3 * 86400,) * 2) + result = br.clean_incomplete(self.root) + self.assertEqual(result['incompleteBackupsRemoved'], 0) + self.assertGreaterEqual(result['incompleteBackupsDeferred'], 3) + for folder in (parent, child, pending, unknown): self.assertTrue(folder.exists()) + + def test_latest_pointer_valid_candidate_fresh_and_symlink_contents_are_protected(self): + pointed = self.candidate('11111111'); br.atomic(self.root / '.runtime/retention/latest.json', {'id': pointed.name}) + valid = good_backup(self.root, '55555555') + fresh = self.candidate('22222222', age=10) + linked = self.candidate('33333333'); outside = self.root / 'business.xlsx'; outside.write_bytes(b'keep') + (linked / 'link').symlink_to(outside) + old = time.time() - 3 * 86400; os.utime(linked, (old, old)) + result = br.clean_incomplete(self.root) + self.assertEqual(result['incompleteBackupsRemoved'], 0) + for p in (pointed, valid, fresh, linked, outside): self.assertTrue(p.exists()) + + def test_monitor_cleanup_precedes_database_access(self): + orphan = self.candidate() + with patch.object(retention, 'Database', side_effect=RuntimeError('Database unavailable')): + with self.assertRaises(RuntimeError): operations.janitor(self.root, self.root / 'support', operations.DEFAULTS) + self.assertFalse(orphan.exists()) + + def test_low_disk_recovery_removes_orphan_before_deferring_new_backup(self): + current = good_backup(self.root, '11111111'); br.rotate(self.root, current); orphan = self.candidate() + class Reply: + def __enter__(self): return self + def __exit__(self, *_): return False + def read(self, *_): return b'{"status":"UP"}' + class Opener: + def open(self, *_args, **_kwargs): return Reply() + class Database: + def __init__(self, root): self.root = root; self.name = 'synthetic'; self.conn = self + def close(self): pass + def clean_owned_scratch(self, *_): return 0 + def current(self): return 'current' + def busy(self): return False + def rows(self, *_): return [{'size': 1}] + with patch.object(retention.urllib.request, 'build_opener', return_value=Opener()), \ + patch.object(retention, 'Database', Database), patch.object(backup_verifier, 'clean_abandoned', return_value=0), \ + patch.object(retention.shutil, 'disk_usage', return_value=shutil._ntuple_diskusage(100, 99, 1)), \ + patch.object(backup, 'create') as create: + with self.assertRaises(retention.Busy): retention.run(self.root) + self.assertFalse(orphan.exists()); self.assertFalse(create.called) + self.assertEqual(br.latest(self.root)[0], current) + + def test_unreadable_process_identity_defers_cleanup(self): + import psutil + orphan = self.candidate() + with patch.object(psutil, 'Process', side_effect=psutil.AccessDenied()): + result = br.clean_incomplete(self.root) + self.assertEqual(result['incompleteBackupsRemoved'], 0); self.assertTrue(orphan.exists()) + + +if __name__ == '__main__': unittest.main()