修复:独立回收中断备份并限制 AI 日志增长

This commit is contained in:
2026-11-10 16:02:46 +08:00
parent dc5b9b76ec
commit cc8832c8e5
15 changed files with 584 additions and 39 deletions
+2
View File
@@ -47,8 +47,10 @@ python3 deploy/docker/manage.py stop
- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。
- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。
- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。
- 未完成的备份候选另行每 5 分钟检查,超过 24 小时、所有者和已登记子进程确认退出后才回收;该步骤不依赖新备份成功,也不要求数据库可连接。最新备份指针、有效候选、活跃进程、身份不完整的旧记录和未知目录均保留。
- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。
- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。
- AI 原生服务输出由受管捕获进程滚动写入,单段 10MiB、每个服务日志总量约 90MiB,归档最长 7 天。后台清理同时覆盖各用户及会话的日志;正在写入的文件继续保护。日志写入失败会限频重试并持续排空输出,不会为清日志终止正在进行的 AI 请求。已有运行实例继续工作,下次正常创建实例时采用新的日志捕获方式。
- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。
Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。
+17
View File
@@ -4,6 +4,7 @@ import os
from pathlib import Path
import sys
import tempfile
import time
import unittest
from unittest.mock import Mock, patch
@@ -24,6 +25,22 @@ import backup_retention as br
class ContainerOperationsTests(unittest.TestCase):
def test_database_unavailable_does_not_block_expired_candidate_recovery(self):
worker = module('container_worker', ROOT / 'deploy/docker/worker.py')
with tempfile.TemporaryDirectory() as directory:
root=Path(directory).resolve();folder=root/'output/backups/20261108T010000Z-12345678';folder.mkdir(parents=True)
(folder/'database.sql.gz').write_bytes(b'interrupted dump');br.register_candidate(folder,root)
value=json.loads((folder/'.owner.json').read_text());value.update({'createdAt':time.time()-3*86400,'ownerPid':2000000000,'ownerCreated':1.0})
br.atomic(folder/'.owner.json',value);os.utime(folder,(time.time()-3*86400,)*2)
br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':'quality_current'})
br.atomic(root/'.runtime/operations/policy.json',worker.operations.DEFAULTS)
with patch.object(worker,'ROOT',root),patch.object(worker.operations,'snapshot',worker.operations.snapshot), \
patch('builtins.print') as output, \
patch.object(backup,'client',side_effect=RuntimeError('Database unavailable')):
with self.assertRaises(RuntimeError):worker.once('monitor')
self.assertFalse(folder.exists())
self.assertEqual(json.loads(output.call_args.args[0])['incompleteBackupsRemoved'],1)
def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678'
+12 -3
View File
@@ -39,17 +39,26 @@ def snapshot(root, support=None, now=None):
mysql = estimates.get(cfg['mysqlDatabase'], 0)
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
native = operations.allocated(root / '.runtime/nao-users')
logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations'))
core = mysql + archive + backups + native + logs
log_paths = log_retention.candidates(root, root / '.runtime/operations')
logs = sum(p.stat().st_blocks * 512 for p in log_paths)
native_logs = sum(p.stat().st_blocks * 512 for p in log_paths if p.is_relative_to(root / '.runtime/nao-users'))
core = mysql + archive + backups + max(0, native - native_logs) + logs
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native,
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': max(0, native - native_logs),
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
'overdueTestResources': overdue}
def once(role):
# 数据库不可连接时也要能释放中断备份占用,不能放在采样或健康检查之后。
if not (ROOT / '.runtime/retention/paused').exists():
try:
result = br.clean_incomplete(ROOT)
if result['incompleteBackupsRemoved']:
print(json.dumps({'event': 'incomplete-backup-cleanup', **result}), flush=True)
except br.Busy: pass
if role == 'monitor':
operations.snapshot = snapshot
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
+2
View File
@@ -85,6 +85,8 @@ PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discove
注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。
未完成备份候选使用独立回收步骤:超过 24 小时且进程身份确认退出后才清理,不等待下一次备份成功;当前备份、有效候选、身份不完整和仍运行的记录保留。AI 用户及会话日志也纳入后台检查;新启动的原生服务通过 `nao_log_stream.py` 捕获输出,单段 10MiB、每服务约 90MiB、归档最长 7 天。捕获进程与原生子进程共享进程组,沿用原有会话身份与停止保护;日志写入异常不会为清理中断 AI 请求。已有实例不因升级被终止,下次正常启动时加载新方式。
```sh
.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD"
.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD"
+9 -6
View File
@@ -47,6 +47,7 @@ def snapshot(root, support=SUPPORT, now=None):
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
native = allocated(root / '.runtime/nao-users')
native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users'))
registered = 0; overdue = 0
for p in resources.registry(root).glob('*.json'):
value = resources.load(root, p.stem)
@@ -61,10 +62,11 @@ def snapshot(root, support=SUPPORT, now=None):
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
# Core service data is the same scope as the earlier 11.86GiB measurement.
# Software/build/delivery artifacts are visible separately and included in growth checks.
core = max(0, mysql - registered) + archives + backups + log_bytes + native
artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs)
native_data = max(0, native - native_logs)
core = max(0, mysql - registered) + archives + backups + log_bytes + native_data
artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs)
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered,
'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered,
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
@@ -143,17 +145,18 @@ def janitor(root, support, policy):
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
try:
with br.lock(root, timeout=0):
result = br.clean_incomplete(root)
db = Database(root)
try:
config = json.loads((root / '.runtime/retention/config.json').read_text())
accepted = br.accepted_backend(root)
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
return {'deferred': 'Application version changed'}
return {**result, 'deferred': 'Application version changed'}
db.current()
if db.busy(): return {'deferred': 'Application jobs are active'}
if db.busy(): return {**result, 'deferred': 'Application jobs are active'}
# Use a short-lived CLI connection so no observer holds a test schema open.
import backup
result = resources.cleanup(root, backup.client)
result.update(resources.cleanup(root, backup.client))
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
result.update(log_retention.retain(root, support,
+1
View File
@@ -372,6 +372,7 @@ def run(root, max_batches=30, grace=1800):
state=json.loads(state_path.read_text()) if state_path.exists() else {}
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
with br.lock(root,timeout=0):
result.update(br.clean_incomplete(root))
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
db=Database(root)
+10
View File
@@ -21,6 +21,16 @@ ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
class OperationsTests(unittest.TestCase):
def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d).resolve();native=root/'.runtime/nao-users/actor'
for relative in ['logs/server.log','private/nao.sqlite']:
path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192)
(root/'.runtime/tool.bin').write_bytes(b'x'*16384)
sample=ops.snapshot(root,root/'support')
self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes']))
self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users'))
def sample(self, **extra):
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
+3 -2
View File
@@ -369,8 +369,7 @@ def create():
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
backup_retention.register_candidate(folder, ROOT)
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
@@ -378,7 +377,9 @@ def create():
process = None
try:
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
backup_retention.candidate_child(folder, pending=True)
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
backup_retention.candidate_child(folder, process.pid)
try:
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
shutil.copyfileobj(process.stdout, compressed)
+73 -9
View File
@@ -4,6 +4,7 @@ from datetime import datetime, timezone
from functools import wraps
import hashlib
import json
import math
import os
from pathlib import Path
import re
@@ -166,16 +167,9 @@ def _rotate(root, selected):
for folder in base.iterdir():
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
continue
# A restore-verified older folder is safe to retire under the shared lock.
# Incomplete candidates are kept for 24h and never treated as good backups.
# 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。
older = verified(folder, checksums=False)
try:
owner = json.loads((folder / '.owner.json').read_text())
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
except (OSError, ValueError):
owned = False
stale = owned and time.time() - folder.stat().st_mtime > 86400
if older is not None or stale:
if older is not None:
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents):
continue
@@ -204,6 +198,7 @@ def _rotate(root, selected):
retire_folder(root,legacy,'legacy')
legacy_info['retired']=True
atomic(registration,legacy_info)
clean_incomplete(root)
clean_retired(root)
return pointer, removed
@@ -282,6 +277,75 @@ def backup_ids(root):
return {p.name for p in base.iterdir()} if base.exists() else set()
def register_candidate(folder, root):
import psutil
folder = Path(folder); info = folder.stat()
atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2,
'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(),
'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(),
'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False})
def candidate_child(folder, pid=None, pending=False):
import psutil
path = Path(folder) / '.owner.json'; value = json.loads(path.read_text())
if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed')
value['childPending'] = pending
if pid is not None:
value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()})
atomic(path, value)
def clean_incomplete(root, now=None):
"""独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。"""
import psutil
root = Path(root).resolve(); now = time.time() if now is None else now
result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0}
base = root / 'output/backups'
if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root')
def exited(item, pid_key, created_key):
if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or
not isinstance(item.get(created_key), (float, int)) or
not math.isfinite(item[created_key]) or item[created_key] <= 0): return False
try:
process = psutil.Process(item[pid_key])
return (abs(process.create_time() - item[created_key]) >= .02 or
not process.is_running() or process.status() == psutil.STATUS_ZOMBIE)
except psutil.NoSuchProcess: return True
except psutil.Error: return False
with lock(root, timeout=0):
pointer = root / '.runtime/retention/latest.json'
try:
if pointer.is_symlink(): return result
protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None
except (OSError, ValueError, KeyError, TypeError): return result
for folder in base.iterdir() if base.exists() else []:
if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue
owner = folder / '.owner.json'
if owner.is_symlink(): continue
try:
value = json.loads(owner.read_text()); info = folder.stat()
if not isinstance(value, dict): continue
if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or
value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or
not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue
if verified(folder, checksums=False) is not None: continue
if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or
value.get('childPending') is not False or not isinstance(value.get('children'), list) or
not exited(value, 'ownerPid', 'ownerCreated') or
not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])):
result['incompleteBackupsDeferred'] += 1; continue
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents): continue
size = sum(p.stat().st_size for p in contents if p.is_file())
if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue
retire_folder(root, folder, 'backup')
result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size
except (OSError, ValueError, TypeError): continue
clean_retired(root)
return result
def discard_failed_new(root, before):
"""Delete only this call's registered, incomplete candidate after it has stopped."""
root=Path(root).resolve();base=root/'output/backups'
+11 -1
View File
@@ -16,7 +16,17 @@ MAX_AGE = 7 * 86400
def candidates(root, support):
root, support = Path(root).resolve(), Path(support).resolve()
paths = set()
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
native_logs = []
native = root / '.runtime/nao-users'
if native.is_dir() and not native.is_symlink():
for actor in native.iterdir():
if actor.is_symlink() or not actor.is_dir(): continue
native_logs.append((actor / 'logs', 1))
sessions = actor / 'sessions'
if sessions.is_dir() and not sessions.is_symlink():
native_logs.extend((session / 'logs', 1) for session in sessions.iterdir()
if session.is_dir() and not session.is_symlink())
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1), *native_logs):
if not base.is_dir() or base.is_symlink():
continue
for directory, dirs, files in os.walk(base, followlinks=False):
+135
View File
@@ -0,0 +1,135 @@
"""为 NAO 原生子进程捕获输出,写入有大小和时间限制的私有日志。"""
import argparse
import codecs
from contextlib import contextmanager
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import signal
import stat
import subprocess
import sys
import time
SEGMENT_BYTES = 10 * 1024**2
TOTAL_BYTES = 90 * 1024**2
MAX_AGE = 7 * 86400
BACKUPS = 8
def safe_file(path):
if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Unsafe log path')
if path.exists():
info = path.stat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe log file')
@contextmanager
def writer_lock(path):
safe_file(path)
fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600)
try:
info = os.fstat(fd)
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe writer lock')
if os.name == 'nt':
import msvcrt
if not info.st_size: os.write(fd, b'0')
os.lseek(fd, 0, 0); msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield
finally: os.close(fd)
def prune(path, total_bytes=TOTAL_BYTES, now=None):
now = time.time() if now is None else now
archives = [path.with_name(path.name + '.' + str(i)) for i in range(1, BACKUPS + 1)]
for archive in archives: safe_file(archive)
existing = sorted((p for p in archives if p.exists()), key=lambda p: p.stat().st_mtime)
total = (path.stat().st_size if path.exists() else 0) + sum(p.stat().st_size for p in existing)
for archive in existing:
if now - archive.stat().st_mtime > MAX_AGE or total > total_bytes:
total -= archive.stat().st_size; archive.unlink()
class PrivateHandler(RotatingFileHandler):
def _open(self):
path = Path(self.baseFilename); safe_file(path)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, 'O_NOFOLLOW', 0), 0o600)
if hasattr(os, 'fchmod'): os.fchmod(fd, 0o600)
else: os.chmod(path, 0o600)
return os.fdopen(fd, 'a', encoding='utf-8', errors='replace')
def handleError(self, record):
# 交给捕获层限频重试;不能因诊断日志失败终止业务子进程。
raise RuntimeError('Native log write failed')
def capture(path, command, segment_bytes=SEGMENT_BYTES, total_bytes=TOTAL_BYTES):
path = Path(path).absolute()
if path.name not in ('server.log', 'fastapi.log') or path.parent.name != 'logs': raise ValueError('Unexpected log name')
safe_file(path); path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with writer_lock(path.with_name('writer-' + path.stem + '.lock')):
prune(path, total_bytes)
handler = PrivateHandler(path, maxBytes=segment_bytes, backupCount=BACKUPS, encoding='utf-8')
handler.setFormatter(logging.Formatter('%(message)s')); handler.terminator = ''
if path.stat().st_size >= segment_bytes or time.time() - path.stat().st_mtime > MAX_AGE:
handler.doRollover(); prune(path, total_bytes)
logger = logging.Logger('native-output', level=logging.INFO); logger.addHandler(handler)
child = None; stopping = False
def stop(signum, frame):
nonlocal stopping
stopping = True
if child is not None and child.poll() is None:
try: child.terminate()
except OSError: pass
previous = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
try:
if stopping: return 0
child = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if stopping: child.terminate()
decoder = codecs.getincrementaldecoder('utf-8')(errors='replace')
retry_at = 0
def write(text):
nonlocal retry_at
if time.monotonic() < retry_at: return
try:
logger.info(text); prune(path, total_bytes)
except (OSError, RuntimeError, ValueError):
# 继续排空管道,防止日志空间不足拖住正在进行的 AI 请求。
retry_at = time.monotonic() + 60
while chunk := child.stdout.read1(min(64 * 1024, max(1, segment_bytes // 8))):
write(decoder.decode(chunk))
tail = decoder.decode(b'', final=True)
if tail: write(tail)
code = child.wait()
return code if code >= 0 else 128 - code
finally:
for signum, old in previous.items(): signal.signal(signum, old)
try: handler.close()
except OSError: pass
if child is not None:
if child.poll() is None:
child.terminate()
try: child.wait(timeout=10)
except subprocess.TimeoutExpired: child.kill(); child.wait()
child.stdout.close()
def main():
os.umask(0o077)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--log', type=Path, required=True)
parser.add_argument('command', nargs=argparse.REMAINDER)
args = parser.parse_args(); command = args.command
if command[:1] == ['--']: command = command[1:]
if not command: parser.error('A child command is required')
return capture(args.log, command)
if __name__ == '__main__':
try: raise SystemExit(main())
except Exception:
print('Native log capture failed', file=sys.stderr); raise SystemExit(1)
+22 -15
View File
@@ -991,7 +991,7 @@ def _model_settings(*, include_options=False):
def _config_fingerprint(model_base: str, model_key: str, model_name: str) -> str:
paths = [TEMPLATES / "RULES.md", TEMPLATES / "agent" / "prompts" / "system.md",
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py",
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py", ROOT / "scripts" / "nao_log_stream.py",
ROOT / "scripts" / "nao_quality_tools.py", ROOT / "scripts" / "nao_quality_mcp.py",
ROOT / "scripts" / "nao_model_settings.py", ROOT / "scripts" / "nao_model_gate.py", ROOT / "scripts" / "nao_evidence_finalizer.py", ROOT / "scripts" / "nao_evidence_report.py", ROOT / "scripts" / "nao_weight_ranking.py",
ROOT / "scripts" / "nao_metric_request.py", ROOT / "scripts" / "nao_analysis_coverage.py", ROOT / "scripts" / "nao_combined_ranking.py",
@@ -1209,6 +1209,11 @@ def _process_matches(record: dict[str, Any], actor_root: Path) -> bool:
if not start or start != record.get("lstart"):
return False
kind = record.get("kind")
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
if wrapper in command:
name = "fastapi.log" if kind == "fastapi" else "server.log"
if "--log " + str(actor_root / "logs" / name) not in command:
return False
if kind == "fastapi":
return str(PYTHON) in command and str(FASTAPI_MAIN) in command
if kind == "server":
@@ -1668,21 +1673,23 @@ class RuntimeManager:
path.touch(mode=0o600, exist_ok=True)
path.chmod(0o600)
processes: list[dict[str, Any]] = []
log_path = logs / "server.log"
fd = _open_server_log(log_path, actor_root)
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
for log_path in (logs / "fastapi.log", logs / "server.log"):
os.close(_open_server_log(log_path, actor_root))
try:
with os.fdopen(fd, "ab") as log:
fastapi = subprocess.Popen(
[str(PYTHON), str(FASTAPI_MAIN)], cwd=project, env=env,
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True,
)
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
server = subprocess.Popen(
[str(NAO_SERVER), "serve", "--port", str(server_port), "--host", "127.0.0.1"],
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL,
stdout=log, stderr=log, start_new_session=True,
)
processes.append(_process_record(server.pid, "server", server_port))
fastapi = subprocess.Popen(
[str(PYTHON), wrapper, "--log", str(logs / "fastapi.log"), "--", str(PYTHON), str(FASTAPI_MAIN)],
cwd=project, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True,
)
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
server = subprocess.Popen(
[str(PYTHON), wrapper, "--log", str(logs / "server.log"), "--", str(NAO_SERVER),
"serve", "--port", str(server_port), "--host", "127.0.0.1"],
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True,
)
processes.append(_process_record(server.pid, "server", server_port))
except Exception:
for record in processes:
if _process_matches(record, actor_root):
+93 -3
View File
@@ -3,16 +3,54 @@ import gzip
import hashlib
import io
import json
import os
from pathlib import Path
import tarfile
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import backup
class BackupRemediationTest(unittest.TestCase):
@contextlib.contextmanager
def private_verifier_fixture(self, root):
"""Exercise verify's Darwin/container path without starting a MySQL server."""
import backup_verifier
config = root / '.runtime/retention/verify-fixture/client.cnf'
config.parent.mkdir(parents=True, exist_ok=True)
config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n')
registration = root / '.runtime/retention/scratch/fixture.json'
registered = []
child_pids = []
@contextlib.contextmanager
def fake_instance(instance_root):
self.assertEqual(Path(instance_root), root)
yield config
def fake_scratch_register(instance_root, database, server):
self.assertEqual(Path(instance_root), root)
self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$')
self.assertEqual(server, 'synthetic-server-uuid')
registration.parent.mkdir(parents=True, exist_ok=True)
registration.write_text('{}')
registered.append((database, server))
return registration
with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \
patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \
patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \
patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \
patch.object(backup.backup_retention, 'scratch_child',
side_effect=lambda path, pid: child_pids.append((Path(path), pid))):
yield {'config': config, 'instance': instance, 'registration': registration,
'registered': registered, 'childPids': child_pids,
'scratchCreated': scratch_created}
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -23,18 +61,46 @@ class BackupRemediationTest(unittest.TestCase):
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
# candidate_child() records process creation time via psutil.
# Reuse this live test process as a synthetic child; no command runs.
self.pid = os.getpid()
def wait(self): return 0
def poll(self): return 0
def kill(self): pass
seen_client = []
def fake_client(statement, database=None):
seen_client.append((statement, database))
self.assertEqual(database, 'quality_fixture')
if statement == 'SHOW TABLES;':
return 'binary_fixture\n'
if statement == 'SELECT COUNT(*) FROM `binary_fixture`;':
return '1\n'
self.fail('Unexpected isolated export SQL: ' + statement)
with patch.object(backup, 'ROOT', root), \
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
patch.object(backup, 'verify_references', return_value={'verified': True}), \
patch.object(backup, 'query_evidence', return_value={}), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', BinaryDump):
folder = backup.create()
self.assertEqual(seen_client, [
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
])
owner = json.loads((folder / '.owner.json').read_text())
self.assertEqual(owner['ownerPid'], os.getpid())
self.assertIsInstance(owner['ownerCreated'], (int, float))
self.assertFalse(owner['childPending'])
self.assertEqual(len(owner['children']), 1)
self.assertEqual(owner['children'][0]['pid'], os.getpid())
self.assertIsInstance(owner['children'][0]['created'], (int, float))
manifest = json.loads((folder / 'manifest.json').read_text())
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
@@ -189,6 +255,9 @@ class BackupRemediationTest(unittest.TestCase):
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('CREATE DATABASE'):
state['database'] = statement.split('`')[1]
return ''
@@ -221,6 +290,7 @@ class BackupRemediationTest(unittest.TestCase):
class FakeMysqlRestore:
def __init__(self, *args, **kwargs):
state['restoreCommand'] = list(args[0])
self.pid = 424242
self.stdin = RecordingInput()
self.returncode = 0
def wait(self):
@@ -230,7 +300,8 @@ class BackupRemediationTest(unittest.TestCase):
def kill(self):
self.returncode = -9
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
result = backup.verify(folder)
@@ -239,6 +310,12 @@ class BackupRemediationTest(unittest.TestCase):
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
self.assertTrue(state['updated'])
self.assertTrue(state['dropped'])
self.assertTrue(private['instance'].called)
self.assertTrue(private['registered'])
self.assertEqual(private['childPids'], [(private['registration'], 424242)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
self.assertEqual(state['database'].split('_')[0], 'quality')
self.assertFalse(result['migrationReady'])
self.assertTrue(result['archiveReferencesRebound'])
@@ -502,17 +579,23 @@ class BackupRemediationTest(unittest.TestCase):
'tableCounts': {}, 'queryEvidence': None,
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
self.assertRaises(backup.BackupFailure) as checksum:
backup.verify(folder)
self.assertEqual((checksum.exception.stage, checksum.exception.code),
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
private['instance'].assert_called_once_with(root)
self.assertEqual(private['registered'], [])
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
state = {'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement.startswith('CREATE DATABASE'):
return ''
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('DROP DATABASE'):
state['dropped'] = True
return ''
@@ -522,6 +605,7 @@ class BackupRemediationTest(unittest.TestCase):
class FailedMysqlRestore:
def __init__(self, *args, **kwargs):
self.pid = 434343
self.stdin = io.BytesIO()
self.returncode = 1
def wait(self):
@@ -531,7 +615,8 @@ class BackupRemediationTest(unittest.TestCase):
def kill(self):
self.returncode = -9
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
self.assertRaises(backup.BackupFailure) as restore:
@@ -539,6 +624,11 @@ class BackupRemediationTest(unittest.TestCase):
self.assertEqual((restore.exception.stage, restore.exception.code),
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
private['instance'].assert_called_once_with(root)
self.assertEqual(private['childPids'], [(private['registration'], 434343)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
old_root = (root / 'old/runtimedata/archive').resolve()
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
+88
View File
@@ -0,0 +1,88 @@
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
import nao_log_stream as stream
import log_retention
class NativeLogTests(unittest.TestCase):
def test_log_write_failure_does_not_interrupt_child_work(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); result = root / 'completed.txt'
command = [sys.executable, '-c', 'import sys,pathlib;sys.stdout.write("x"*300000);pathlib.Path(sys.argv[1]).write_text("done")', str(result)]
with patch.object(stream.PrivateHandler, 'emit', side_effect=OSError('Disk full')):
self.assertEqual(stream.capture(root / 'logs/server.log', command), 0)
self.assertEqual(result.read_text(), 'done')
def test_real_child_output_rolls_and_stays_bounded(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory).resolve() / 'logs/server.log'
command = [sys.executable, '-c', 'import sys;sys.stdout.write("old line\\n"*6000+"LAST_MARKER\\n");sys.stderr.write("stderr marker\\n")']
self.assertEqual(stream.capture(path, command, segment_bytes=1024, total_bytes=9000), 0)
files = list(path.parent.glob('server.log*'))
self.assertGreater(len(files), 1); self.assertLessEqual(sum(p.stat().st_size for p in files), 9000)
contents = ''.join(p.read_text() for p in files)
self.assertIn('LAST_MARKER', contents); self.assertIn('stderr marker', contents)
for p in files: self.assertEqual(p.stat().st_mode & 0o077, 0)
def test_oversized_legacy_log_and_expired_archives_are_bounded_on_start(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory).resolve() / 'logs/server.log'; path.parent.mkdir()
path.write_bytes(b'old' * 10000); old = path.with_name('server.log.8'); old.write_bytes(b'expired')
os.utime(old, (time.time() - 9 * 86400,) * 2)
stream.capture(path, [sys.executable, '-c', 'print("new")'], segment_bytes=1024, total_bytes=9000)
self.assertIn('new', path.read_text()); self.assertFalse(old.exists())
self.assertLessEqual(sum(p.stat().st_size for p in path.parent.glob('server.log*')), 9000)
def test_symlink_log_or_archive_is_rejected_before_child_launch(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); path = root / 'logs/server.log'; path.parent.mkdir()
outside = root / 'data.xlsx'; outside.write_bytes(b'business data')
path.with_name('server.log.1').symlink_to(outside)
with self.assertRaises(ValueError): stream.capture(path, [sys.executable, '-c', 'print("no")'])
self.assertEqual(outside.read_bytes(), b'business data')
@unittest.skipIf(os.name == 'nt', 'POSIX process-group shutdown contract')
def test_stop_keeps_wrapper_until_term_ignoring_child_can_be_killed_as_a_group(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); path = root / 'logs/fastapi.log'
command = [sys.executable, str(Path(stream.__file__).resolve()), '--log', str(path), '--', sys.executable, '-u', '-c',
'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);print("READY",flush=True);time.sleep(60)']
process = subprocess.Popen(command, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
deadline = time.monotonic() + 5
while not path.exists() or 'READY' not in path.read_text():
if time.monotonic() > deadline: self.fail('Native wrapper did not start')
time.sleep(.02)
import nao_quality_runtime as runtime
with patch.object(runtime, 'PYTHON', Path(sys.executable)), patch.object(runtime, 'FASTAPI_MAIN', Path('-c')):
record=runtime._process_record(process.pid,'fastapi',5283)
self.assertTrue(runtime._process_matches(record,root))
self.assertFalse(runtime._process_matches(record,root/'another-session'))
os.killpg(process.pid, signal.SIGTERM); time.sleep(.1)
self.assertIsNone(process.poll())
os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
finally:
if process.poll() is None: os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
def test_native_actor_and_session_logs_are_cleaned_but_business_files_and_open_logs_are_kept(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); support = root / 'support'
actor = root / '.runtime/nao-users/actor'
paths = [actor / 'logs/server.log', actor / 'sessions/session/logs/fastapi.log.1']
for path in paths:
path.parent.mkdir(parents=True); path.write_bytes(b'old diagnostics\n'); os.utime(path, (time.time() - 9 * 86400,) * 2)
private = actor / 'sessions/session/private/nao.sqlite'; private.parent.mkdir(); private.write_bytes(b'keep')
self.assertTrue(set(paths).issubset(log_retention.candidates(root, support)))
result = log_retention.retain(root, support, open_paths={str(paths[0])})
self.assertEqual(result['logsRemoved'], 1); self.assertTrue(paths[0].exists()); self.assertTrue(private.exists())
if __name__ == '__main__': unittest.main()
+106
View File
@@ -0,0 +1,106 @@
import json
import os
from pathlib import Path
import shutil
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
import backup
import backup_retention as br
import backup_verifier
import operations
import retention
from test_retention import good_backup
class CandidateRecoveryTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(); self.root = Path(self.temporary.name).resolve()
def tearDown(self): self.temporary.cleanup()
def candidate(self, suffix='22222222', dead=True, age=3 * 86400):
folder = self.root / 'output/backups' / ('20261108T010000Z-' + suffix)
folder.mkdir(parents=True); (folder / 'database.sql.gz').write_bytes(b'partial dump')
br.register_candidate(folder, self.root)
owner = json.loads((folder / '.owner.json').read_text()); owner['createdAt'] -= age
if dead: owner.update({'ownerPid': 2000000000, 'ownerCreated': 1.0})
br.atomic(folder / '.owner.json', owner); os.utime(folder, (time.time() - age,) * 2)
return folder
def test_expired_dead_candidate_is_cleaned_without_a_new_backup(self):
current = good_backup(self.root, '11111111'); br.rotate(self.root, current)
orphan = self.candidate()
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 1)
self.assertGreater(result['incompleteBackupBytesRemoved'], 0)
self.assertFalse(orphan.exists()); self.assertEqual(br.latest(self.root)[0], current)
def test_live_parent_live_child_pending_and_unknown_records_are_preserved(self):
parent = self.candidate('11111111', dead=False)
child = self.candidate('22222222'); br.candidate_child(parent, os.getpid())
value = json.loads((parent / '.owner.json').read_text())
child_owner = json.loads((child / '.owner.json').read_text()); child_owner['children'] = value['children']
br.atomic(child / '.owner.json', child_owner); os.utime(child, (time.time() - 3 * 86400,) * 2)
pending = self.candidate('33333333'); owner = json.loads((pending / '.owner.json').read_text())
owner['childPending'] = True; br.atomic(pending / '.owner.json', owner); os.utime(pending, (time.time() - 3 * 86400,) * 2)
unknown = self.candidate('44444444'); owner = json.loads((unknown / '.owner.json').read_text())
owner.pop('ownerCreated'); br.atomic(unknown / '.owner.json', owner); os.utime(unknown, (time.time() - 3 * 86400,) * 2)
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0)
self.assertGreaterEqual(result['incompleteBackupsDeferred'], 3)
for folder in (parent, child, pending, unknown): self.assertTrue(folder.exists())
def test_latest_pointer_valid_candidate_fresh_and_symlink_contents_are_protected(self):
pointed = self.candidate('11111111'); br.atomic(self.root / '.runtime/retention/latest.json', {'id': pointed.name})
valid = good_backup(self.root, '55555555')
fresh = self.candidate('22222222', age=10)
linked = self.candidate('33333333'); outside = self.root / 'business.xlsx'; outside.write_bytes(b'keep')
(linked / 'link').symlink_to(outside)
old = time.time() - 3 * 86400; os.utime(linked, (old, old))
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0)
for p in (pointed, valid, fresh, linked, outside): self.assertTrue(p.exists())
def test_monitor_cleanup_precedes_database_access(self):
orphan = self.candidate()
with patch.object(retention, 'Database', side_effect=RuntimeError('Database unavailable')):
with self.assertRaises(RuntimeError): operations.janitor(self.root, self.root / 'support', operations.DEFAULTS)
self.assertFalse(orphan.exists())
def test_low_disk_recovery_removes_orphan_before_deferring_new_backup(self):
current = good_backup(self.root, '11111111'); br.rotate(self.root, current); orphan = self.candidate()
class Reply:
def __enter__(self): return self
def __exit__(self, *_): return False
def read(self, *_): return b'{"status":"UP"}'
class Opener:
def open(self, *_args, **_kwargs): return Reply()
class Database:
def __init__(self, root): self.root = root; self.name = 'synthetic'; self.conn = self
def close(self): pass
def clean_owned_scratch(self, *_): return 0
def current(self): return 'current'
def busy(self): return False
def rows(self, *_): return [{'size': 1}]
with patch.object(retention.urllib.request, 'build_opener', return_value=Opener()), \
patch.object(retention, 'Database', Database), patch.object(backup_verifier, 'clean_abandoned', return_value=0), \
patch.object(retention.shutil, 'disk_usage', return_value=shutil._ntuple_diskusage(100, 99, 1)), \
patch.object(backup, 'create') as create:
with self.assertRaises(retention.Busy): retention.run(self.root)
self.assertFalse(orphan.exists()); self.assertFalse(create.called)
self.assertEqual(br.latest(self.root)[0], current)
def test_unreadable_process_identity_defers_cleanup(self):
import psutil
orphan = self.candidate()
with patch.object(psutil, 'Process', side_effect=psutil.AccessDenied()):
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0); self.assertTrue(orphan.exists())
if __name__ == '__main__': unittest.main()