修复:独立回收中断备份并限制 AI 日志增长

This commit is contained in:
2026-11-10 16:02:46 +08:00
parent dc5b9b76ec
commit cc8832c8e5
15 changed files with 584 additions and 39 deletions
+3 -2
View File
@@ -369,8 +369,7 @@ def create():
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
backup_retention.register_candidate(folder, ROOT)
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
@@ -378,7 +377,9 @@ def create():
process = None
try:
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
backup_retention.candidate_child(folder, pending=True)
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
backup_retention.candidate_child(folder, process.pid)
try:
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
shutil.copyfileobj(process.stdout, compressed)
+73 -9
View File
@@ -4,6 +4,7 @@ from datetime import datetime, timezone
from functools import wraps
import hashlib
import json
import math
import os
from pathlib import Path
import re
@@ -166,16 +167,9 @@ def _rotate(root, selected):
for folder in base.iterdir():
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
continue
# A restore-verified older folder is safe to retire under the shared lock.
# Incomplete candidates are kept for 24h and never treated as good backups.
# 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。
older = verified(folder, checksums=False)
try:
owner = json.loads((folder / '.owner.json').read_text())
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
except (OSError, ValueError):
owned = False
stale = owned and time.time() - folder.stat().st_mtime > 86400
if older is not None or stale:
if older is not None:
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents):
continue
@@ -204,6 +198,7 @@ def _rotate(root, selected):
retire_folder(root,legacy,'legacy')
legacy_info['retired']=True
atomic(registration,legacy_info)
clean_incomplete(root)
clean_retired(root)
return pointer, removed
@@ -282,6 +277,75 @@ def backup_ids(root):
return {p.name for p in base.iterdir()} if base.exists() else set()
def register_candidate(folder, root):
import psutil
folder = Path(folder); info = folder.stat()
atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2,
'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(),
'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(),
'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False})
def candidate_child(folder, pid=None, pending=False):
import psutil
path = Path(folder) / '.owner.json'; value = json.loads(path.read_text())
if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed')
value['childPending'] = pending
if pid is not None:
value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()})
atomic(path, value)
def clean_incomplete(root, now=None):
"""独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。"""
import psutil
root = Path(root).resolve(); now = time.time() if now is None else now
result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0}
base = root / 'output/backups'
if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root')
def exited(item, pid_key, created_key):
if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or
not isinstance(item.get(created_key), (float, int)) or
not math.isfinite(item[created_key]) or item[created_key] <= 0): return False
try:
process = psutil.Process(item[pid_key])
return (abs(process.create_time() - item[created_key]) >= .02 or
not process.is_running() or process.status() == psutil.STATUS_ZOMBIE)
except psutil.NoSuchProcess: return True
except psutil.Error: return False
with lock(root, timeout=0):
pointer = root / '.runtime/retention/latest.json'
try:
if pointer.is_symlink(): return result
protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None
except (OSError, ValueError, KeyError, TypeError): return result
for folder in base.iterdir() if base.exists() else []:
if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue
owner = folder / '.owner.json'
if owner.is_symlink(): continue
try:
value = json.loads(owner.read_text()); info = folder.stat()
if not isinstance(value, dict): continue
if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or
value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or
not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue
if verified(folder, checksums=False) is not None: continue
if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or
value.get('childPending') is not False or not isinstance(value.get('children'), list) or
not exited(value, 'ownerPid', 'ownerCreated') or
not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])):
result['incompleteBackupsDeferred'] += 1; continue
contents = list(folder.rglob('*'))
if any(p.is_symlink() for p in contents): continue
size = sum(p.stat().st_size for p in contents if p.is_file())
if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue
retire_folder(root, folder, 'backup')
result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size
except (OSError, ValueError, TypeError): continue
clean_retired(root)
return result
def discard_failed_new(root, before):
"""Delete only this call's registered, incomplete candidate after it has stopped."""
root=Path(root).resolve();base=root/'output/backups'
+11 -1
View File
@@ -16,7 +16,17 @@ MAX_AGE = 7 * 86400
def candidates(root, support):
root, support = Path(root).resolve(), Path(support).resolve()
paths = set()
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
native_logs = []
native = root / '.runtime/nao-users'
if native.is_dir() and not native.is_symlink():
for actor in native.iterdir():
if actor.is_symlink() or not actor.is_dir(): continue
native_logs.append((actor / 'logs', 1))
sessions = actor / 'sessions'
if sessions.is_dir() and not sessions.is_symlink():
native_logs.extend((session / 'logs', 1) for session in sessions.iterdir()
if session.is_dir() and not session.is_symlink())
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1), *native_logs):
if not base.is_dir() or base.is_symlink():
continue
for directory, dirs, files in os.walk(base, followlinks=False):
+135
View File
@@ -0,0 +1,135 @@
"""为 NAO 原生子进程捕获输出,写入有大小和时间限制的私有日志。"""
import argparse
import codecs
from contextlib import contextmanager
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import signal
import stat
import subprocess
import sys
import time
SEGMENT_BYTES = 10 * 1024**2
TOTAL_BYTES = 90 * 1024**2
MAX_AGE = 7 * 86400
BACKUPS = 8
def safe_file(path):
if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Unsafe log path')
if path.exists():
info = path.stat()
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe log file')
@contextmanager
def writer_lock(path):
safe_file(path)
fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600)
try:
info = os.fstat(fd)
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe writer lock')
if os.name == 'nt':
import msvcrt
if not info.st_size: os.write(fd, b'0')
os.lseek(fd, 0, 0); msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield
finally: os.close(fd)
def prune(path, total_bytes=TOTAL_BYTES, now=None):
now = time.time() if now is None else now
archives = [path.with_name(path.name + '.' + str(i)) for i in range(1, BACKUPS + 1)]
for archive in archives: safe_file(archive)
existing = sorted((p for p in archives if p.exists()), key=lambda p: p.stat().st_mtime)
total = (path.stat().st_size if path.exists() else 0) + sum(p.stat().st_size for p in existing)
for archive in existing:
if now - archive.stat().st_mtime > MAX_AGE or total > total_bytes:
total -= archive.stat().st_size; archive.unlink()
class PrivateHandler(RotatingFileHandler):
def _open(self):
path = Path(self.baseFilename); safe_file(path)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, 'O_NOFOLLOW', 0), 0o600)
if hasattr(os, 'fchmod'): os.fchmod(fd, 0o600)
else: os.chmod(path, 0o600)
return os.fdopen(fd, 'a', encoding='utf-8', errors='replace')
def handleError(self, record):
# 交给捕获层限频重试;不能因诊断日志失败终止业务子进程。
raise RuntimeError('Native log write failed')
def capture(path, command, segment_bytes=SEGMENT_BYTES, total_bytes=TOTAL_BYTES):
path = Path(path).absolute()
if path.name not in ('server.log', 'fastapi.log') or path.parent.name != 'logs': raise ValueError('Unexpected log name')
safe_file(path); path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with writer_lock(path.with_name('writer-' + path.stem + '.lock')):
prune(path, total_bytes)
handler = PrivateHandler(path, maxBytes=segment_bytes, backupCount=BACKUPS, encoding='utf-8')
handler.setFormatter(logging.Formatter('%(message)s')); handler.terminator = ''
if path.stat().st_size >= segment_bytes or time.time() - path.stat().st_mtime > MAX_AGE:
handler.doRollover(); prune(path, total_bytes)
logger = logging.Logger('native-output', level=logging.INFO); logger.addHandler(handler)
child = None; stopping = False
def stop(signum, frame):
nonlocal stopping
stopping = True
if child is not None and child.poll() is None:
try: child.terminate()
except OSError: pass
previous = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
try:
if stopping: return 0
child = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if stopping: child.terminate()
decoder = codecs.getincrementaldecoder('utf-8')(errors='replace')
retry_at = 0
def write(text):
nonlocal retry_at
if time.monotonic() < retry_at: return
try:
logger.info(text); prune(path, total_bytes)
except (OSError, RuntimeError, ValueError):
# 继续排空管道,防止日志空间不足拖住正在进行的 AI 请求。
retry_at = time.monotonic() + 60
while chunk := child.stdout.read1(min(64 * 1024, max(1, segment_bytes // 8))):
write(decoder.decode(chunk))
tail = decoder.decode(b'', final=True)
if tail: write(tail)
code = child.wait()
return code if code >= 0 else 128 - code
finally:
for signum, old in previous.items(): signal.signal(signum, old)
try: handler.close()
except OSError: pass
if child is not None:
if child.poll() is None:
child.terminate()
try: child.wait(timeout=10)
except subprocess.TimeoutExpired: child.kill(); child.wait()
child.stdout.close()
def main():
os.umask(0o077)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--log', type=Path, required=True)
parser.add_argument('command', nargs=argparse.REMAINDER)
args = parser.parse_args(); command = args.command
if command[:1] == ['--']: command = command[1:]
if not command: parser.error('A child command is required')
return capture(args.log, command)
if __name__ == '__main__':
try: raise SystemExit(main())
except Exception:
print('Native log capture failed', file=sys.stderr); raise SystemExit(1)
+22 -15
View File
@@ -991,7 +991,7 @@ def _model_settings(*, include_options=False):
def _config_fingerprint(model_base: str, model_key: str, model_name: str) -> str:
paths = [TEMPLATES / "RULES.md", TEMPLATES / "agent" / "prompts" / "system.md",
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py",
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py", ROOT / "scripts" / "nao_log_stream.py",
ROOT / "scripts" / "nao_quality_tools.py", ROOT / "scripts" / "nao_quality_mcp.py",
ROOT / "scripts" / "nao_model_settings.py", ROOT / "scripts" / "nao_model_gate.py", ROOT / "scripts" / "nao_evidence_finalizer.py", ROOT / "scripts" / "nao_evidence_report.py", ROOT / "scripts" / "nao_weight_ranking.py",
ROOT / "scripts" / "nao_metric_request.py", ROOT / "scripts" / "nao_analysis_coverage.py", ROOT / "scripts" / "nao_combined_ranking.py",
@@ -1209,6 +1209,11 @@ def _process_matches(record: dict[str, Any], actor_root: Path) -> bool:
if not start or start != record.get("lstart"):
return False
kind = record.get("kind")
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
if wrapper in command:
name = "fastapi.log" if kind == "fastapi" else "server.log"
if "--log " + str(actor_root / "logs" / name) not in command:
return False
if kind == "fastapi":
return str(PYTHON) in command and str(FASTAPI_MAIN) in command
if kind == "server":
@@ -1668,21 +1673,23 @@ class RuntimeManager:
path.touch(mode=0o600, exist_ok=True)
path.chmod(0o600)
processes: list[dict[str, Any]] = []
log_path = logs / "server.log"
fd = _open_server_log(log_path, actor_root)
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
for log_path in (logs / "fastapi.log", logs / "server.log"):
os.close(_open_server_log(log_path, actor_root))
try:
with os.fdopen(fd, "ab") as log:
fastapi = subprocess.Popen(
[str(PYTHON), str(FASTAPI_MAIN)], cwd=project, env=env,
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True,
)
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
server = subprocess.Popen(
[str(NAO_SERVER), "serve", "--port", str(server_port), "--host", "127.0.0.1"],
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL,
stdout=log, stderr=log, start_new_session=True,
)
processes.append(_process_record(server.pid, "server", server_port))
fastapi = subprocess.Popen(
[str(PYTHON), wrapper, "--log", str(logs / "fastapi.log"), "--", str(PYTHON), str(FASTAPI_MAIN)],
cwd=project, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True,
)
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
server = subprocess.Popen(
[str(PYTHON), wrapper, "--log", str(logs / "server.log"), "--", str(NAO_SERVER),
"serve", "--port", str(server_port), "--host", "127.0.0.1"],
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True,
)
processes.append(_process_record(server.pid, "server", server_port))
except Exception:
for record in processes:
if _process_matches(record, actor_root):
+93 -3
View File
@@ -3,16 +3,54 @@ import gzip
import hashlib
import io
import json
import os
from pathlib import Path
import tarfile
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import backup
class BackupRemediationTest(unittest.TestCase):
@contextlib.contextmanager
def private_verifier_fixture(self, root):
"""Exercise verify's Darwin/container path without starting a MySQL server."""
import backup_verifier
config = root / '.runtime/retention/verify-fixture/client.cnf'
config.parent.mkdir(parents=True, exist_ok=True)
config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n')
registration = root / '.runtime/retention/scratch/fixture.json'
registered = []
child_pids = []
@contextlib.contextmanager
def fake_instance(instance_root):
self.assertEqual(Path(instance_root), root)
yield config
def fake_scratch_register(instance_root, database, server):
self.assertEqual(Path(instance_root), root)
self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$')
self.assertEqual(server, 'synthetic-server-uuid')
registration.parent.mkdir(parents=True, exist_ok=True)
registration.write_text('{}')
registered.append((database, server))
return registration
with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \
patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \
patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \
patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \
patch.object(backup.backup_retention, 'scratch_child',
side_effect=lambda path, pid: child_pids.append((Path(path), pid))):
yield {'config': config, 'instance': instance, 'registration': registration,
'registered': registered, 'childPids': child_pids,
'scratchCreated': scratch_created}
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -23,18 +61,46 @@ class BackupRemediationTest(unittest.TestCase):
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
# candidate_child() records process creation time via psutil.
# Reuse this live test process as a synthetic child; no command runs.
self.pid = os.getpid()
def wait(self): return 0
def poll(self): return 0
def kill(self): pass
seen_client = []
def fake_client(statement, database=None):
seen_client.append((statement, database))
self.assertEqual(database, 'quality_fixture')
if statement == 'SHOW TABLES;':
return 'binary_fixture\n'
if statement == 'SELECT COUNT(*) FROM `binary_fixture`;':
return '1\n'
self.fail('Unexpected isolated export SQL: ' + statement)
with patch.object(backup, 'ROOT', root), \
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
patch.object(backup, 'verify_references', return_value={'verified': True}), \
patch.object(backup, 'query_evidence', return_value={}), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', BinaryDump):
folder = backup.create()
self.assertEqual(seen_client, [
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
])
owner = json.loads((folder / '.owner.json').read_text())
self.assertEqual(owner['ownerPid'], os.getpid())
self.assertIsInstance(owner['ownerCreated'], (int, float))
self.assertFalse(owner['childPending'])
self.assertEqual(len(owner['children']), 1)
self.assertEqual(owner['children'][0]['pid'], os.getpid())
self.assertIsInstance(owner['children'][0]['created'], (int, float))
manifest = json.loads((folder / 'manifest.json').read_text())
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
@@ -189,6 +255,9 @@ class BackupRemediationTest(unittest.TestCase):
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('CREATE DATABASE'):
state['database'] = statement.split('`')[1]
return ''
@@ -221,6 +290,7 @@ class BackupRemediationTest(unittest.TestCase):
class FakeMysqlRestore:
def __init__(self, *args, **kwargs):
state['restoreCommand'] = list(args[0])
self.pid = 424242
self.stdin = RecordingInput()
self.returncode = 0
def wait(self):
@@ -230,7 +300,8 @@ class BackupRemediationTest(unittest.TestCase):
def kill(self):
self.returncode = -9
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
result = backup.verify(folder)
@@ -239,6 +310,12 @@ class BackupRemediationTest(unittest.TestCase):
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
self.assertTrue(state['updated'])
self.assertTrue(state['dropped'])
self.assertTrue(private['instance'].called)
self.assertTrue(private['registered'])
self.assertEqual(private['childPids'], [(private['registration'], 424242)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
self.assertEqual(state['database'].split('_')[0], 'quality')
self.assertFalse(result['migrationReady'])
self.assertTrue(result['archiveReferencesRebound'])
@@ -502,17 +579,23 @@ class BackupRemediationTest(unittest.TestCase):
'tableCounts': {}, 'queryEvidence': None,
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
self.assertRaises(backup.BackupFailure) as checksum:
backup.verify(folder)
self.assertEqual((checksum.exception.stage, checksum.exception.code),
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
private['instance'].assert_called_once_with(root)
self.assertEqual(private['registered'], [])
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
state = {'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement.startswith('CREATE DATABASE'):
return ''
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('DROP DATABASE'):
state['dropped'] = True
return ''
@@ -522,6 +605,7 @@ class BackupRemediationTest(unittest.TestCase):
class FailedMysqlRestore:
def __init__(self, *args, **kwargs):
self.pid = 434343
self.stdin = io.BytesIO()
self.returncode = 1
def wait(self):
@@ -531,7 +615,8 @@ class BackupRemediationTest(unittest.TestCase):
def kill(self):
self.returncode = -9
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
self.assertRaises(backup.BackupFailure) as restore:
@@ -539,6 +624,11 @@ class BackupRemediationTest(unittest.TestCase):
self.assertEqual((restore.exception.stage, restore.exception.code),
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
private['instance'].assert_called_once_with(root)
self.assertEqual(private['childPids'], [(private['registration'], 434343)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
old_root = (root / 'old/runtimedata/archive').resolve()
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
+88
View File
@@ -0,0 +1,88 @@
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
import nao_log_stream as stream
import log_retention
class NativeLogTests(unittest.TestCase):
def test_log_write_failure_does_not_interrupt_child_work(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); result = root / 'completed.txt'
command = [sys.executable, '-c', 'import sys,pathlib;sys.stdout.write("x"*300000);pathlib.Path(sys.argv[1]).write_text("done")', str(result)]
with patch.object(stream.PrivateHandler, 'emit', side_effect=OSError('Disk full')):
self.assertEqual(stream.capture(root / 'logs/server.log', command), 0)
self.assertEqual(result.read_text(), 'done')
def test_real_child_output_rolls_and_stays_bounded(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory).resolve() / 'logs/server.log'
command = [sys.executable, '-c', 'import sys;sys.stdout.write("old line\\n"*6000+"LAST_MARKER\\n");sys.stderr.write("stderr marker\\n")']
self.assertEqual(stream.capture(path, command, segment_bytes=1024, total_bytes=9000), 0)
files = list(path.parent.glob('server.log*'))
self.assertGreater(len(files), 1); self.assertLessEqual(sum(p.stat().st_size for p in files), 9000)
contents = ''.join(p.read_text() for p in files)
self.assertIn('LAST_MARKER', contents); self.assertIn('stderr marker', contents)
for p in files: self.assertEqual(p.stat().st_mode & 0o077, 0)
def test_oversized_legacy_log_and_expired_archives_are_bounded_on_start(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory).resolve() / 'logs/server.log'; path.parent.mkdir()
path.write_bytes(b'old' * 10000); old = path.with_name('server.log.8'); old.write_bytes(b'expired')
os.utime(old, (time.time() - 9 * 86400,) * 2)
stream.capture(path, [sys.executable, '-c', 'print("new")'], segment_bytes=1024, total_bytes=9000)
self.assertIn('new', path.read_text()); self.assertFalse(old.exists())
self.assertLessEqual(sum(p.stat().st_size for p in path.parent.glob('server.log*')), 9000)
def test_symlink_log_or_archive_is_rejected_before_child_launch(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); path = root / 'logs/server.log'; path.parent.mkdir()
outside = root / 'data.xlsx'; outside.write_bytes(b'business data')
path.with_name('server.log.1').symlink_to(outside)
with self.assertRaises(ValueError): stream.capture(path, [sys.executable, '-c', 'print("no")'])
self.assertEqual(outside.read_bytes(), b'business data')
@unittest.skipIf(os.name == 'nt', 'POSIX process-group shutdown contract')
def test_stop_keeps_wrapper_until_term_ignoring_child_can_be_killed_as_a_group(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); path = root / 'logs/fastapi.log'
command = [sys.executable, str(Path(stream.__file__).resolve()), '--log', str(path), '--', sys.executable, '-u', '-c',
'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);print("READY",flush=True);time.sleep(60)']
process = subprocess.Popen(command, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
deadline = time.monotonic() + 5
while not path.exists() or 'READY' not in path.read_text():
if time.monotonic() > deadline: self.fail('Native wrapper did not start')
time.sleep(.02)
import nao_quality_runtime as runtime
with patch.object(runtime, 'PYTHON', Path(sys.executable)), patch.object(runtime, 'FASTAPI_MAIN', Path('-c')):
record=runtime._process_record(process.pid,'fastapi',5283)
self.assertTrue(runtime._process_matches(record,root))
self.assertFalse(runtime._process_matches(record,root/'another-session'))
os.killpg(process.pid, signal.SIGTERM); time.sleep(.1)
self.assertIsNone(process.poll())
os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
finally:
if process.poll() is None: os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
def test_native_actor_and_session_logs_are_cleaned_but_business_files_and_open_logs_are_kept(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve(); support = root / 'support'
actor = root / '.runtime/nao-users/actor'
paths = [actor / 'logs/server.log', actor / 'sessions/session/logs/fastapi.log.1']
for path in paths:
path.parent.mkdir(parents=True); path.write_bytes(b'old diagnostics\n'); os.utime(path, (time.time() - 9 * 86400,) * 2)
private = actor / 'sessions/session/private/nao.sqlite'; private.parent.mkdir(); private.write_bytes(b'keep')
self.assertTrue(set(paths).issubset(log_retention.candidates(root, support)))
result = log_retention.retain(root, support, open_paths={str(paths[0])})
self.assertEqual(result['logsRemoved'], 1); self.assertTrue(paths[0].exists()); self.assertTrue(private.exists())
if __name__ == '__main__': unittest.main()
+106
View File
@@ -0,0 +1,106 @@
import json
import os
from pathlib import Path
import shutil
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
import backup
import backup_retention as br
import backup_verifier
import operations
import retention
from test_retention import good_backup
class CandidateRecoveryTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(); self.root = Path(self.temporary.name).resolve()
def tearDown(self): self.temporary.cleanup()
def candidate(self, suffix='22222222', dead=True, age=3 * 86400):
folder = self.root / 'output/backups' / ('20261108T010000Z-' + suffix)
folder.mkdir(parents=True); (folder / 'database.sql.gz').write_bytes(b'partial dump')
br.register_candidate(folder, self.root)
owner = json.loads((folder / '.owner.json').read_text()); owner['createdAt'] -= age
if dead: owner.update({'ownerPid': 2000000000, 'ownerCreated': 1.0})
br.atomic(folder / '.owner.json', owner); os.utime(folder, (time.time() - age,) * 2)
return folder
def test_expired_dead_candidate_is_cleaned_without_a_new_backup(self):
current = good_backup(self.root, '11111111'); br.rotate(self.root, current)
orphan = self.candidate()
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 1)
self.assertGreater(result['incompleteBackupBytesRemoved'], 0)
self.assertFalse(orphan.exists()); self.assertEqual(br.latest(self.root)[0], current)
def test_live_parent_live_child_pending_and_unknown_records_are_preserved(self):
parent = self.candidate('11111111', dead=False)
child = self.candidate('22222222'); br.candidate_child(parent, os.getpid())
value = json.loads((parent / '.owner.json').read_text())
child_owner = json.loads((child / '.owner.json').read_text()); child_owner['children'] = value['children']
br.atomic(child / '.owner.json', child_owner); os.utime(child, (time.time() - 3 * 86400,) * 2)
pending = self.candidate('33333333'); owner = json.loads((pending / '.owner.json').read_text())
owner['childPending'] = True; br.atomic(pending / '.owner.json', owner); os.utime(pending, (time.time() - 3 * 86400,) * 2)
unknown = self.candidate('44444444'); owner = json.loads((unknown / '.owner.json').read_text())
owner.pop('ownerCreated'); br.atomic(unknown / '.owner.json', owner); os.utime(unknown, (time.time() - 3 * 86400,) * 2)
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0)
self.assertGreaterEqual(result['incompleteBackupsDeferred'], 3)
for folder in (parent, child, pending, unknown): self.assertTrue(folder.exists())
def test_latest_pointer_valid_candidate_fresh_and_symlink_contents_are_protected(self):
pointed = self.candidate('11111111'); br.atomic(self.root / '.runtime/retention/latest.json', {'id': pointed.name})
valid = good_backup(self.root, '55555555')
fresh = self.candidate('22222222', age=10)
linked = self.candidate('33333333'); outside = self.root / 'business.xlsx'; outside.write_bytes(b'keep')
(linked / 'link').symlink_to(outside)
old = time.time() - 3 * 86400; os.utime(linked, (old, old))
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0)
for p in (pointed, valid, fresh, linked, outside): self.assertTrue(p.exists())
def test_monitor_cleanup_precedes_database_access(self):
orphan = self.candidate()
with patch.object(retention, 'Database', side_effect=RuntimeError('Database unavailable')):
with self.assertRaises(RuntimeError): operations.janitor(self.root, self.root / 'support', operations.DEFAULTS)
self.assertFalse(orphan.exists())
def test_low_disk_recovery_removes_orphan_before_deferring_new_backup(self):
current = good_backup(self.root, '11111111'); br.rotate(self.root, current); orphan = self.candidate()
class Reply:
def __enter__(self): return self
def __exit__(self, *_): return False
def read(self, *_): return b'{"status":"UP"}'
class Opener:
def open(self, *_args, **_kwargs): return Reply()
class Database:
def __init__(self, root): self.root = root; self.name = 'synthetic'; self.conn = self
def close(self): pass
def clean_owned_scratch(self, *_): return 0
def current(self): return 'current'
def busy(self): return False
def rows(self, *_): return [{'size': 1}]
with patch.object(retention.urllib.request, 'build_opener', return_value=Opener()), \
patch.object(retention, 'Database', Database), patch.object(backup_verifier, 'clean_abandoned', return_value=0), \
patch.object(retention.shutil, 'disk_usage', return_value=shutil._ntuple_diskusage(100, 99, 1)), \
patch.object(backup, 'create') as create:
with self.assertRaises(retention.Busy): retention.run(self.root)
self.assertFalse(orphan.exists()); self.assertFalse(create.called)
self.assertEqual(br.latest(self.root)[0], current)
def test_unreadable_process_identity_defers_cleanup(self):
import psutil
orphan = self.candidate()
with patch.object(psutil, 'Process', side_effect=psutil.AccessDenied()):
result = br.clean_incomplete(self.root)
self.assertEqual(result['incompleteBackupsRemoved'], 0); self.assertTrue(orphan.exists())
if __name__ == '__main__': unittest.main()