修复:独立回收中断备份并限制 AI 日志增长
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
"""为 NAO 原生子进程捕获输出,写入有大小和时间限制的私有日志。"""
|
||||
import argparse
|
||||
import codecs
|
||||
from contextlib import contextmanager
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
SEGMENT_BYTES = 10 * 1024**2
|
||||
TOTAL_BYTES = 90 * 1024**2
|
||||
MAX_AGE = 7 * 86400
|
||||
BACKUPS = 8
|
||||
|
||||
|
||||
def safe_file(path):
|
||||
if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Unsafe log path')
|
||||
if path.exists():
|
||||
info = path.stat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe log file')
|
||||
|
||||
|
||||
@contextmanager
|
||||
def writer_lock(path):
|
||||
safe_file(path)
|
||||
fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe writer lock')
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
if not info.st_size: os.write(fd, b'0')
|
||||
os.lseek(fd, 0, 0); msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
yield
|
||||
finally: os.close(fd)
|
||||
|
||||
|
||||
def prune(path, total_bytes=TOTAL_BYTES, now=None):
|
||||
now = time.time() if now is None else now
|
||||
archives = [path.with_name(path.name + '.' + str(i)) for i in range(1, BACKUPS + 1)]
|
||||
for archive in archives: safe_file(archive)
|
||||
existing = sorted((p for p in archives if p.exists()), key=lambda p: p.stat().st_mtime)
|
||||
total = (path.stat().st_size if path.exists() else 0) + sum(p.stat().st_size for p in existing)
|
||||
for archive in existing:
|
||||
if now - archive.stat().st_mtime > MAX_AGE or total > total_bytes:
|
||||
total -= archive.stat().st_size; archive.unlink()
|
||||
|
||||
|
||||
class PrivateHandler(RotatingFileHandler):
|
||||
def _open(self):
|
||||
path = Path(self.baseFilename); safe_file(path)
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, 'O_NOFOLLOW', 0), 0o600)
|
||||
if hasattr(os, 'fchmod'): os.fchmod(fd, 0o600)
|
||||
else: os.chmod(path, 0o600)
|
||||
return os.fdopen(fd, 'a', encoding='utf-8', errors='replace')
|
||||
|
||||
def handleError(self, record):
|
||||
# 交给捕获层限频重试;不能因诊断日志失败终止业务子进程。
|
||||
raise RuntimeError('Native log write failed')
|
||||
|
||||
|
||||
def capture(path, command, segment_bytes=SEGMENT_BYTES, total_bytes=TOTAL_BYTES):
|
||||
path = Path(path).absolute()
|
||||
if path.name not in ('server.log', 'fastapi.log') or path.parent.name != 'logs': raise ValueError('Unexpected log name')
|
||||
safe_file(path); path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with writer_lock(path.with_name('writer-' + path.stem + '.lock')):
|
||||
prune(path, total_bytes)
|
||||
handler = PrivateHandler(path, maxBytes=segment_bytes, backupCount=BACKUPS, encoding='utf-8')
|
||||
handler.setFormatter(logging.Formatter('%(message)s')); handler.terminator = ''
|
||||
if path.stat().st_size >= segment_bytes or time.time() - path.stat().st_mtime > MAX_AGE:
|
||||
handler.doRollover(); prune(path, total_bytes)
|
||||
logger = logging.Logger('native-output', level=logging.INFO); logger.addHandler(handler)
|
||||
child = None; stopping = False
|
||||
def stop(signum, frame):
|
||||
nonlocal stopping
|
||||
stopping = True
|
||||
if child is not None and child.poll() is None:
|
||||
try: child.terminate()
|
||||
except OSError: pass
|
||||
previous = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
try:
|
||||
if stopping: return 0
|
||||
child = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
if stopping: child.terminate()
|
||||
decoder = codecs.getincrementaldecoder('utf-8')(errors='replace')
|
||||
retry_at = 0
|
||||
def write(text):
|
||||
nonlocal retry_at
|
||||
if time.monotonic() < retry_at: return
|
||||
try:
|
||||
logger.info(text); prune(path, total_bytes)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
# 继续排空管道,防止日志空间不足拖住正在进行的 AI 请求。
|
||||
retry_at = time.monotonic() + 60
|
||||
while chunk := child.stdout.read1(min(64 * 1024, max(1, segment_bytes // 8))):
|
||||
write(decoder.decode(chunk))
|
||||
tail = decoder.decode(b'', final=True)
|
||||
if tail: write(tail)
|
||||
code = child.wait()
|
||||
return code if code >= 0 else 128 - code
|
||||
finally:
|
||||
for signum, old in previous.items(): signal.signal(signum, old)
|
||||
try: handler.close()
|
||||
except OSError: pass
|
||||
if child is not None:
|
||||
if child.poll() is None:
|
||||
child.terminate()
|
||||
try: child.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired: child.kill(); child.wait()
|
||||
child.stdout.close()
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--log', type=Path, required=True)
|
||||
parser.add_argument('command', nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args(); command = args.command
|
||||
if command[:1] == ['--']: command = command[1:]
|
||||
if not command: parser.error('A child command is required')
|
||||
return capture(args.log, command)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: raise SystemExit(main())
|
||||
except Exception:
|
||||
print('Native log capture failed', file=sys.stderr); raise SystemExit(1)
|
||||
Reference in New Issue
Block a user