"""Conservative retention of application logs, with native MySQL reopening.""" import gzip import json import os from pathlib import Path import re import time import uuid import psutil MAX_BYTES = 100 * 1024**2 MAX_AGE = 7 * 86400 def candidates(root, support): root, support = Path(root).resolve(), Path(support).resolve() paths = set() for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)): if not base.is_dir() or base.is_symlink(): continue for directory, dirs, files in os.walk(base, followlinks=False): parent = Path(directory) if len(parent.relative_to(base).parts) >= depth - 1: dirs[:] = [] else: dirs[:] = [d for d in dirs if not (parent / d).is_symlink() and d not in ('backups', 'mysql-data', 'mysql', 'jdk', 'm2', 'backend-builds', 'node_modules')] for name in files: if re.fullmatch(r'.+\.(?:log|out|err)(?:\.[a-zA-Z0-9_-]+)*(?:\.gz)?', name): p = parent / name if not p.is_symlink() and p.is_file() and p.stat().st_nlink == 1: paths.add(p) return sorted(paths) def opened_files(): """Unknown access is a reason to defer deletion, never a reason to assume closed.""" paths = set() for process in psutil.process_iter(['uids']): try: if process.info['uids'].real != os.getuid(): continue for f in process.open_files(): paths.add(str(Path(f.path).absolute())) except psutil.NoSuchProcess: continue except psutil.Error as error: # macOS restricts system/GUI agents unrelated to these application log # directories. A denied project runtime/interpreter still defers cleanup. try: executable = Path(process.exe()).resolve() interpreter = re.fullmatch(r'(?:python.*|Python|java|node|bash|zsh|sh|uv|mvn)', executable.name) if executable.is_relative_to(Path.cwd().resolve()) or interpreter: raise RuntimeError('Project writer identity check unavailable') from error except psutil.NoSuchProcess: continue except psutil.Error: raise RuntimeError('Writer identity check unavailable') from error return paths def fingerprint(path): s = path.stat() return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns def category(path): name = path.name.lstrip('.') name = re.sub(r'\.\d{4}-\d{2}-\d{2}.*', '', name) name = re.sub(r'\.roll-.*', '', name) name = re.sub(r'\.(?:log|out|err)(?:\..*)?$', '', name) return name def rotate_mysql(root, execute, br, now=None): root = Path(root).resolve(); now = time.time() if now is None else now source = root / 'output/logs/mysql-server.log' journal = root / '.runtime/operations/mysql-log-rotation.json' if any(p.is_symlink() for p in (root / 'output', source.parent, source, root / '.runtime', journal.parent, journal)): raise ValueError('Unsafe MySQL log path') pending = None if journal.exists(): pending = json.loads(journal.read_text()) if (pending.get('root') != str(root) or pending.get('kind') != 'quality-mysql-log-rotation' or not re.fullmatch(r'mysql-server\.roll-\d+-[a-f0-9]{8}\.log', pending.get('archive', ''))): raise ValueError('Unknown MySQL log rotation intent') elif source.is_file() and source.stat().st_size: old_day = time.localtime(source.stat().st_mtime)[:3] if source.stat().st_size < 10 * 1024**2 and old_day == time.localtime(now)[:3]: return 0 pending = {'kind': 'quality-mysql-log-rotation', 'root': str(root), 'archive': 'mysql-server.roll-' + str(int(now)) + '-' + uuid.uuid4().hex[:8] + '.log', 'fingerprint': list(fingerprint(source))} br.atomic(journal, pending) if pending is None: return 0 target = source.with_name(pending['archive']) if target.is_symlink(): raise ValueError('Unsafe MySQL rotated log') if not target.exists(): if not source.is_file() or list(fingerprint(source)) != pending['fingerprint']: journal.unlink(); return 0 source.rename(target) # MySQL closes the old descriptor and reopens its configured path. No truncation. execute('FLUSH ERROR LOGS;', None) if not source.is_file() or source.is_symlink() or source.stat().st_ino == target.stat().st_ino: raise RuntimeError('MySQL error log reopening not confirmed') source.chmod(0o600) target.chmod(0o600) journal.unlink() return 1 def retain(root, support, now=None, open_paths=None, max_bytes=MAX_BYTES, max_age=MAX_AGE): now = time.time() if now is None else now opened = opened_files() if open_paths is None else set(open_paths) groups = {}; result = {'logsRemoved': 0, 'logBytesRemoved': 0, 'logsDeferred': 0} for p in candidates(root, support): groups.setdefault(category(p), []).append(p) for files in groups.values(): files.sort(key=lambda p: p.stat().st_mtime) total = sum(p.stat().st_size for p in files) for p in files: s = p.stat(); stamp = fingerprint(p) if now - s.st_mtime <= max_age and total <= max_bytes: continue # Native appenders own these live files. The monitor handles archives; # MySQL is reopened by its dedicated rotation function above. if ((p.parent == Path(root) / 'output/logs' and p.name in ('backend.log', 'mysql-server.log', 'backend-console.log')) or (p.parent == Path(support) / 'logs' and p.name in ('guardian.log', 'retention.log', 'operations.log'))): result['logsDeferred'] += 1; continue if str(p.absolute()) in opened: result['logsDeferred'] += 1; continue # Recheck descriptors immediately before mutation. Data/config/archives # never enter this allowlist, and active writers are never truncated. if p.is_symlink() or fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened): result['logsDeferred'] += 1; continue if s.st_size > max_bytes and now - s.st_mtime <= max_age and len(files) == 1: # Keep the newest bounded tail of a closed oversized console log. temp = p.with_name(p.name + '.' + uuid.uuid4().hex + '.tmp') try: with p.open('rb') as source, temp.open('xb') as output: temp.chmod(0o600); source.seek(max(0, s.st_size - max_bytes)) source.readline() # Drop a possible partial leading line. while block := source.read(1024 * 1024): output.write(block) output.flush(); os.fsync(output.fileno()) if fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened): result['logsDeferred'] += 1; continue os.utime(temp, ns=(s.st_atime_ns, s.st_mtime_ns)); temp.replace(p) removed = s.st_size - p.stat().st_size total -= removed; result['logBytesRemoved'] += removed finally: temp.unlink(missing_ok=True) else: p.unlink(); total -= s.st_size result['logsRemoved'] += 1; result['logBytesRemoved'] += s.st_size return result