新增:Linux Docker 部署与自动回收、备份及容量监测

This commit is contained in:
2026-11-10 14:53:07 +08:00
parent 3962521a5c
commit dc5b9b76ec
45 changed files with 5591 additions and 35 deletions
+158
View File
@@ -0,0 +1,158 @@
"""Conservative retention of application logs, with native MySQL reopening."""
import gzip
import json
import os
from pathlib import Path
import re
import time
import uuid
import psutil
MAX_BYTES = 100 * 1024**2
MAX_AGE = 7 * 86400
def candidates(root, support):
root, support = Path(root).resolve(), Path(support).resolve()
paths = set()
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
if not base.is_dir() or base.is_symlink():
continue
for directory, dirs, files in os.walk(base, followlinks=False):
parent = Path(directory)
if len(parent.relative_to(base).parts) >= depth - 1:
dirs[:] = []
else:
dirs[:] = [d for d in dirs if not (parent / d).is_symlink() and d not in
('backups', 'mysql-data', 'mysql', 'jdk', 'm2', 'backend-builds', 'node_modules')]
for name in files:
if re.fullmatch(r'.+\.(?:log|out|err)(?:\.[a-zA-Z0-9_-]+)*(?:\.gz)?', name):
p = parent / name
if not p.is_symlink() and p.is_file() and p.stat().st_nlink == 1:
paths.add(p)
return sorted(paths)
def opened_files():
"""Unknown access is a reason to defer deletion, never a reason to assume closed."""
paths = set()
for process in psutil.process_iter(['uids']):
try:
if process.info['uids'].real != os.getuid():
continue
for f in process.open_files():
paths.add(str(Path(f.path).absolute()))
except psutil.NoSuchProcess:
continue
except psutil.Error as error:
# macOS restricts system/GUI agents unrelated to these application log
# directories. A denied project runtime/interpreter still defers cleanup.
try:
executable = Path(process.exe()).resolve()
interpreter = re.fullmatch(r'(?:python.*|Python|java|node|bash|zsh|sh|uv|mvn)', executable.name)
if executable.is_relative_to(Path.cwd().resolve()) or interpreter:
raise RuntimeError('Project writer identity check unavailable') from error
except psutil.NoSuchProcess:
continue
except psutil.Error:
raise RuntimeError('Writer identity check unavailable') from error
return paths
def fingerprint(path):
s = path.stat()
return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns
def category(path):
name = path.name.lstrip('.')
name = re.sub(r'\.\d{4}-\d{2}-\d{2}.*', '', name)
name = re.sub(r'\.roll-.*', '', name)
name = re.sub(r'\.(?:log|out|err)(?:\..*)?$', '', name)
return name
def rotate_mysql(root, execute, br, now=None):
root = Path(root).resolve(); now = time.time() if now is None else now
source = root / 'output/logs/mysql-server.log'
journal = root / '.runtime/operations/mysql-log-rotation.json'
if any(p.is_symlink() for p in (root / 'output', source.parent, source, root / '.runtime', journal.parent, journal)):
raise ValueError('Unsafe MySQL log path')
pending = None
if journal.exists():
pending = json.loads(journal.read_text())
if (pending.get('root') != str(root) or pending.get('kind') != 'quality-mysql-log-rotation' or
not re.fullmatch(r'mysql-server\.roll-\d+-[a-f0-9]{8}\.log', pending.get('archive', ''))):
raise ValueError('Unknown MySQL log rotation intent')
elif source.is_file() and source.stat().st_size:
old_day = time.localtime(source.stat().st_mtime)[:3]
if source.stat().st_size < 10 * 1024**2 and old_day == time.localtime(now)[:3]:
return 0
pending = {'kind': 'quality-mysql-log-rotation', 'root': str(root),
'archive': 'mysql-server.roll-' + str(int(now)) + '-' + uuid.uuid4().hex[:8] + '.log',
'fingerprint': list(fingerprint(source))}
br.atomic(journal, pending)
if pending is None:
return 0
target = source.with_name(pending['archive'])
if target.is_symlink():
raise ValueError('Unsafe MySQL rotated log')
if not target.exists():
if not source.is_file() or list(fingerprint(source)) != pending['fingerprint']:
journal.unlink(); return 0
source.rename(target)
# MySQL closes the old descriptor and reopens its configured path. No truncation.
execute('FLUSH ERROR LOGS;', None)
if not source.is_file() or source.is_symlink() or source.stat().st_ino == target.stat().st_ino:
raise RuntimeError('MySQL error log reopening not confirmed')
source.chmod(0o600)
target.chmod(0o600)
journal.unlink()
return 1
def retain(root, support, now=None, open_paths=None, max_bytes=MAX_BYTES, max_age=MAX_AGE):
now = time.time() if now is None else now
opened = opened_files() if open_paths is None else set(open_paths)
groups = {}; result = {'logsRemoved': 0, 'logBytesRemoved': 0, 'logsDeferred': 0}
for p in candidates(root, support):
groups.setdefault(category(p), []).append(p)
for files in groups.values():
files.sort(key=lambda p: p.stat().st_mtime)
total = sum(p.stat().st_size for p in files)
for p in files:
s = p.stat(); stamp = fingerprint(p)
if now - s.st_mtime <= max_age and total <= max_bytes:
continue
# Native appenders own these live files. The monitor handles archives;
# MySQL is reopened by its dedicated rotation function above.
if ((p.parent == Path(root) / 'output/logs' and p.name in ('backend.log', 'mysql-server.log', 'backend-console.log')) or
(p.parent == Path(support) / 'logs' and p.name in ('guardian.log', 'retention.log', 'operations.log'))):
result['logsDeferred'] += 1; continue
if str(p.absolute()) in opened:
result['logsDeferred'] += 1; continue
# Recheck descriptors immediately before mutation. Data/config/archives
# never enter this allowlist, and active writers are never truncated.
if p.is_symlink() or fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
result['logsDeferred'] += 1; continue
if s.st_size > max_bytes and now - s.st_mtime <= max_age and len(files) == 1:
# Keep the newest bounded tail of a closed oversized console log.
temp = p.with_name(p.name + '.' + uuid.uuid4().hex + '.tmp')
try:
with p.open('rb') as source, temp.open('xb') as output:
temp.chmod(0o600); source.seek(max(0, s.st_size - max_bytes))
source.readline() # Drop a possible partial leading line.
while block := source.read(1024 * 1024): output.write(block)
output.flush(); os.fsync(output.fileno())
if fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
result['logsDeferred'] += 1; continue
os.utime(temp, ns=(s.st_atime_ns, s.st_mtime_ns)); temp.replace(p)
removed = s.st_size - p.stat().st_size
total -= removed; result['logBytesRemoved'] += removed
finally:
temp.unlink(missing_ok=True)
else:
p.unlink(); total -= s.st_size
result['logsRemoved'] += 1; result['logBytesRemoved'] += s.st_size
return result