修复:独立回收中断备份并限制 AI 日志增长

This commit is contained in:
2026-11-10 16:02:46 +08:00
parent dc5b9b76ec
commit cc8832c8e5
15 changed files with 584 additions and 39 deletions
+2
View File
@@ -47,8 +47,10 @@ python3 deploy/docker/manage.py stop
- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。
- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。
- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。
- 未完成的备份候选另行每 5 分钟检查,超过 24 小时、所有者和已登记子进程确认退出后才回收;该步骤不依赖新备份成功,也不要求数据库可连接。最新备份指针、有效候选、活跃进程、身份不完整的旧记录和未知目录均保留。
- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。
- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。
- AI 原生服务输出由受管捕获进程滚动写入,单段 10MiB、每个服务日志总量约 90MiB,归档最长 7 天。后台清理同时覆盖各用户及会话的日志;正在写入的文件继续保护。日志写入失败会限频重试并持续排空输出,不会为清日志终止正在进行的 AI 请求。已有运行实例继续工作,下次正常创建实例时采用新的日志捕获方式。
- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。
Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。
+17
View File
@@ -4,6 +4,7 @@ import os
from pathlib import Path
import sys
import tempfile
import time
import unittest
from unittest.mock import Mock, patch
@@ -24,6 +25,22 @@ import backup_retention as br
class ContainerOperationsTests(unittest.TestCase):
def test_database_unavailable_does_not_block_expired_candidate_recovery(self):
worker = module('container_worker', ROOT / 'deploy/docker/worker.py')
with tempfile.TemporaryDirectory() as directory:
root=Path(directory).resolve();folder=root/'output/backups/20261108T010000Z-12345678';folder.mkdir(parents=True)
(folder/'database.sql.gz').write_bytes(b'interrupted dump');br.register_candidate(folder,root)
value=json.loads((folder/'.owner.json').read_text());value.update({'createdAt':time.time()-3*86400,'ownerPid':2000000000,'ownerCreated':1.0})
br.atomic(folder/'.owner.json',value);os.utime(folder,(time.time()-3*86400,)*2)
br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':'quality_current'})
br.atomic(root/'.runtime/operations/policy.json',worker.operations.DEFAULTS)
with patch.object(worker,'ROOT',root),patch.object(worker.operations,'snapshot',worker.operations.snapshot), \
patch('builtins.print') as output, \
patch.object(backup,'client',side_effect=RuntimeError('Database unavailable')):
with self.assertRaises(RuntimeError):worker.once('monitor')
self.assertFalse(folder.exists())
self.assertEqual(json.loads(output.call_args.args[0])['incompleteBackupsRemoved'],1)
def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678'
+12 -3
View File
@@ -39,17 +39,26 @@ def snapshot(root, support=None, now=None):
mysql = estimates.get(cfg['mysqlDatabase'], 0)
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
native = operations.allocated(root / '.runtime/nao-users')
logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations'))
core = mysql + archive + backups + native + logs
log_paths = log_retention.candidates(root, root / '.runtime/operations')
logs = sum(p.stat().st_blocks * 512 for p in log_paths)
native_logs = sum(p.stat().st_blocks * 512 for p in log_paths if p.is_relative_to(root / '.runtime/nao-users'))
core = mysql + archive + backups + max(0, native - native_logs) + logs
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native,
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': max(0, native - native_logs),
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
'overdueTestResources': overdue}
def once(role):
# 数据库不可连接时也要能释放中断备份占用,不能放在采样或健康检查之后。
if not (ROOT / '.runtime/retention/paused').exists():
try:
result = br.clean_incomplete(ROOT)
if result['incompleteBackupsRemoved']:
print(json.dumps({'event': 'incomplete-backup-cleanup', **result}), flush=True)
except br.Busy: pass
if role == 'monitor':
operations.snapshot = snapshot
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
+2
View File
@@ -85,6 +85,8 @@ PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discove
注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。
未完成备份候选使用独立回收步骤:超过 24 小时且进程身份确认退出后才清理,不等待下一次备份成功;当前备份、有效候选、身份不完整和仍运行的记录保留。AI 用户及会话日志也纳入后台检查;新启动的原生服务通过 `nao_log_stream.py` 捕获输出,单段 10MiB、每服务约 90MiB、归档最长 7 天。捕获进程与原生子进程共享进程组,沿用原有会话身份与停止保护;日志写入异常不会为清理中断 AI 请求。已有实例不因升级被终止,下次正常启动时加载新方式。
```sh
.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD"
.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD"
+9 -6
View File
@@ -47,6 +47,7 @@ def snapshot(root, support=SUPPORT, now=None):
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
native = allocated(root / '.runtime/nao-users')
native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users'))
registered = 0; overdue = 0
for p in resources.registry(root).glob('*.json'):
value = resources.load(root, p.stem)
@@ -61,10 +62,11 @@ def snapshot(root, support=SUPPORT, now=None):
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
# Core service data is the same scope as the earlier 11.86GiB measurement.
# Software/build/delivery artifacts are visible separately and included in growth checks.
core = max(0, mysql - registered) + archives + backups + log_bytes + native
artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs)
native_data = max(0, native - native_logs)
core = max(0, mysql - registered) + archives + backups + log_bytes + native_data
artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs)
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered,
'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered,
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
@@ -143,17 +145,18 @@ def janitor(root, support, policy):
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
try:
with br.lock(root, timeout=0):
result = br.clean_incomplete(root)
db = Database(root)
try:
config = json.loads((root / '.runtime/retention/config.json').read_text())
accepted = br.accepted_backend(root)
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
return {'deferred': 'Application version changed'}
return {**result, 'deferred': 'Application version changed'}
db.current()
if db.busy(): return {'deferred': 'Application jobs are active'}
if db.busy(): return {**result, 'deferred': 'Application jobs are active'}
# Use a short-lived CLI connection so no observer holds a test schema open.
import backup
result = resources.cleanup(root, backup.client)
result.update(resources.cleanup(root, backup.client))
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
result.update(log_retention.retain(root, support,
+1
View File
@@ -372,6 +372,7 @@ def run(root, max_batches=30, grace=1800):
state=json.loads(state_path.read_text()) if state_path.exists() else {}
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
with br.lock(root,timeout=0):
result.update(br.clean_incomplete(root))
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
db=Database(root)
+10
View File
@@ -21,6 +21,16 @@ ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
class OperationsTests(unittest.TestCase):
def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d).resolve();native=root/'.runtime/nao-users/actor'
for relative in ['logs/server.log','private/nao.sqlite']:
path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192)
(root/'.runtime/tool.bin').write_bytes(b'x'*16384)
sample=ops.snapshot(root,root/'support')
self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes']))
self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users'))
def sample(self, **extra):
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}