修复:独立回收中断备份并限制 AI 日志增长
This commit is contained in:
@@ -47,8 +47,10 @@ python3 deploy/docker/manage.py stop
|
|||||||
- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。
|
- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。
|
||||||
- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。
|
- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。
|
||||||
- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。
|
- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。
|
||||||
|
- 未完成的备份候选另行每 5 分钟检查,超过 24 小时、所有者和已登记子进程确认退出后才回收;该步骤不依赖新备份成功,也不要求数据库可连接。最新备份指针、有效候选、活跃进程、身份不完整的旧记录和未知目录均保留。
|
||||||
- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。
|
- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。
|
||||||
- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。
|
- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。
|
||||||
|
- AI 原生服务输出由受管捕获进程滚动写入,单段 10MiB、每个服务日志总量约 90MiB,归档最长 7 天。后台清理同时覆盖各用户及会话的日志;正在写入的文件继续保护。日志写入失败会限频重试并持续排空输出,不会为清日志终止正在进行的 AI 请求。已有运行实例继续工作,下次正常创建实例时采用新的日志捕获方式。
|
||||||
- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。
|
- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。
|
||||||
|
|
||||||
Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。
|
Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
@@ -24,6 +25,22 @@ import backup_retention as br
|
|||||||
|
|
||||||
|
|
||||||
class ContainerOperationsTests(unittest.TestCase):
|
class ContainerOperationsTests(unittest.TestCase):
|
||||||
|
def test_database_unavailable_does_not_block_expired_candidate_recovery(self):
|
||||||
|
worker = module('container_worker', ROOT / 'deploy/docker/worker.py')
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root=Path(directory).resolve();folder=root/'output/backups/20261108T010000Z-12345678';folder.mkdir(parents=True)
|
||||||
|
(folder/'database.sql.gz').write_bytes(b'interrupted dump');br.register_candidate(folder,root)
|
||||||
|
value=json.loads((folder/'.owner.json').read_text());value.update({'createdAt':time.time()-3*86400,'ownerPid':2000000000,'ownerCreated':1.0})
|
||||||
|
br.atomic(folder/'.owner.json',value);os.utime(folder,(time.time()-3*86400,)*2)
|
||||||
|
br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':'quality_current'})
|
||||||
|
br.atomic(root/'.runtime/operations/policy.json',worker.operations.DEFAULTS)
|
||||||
|
with patch.object(worker,'ROOT',root),patch.object(worker.operations,'snapshot',worker.operations.snapshot), \
|
||||||
|
patch('builtins.print') as output, \
|
||||||
|
patch.object(backup,'client',side_effect=RuntimeError('Database unavailable')):
|
||||||
|
with self.assertRaises(RuntimeError):worker.once('monitor')
|
||||||
|
self.assertFalse(folder.exists())
|
||||||
|
self.assertEqual(json.loads(output.call_args.args[0])['incompleteBackupsRemoved'],1)
|
||||||
|
|
||||||
def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self):
|
def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678'
|
root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678'
|
||||||
|
|||||||
+12
-3
@@ -39,17 +39,26 @@ def snapshot(root, support=None, now=None):
|
|||||||
mysql = estimates.get(cfg['mysqlDatabase'], 0)
|
mysql = estimates.get(cfg['mysqlDatabase'], 0)
|
||||||
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
|
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
|
||||||
native = operations.allocated(root / '.runtime/nao-users')
|
native = operations.allocated(root / '.runtime/nao-users')
|
||||||
logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations'))
|
log_paths = log_retention.candidates(root, root / '.runtime/operations')
|
||||||
core = mysql + archive + backups + native + logs
|
logs = sum(p.stat().st_blocks * 512 for p in log_paths)
|
||||||
|
native_logs = sum(p.stat().st_blocks * 512 for p in log_paths if p.is_relative_to(root / '.runtime/nao-users'))
|
||||||
|
core = mysql + archive + backups + max(0, native - native_logs) + logs
|
||||||
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
|
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
|
||||||
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
|
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
|
||||||
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native,
|
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': max(0, native - native_logs),
|
||||||
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||||||
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
|
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
|
||||||
'overdueTestResources': overdue}
|
'overdueTestResources': overdue}
|
||||||
|
|
||||||
|
|
||||||
def once(role):
|
def once(role):
|
||||||
|
# 数据库不可连接时也要能释放中断备份占用,不能放在采样或健康检查之后。
|
||||||
|
if not (ROOT / '.runtime/retention/paused').exists():
|
||||||
|
try:
|
||||||
|
result = br.clean_incomplete(ROOT)
|
||||||
|
if result['incompleteBackupsRemoved']:
|
||||||
|
print(json.dumps({'event': 'incomplete-backup-cleanup', **result}), flush=True)
|
||||||
|
except br.Busy: pass
|
||||||
if role == 'monitor':
|
if role == 'monitor':
|
||||||
operations.snapshot = snapshot
|
operations.snapshot = snapshot
|
||||||
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
|
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
|
||||||
|
|||||||
@@ -85,6 +85,8 @@ PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discove
|
|||||||
|
|
||||||
注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。
|
注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。
|
||||||
|
|
||||||
|
未完成备份候选使用独立回收步骤:超过 24 小时且进程身份确认退出后才清理,不等待下一次备份成功;当前备份、有效候选、身份不完整和仍运行的记录保留。AI 用户及会话日志也纳入后台检查;新启动的原生服务通过 `nao_log_stream.py` 捕获输出,单段 10MiB、每服务约 90MiB、归档最长 7 天。捕获进程与原生子进程共享进程组,沿用原有会话身份与停止保护;日志写入异常不会为清理中断 AI 请求。已有实例不因升级被终止,下次正常启动时加载新方式。
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD"
|
.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD"
|
||||||
.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD"
|
.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD"
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ def snapshot(root, support=SUPPORT, now=None):
|
|||||||
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
|
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
|
||||||
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
|
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
|
||||||
native = allocated(root / '.runtime/nao-users')
|
native = allocated(root / '.runtime/nao-users')
|
||||||
|
native_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime/nao-users'))
|
||||||
registered = 0; overdue = 0
|
registered = 0; overdue = 0
|
||||||
for p in resources.registry(root).glob('*.json'):
|
for p in resources.registry(root).glob('*.json'):
|
||||||
value = resources.load(root, p.stem)
|
value = resources.load(root, p.stem)
|
||||||
@@ -61,10 +62,11 @@ def snapshot(root, support=SUPPORT, now=None):
|
|||||||
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
|
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
|
||||||
# Core service data is the same scope as the earlier 11.86GiB measurement.
|
# Core service data is the same scope as the earlier 11.86GiB measurement.
|
||||||
# Software/build/delivery artifacts are visible separately and included in growth checks.
|
# Software/build/delivery artifacts are visible separately and included in growth checks.
|
||||||
core = max(0, mysql - registered) + archives + backups + log_bytes + native
|
native_data = max(0, native - native_logs)
|
||||||
artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs)
|
core = max(0, mysql - registered) + archives + backups + log_bytes + native_data
|
||||||
|
artifacts = max(0, runtime - mysql - private - native - (runtime_logs - native_logs)) + max(0, outputs - backups - output_logs)
|
||||||
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
|
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
|
||||||
'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered,
|
'logBytes': log_bytes, 'nativeDataBytes': native_data, 'registeredTestBytes': registered,
|
||||||
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||||||
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
|
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
|
||||||
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
|
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
|
||||||
@@ -143,17 +145,18 @@ def janitor(root, support, policy):
|
|||||||
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
|
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
|
||||||
try:
|
try:
|
||||||
with br.lock(root, timeout=0):
|
with br.lock(root, timeout=0):
|
||||||
|
result = br.clean_incomplete(root)
|
||||||
db = Database(root)
|
db = Database(root)
|
||||||
try:
|
try:
|
||||||
config = json.loads((root / '.runtime/retention/config.json').read_text())
|
config = json.loads((root / '.runtime/retention/config.json').read_text())
|
||||||
accepted = br.accepted_backend(root)
|
accepted = br.accepted_backend(root)
|
||||||
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
|
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
|
||||||
return {'deferred': 'Application version changed'}
|
return {**result, 'deferred': 'Application version changed'}
|
||||||
db.current()
|
db.current()
|
||||||
if db.busy(): return {'deferred': 'Application jobs are active'}
|
if db.busy(): return {**result, 'deferred': 'Application jobs are active'}
|
||||||
# Use a short-lived CLI connection so no observer holds a test schema open.
|
# Use a short-lived CLI connection so no observer holds a test schema open.
|
||||||
import backup
|
import backup
|
||||||
result = resources.cleanup(root, backup.client)
|
result.update(resources.cleanup(root, backup.client))
|
||||||
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
|
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
|
||||||
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
|
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
|
||||||
result.update(log_retention.retain(root, support,
|
result.update(log_retention.retain(root, support,
|
||||||
|
|||||||
@@ -372,6 +372,7 @@ def run(root, max_batches=30, grace=1800):
|
|||||||
state=json.loads(state_path.read_text()) if state_path.exists() else {}
|
state=json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||||
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
|
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
|
||||||
with br.lock(root,timeout=0):
|
with br.lock(root,timeout=0):
|
||||||
|
result.update(br.clean_incomplete(root))
|
||||||
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
|
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
|
||||||
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
|
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
|
||||||
db=Database(root)
|
db=Database(root)
|
||||||
|
|||||||
@@ -21,6 +21,16 @@ ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
|
|||||||
|
|
||||||
|
|
||||||
class OperationsTests(unittest.TestCase):
|
class OperationsTests(unittest.TestCase):
|
||||||
|
def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root=Path(d).resolve();native=root/'.runtime/nao-users/actor'
|
||||||
|
for relative in ['logs/server.log','private/nao.sqlite']:
|
||||||
|
path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192)
|
||||||
|
(root/'.runtime/tool.bin').write_bytes(b'x'*16384)
|
||||||
|
sample=ops.snapshot(root,root/'support')
|
||||||
|
self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes']))
|
||||||
|
self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users'))
|
||||||
|
|
||||||
def sample(self, **extra):
|
def sample(self, **extra):
|
||||||
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
|
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
|
||||||
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
|
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
|
||||||
|
|||||||
+3
-2
@@ -369,8 +369,7 @@ def create():
|
|||||||
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
||||||
folder.mkdir(parents=True, exist_ok=False)
|
folder.mkdir(parents=True, exist_ok=False)
|
||||||
folder.chmod(0o700)
|
folder.chmod(0o700)
|
||||||
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
|
backup_retention.register_candidate(folder, ROOT)
|
||||||
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
|
|
||||||
sql = folder / 'database.sql.gz'
|
sql = folder / 'database.sql.gz'
|
||||||
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||||
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
||||||
@@ -378,7 +377,9 @@ def create():
|
|||||||
process = None
|
process = None
|
||||||
try:
|
try:
|
||||||
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
||||||
|
backup_retention.candidate_child(folder, pending=True)
|
||||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
||||||
|
backup_retention.candidate_child(folder, process.pid)
|
||||||
try:
|
try:
|
||||||
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
|
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
|
||||||
shutil.copyfileobj(process.stdout, compressed)
|
shutil.copyfileobj(process.stdout, compressed)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from datetime import datetime, timezone
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
@@ -166,16 +167,9 @@ def _rotate(root, selected):
|
|||||||
for folder in base.iterdir():
|
for folder in base.iterdir():
|
||||||
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
||||||
continue
|
continue
|
||||||
# A restore-verified older folder is safe to retire under the shared lock.
|
# 旧有效备份仅在新有效备份发布后退役;未完成候选使用独立的所有权检查。
|
||||||
# Incomplete candidates are kept for 24h and never treated as good backups.
|
|
||||||
older = verified(folder, checksums=False)
|
older = verified(folder, checksums=False)
|
||||||
try:
|
if older is not None:
|
||||||
owner = json.loads((folder / '.owner.json').read_text())
|
|
||||||
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
|
|
||||||
except (OSError, ValueError):
|
|
||||||
owned = False
|
|
||||||
stale = owned and time.time() - folder.stat().st_mtime > 86400
|
|
||||||
if older is not None or stale:
|
|
||||||
contents = list(folder.rglob('*'))
|
contents = list(folder.rglob('*'))
|
||||||
if any(p.is_symlink() for p in contents):
|
if any(p.is_symlink() for p in contents):
|
||||||
continue
|
continue
|
||||||
@@ -204,6 +198,7 @@ def _rotate(root, selected):
|
|||||||
retire_folder(root,legacy,'legacy')
|
retire_folder(root,legacy,'legacy')
|
||||||
legacy_info['retired']=True
|
legacy_info['retired']=True
|
||||||
atomic(registration,legacy_info)
|
atomic(registration,legacy_info)
|
||||||
|
clean_incomplete(root)
|
||||||
clean_retired(root)
|
clean_retired(root)
|
||||||
return pointer, removed
|
return pointer, removed
|
||||||
|
|
||||||
@@ -282,6 +277,75 @@ def backup_ids(root):
|
|||||||
return {p.name for p in base.iterdir()} if base.exists() else set()
|
return {p.name for p in base.iterdir()} if base.exists() else set()
|
||||||
|
|
||||||
|
|
||||||
|
def register_candidate(folder, root):
|
||||||
|
import psutil
|
||||||
|
folder = Path(folder); info = folder.stat()
|
||||||
|
atomic(folder / '.owner.json', {'kind': 'quality-backup', 'formatVersion': 2,
|
||||||
|
'root': str(Path(root).resolve()), 'id': folder.name, 'createdAt': time.time(),
|
||||||
|
'ownerPid': os.getpid(), 'ownerCreated': psutil.Process().create_time(),
|
||||||
|
'directoryIdentity': [info.st_dev, info.st_ino], 'children': [], 'childPending': False})
|
||||||
|
|
||||||
|
|
||||||
|
def candidate_child(folder, pid=None, pending=False):
|
||||||
|
import psutil
|
||||||
|
path = Path(folder) / '.owner.json'; value = json.loads(path.read_text())
|
||||||
|
if value['ownerPid'] != os.getpid(): raise ValueError('Candidate owner changed')
|
||||||
|
value['childPending'] = pending
|
||||||
|
if pid is not None:
|
||||||
|
value['children'].append({'pid': pid, 'created': psutil.Process(pid).create_time()})
|
||||||
|
atomic(path, value)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_incomplete(root, now=None):
|
||||||
|
"""独立回收超过24小时的已登记、已退出候选;不依赖新备份或业务库可用。"""
|
||||||
|
import psutil
|
||||||
|
root = Path(root).resolve(); now = time.time() if now is None else now
|
||||||
|
result = {'incompleteBackupsRemoved': 0, 'incompleteBackupBytesRemoved': 0, 'incompleteBackupsDeferred': 0}
|
||||||
|
base = root / 'output/backups'
|
||||||
|
if any(p.is_symlink() for p in (root / 'output', base)): raise ValueError('Unsafe backup root')
|
||||||
|
def exited(item, pid_key, created_key):
|
||||||
|
if (not isinstance(item.get(pid_key), int) or item[pid_key] <= 1 or
|
||||||
|
not isinstance(item.get(created_key), (float, int)) or
|
||||||
|
not math.isfinite(item[created_key]) or item[created_key] <= 0): return False
|
||||||
|
try:
|
||||||
|
process = psutil.Process(item[pid_key])
|
||||||
|
return (abs(process.create_time() - item[created_key]) >= .02 or
|
||||||
|
not process.is_running() or process.status() == psutil.STATUS_ZOMBIE)
|
||||||
|
except psutil.NoSuchProcess: return True
|
||||||
|
except psutil.Error: return False
|
||||||
|
with lock(root, timeout=0):
|
||||||
|
pointer = root / '.runtime/retention/latest.json'
|
||||||
|
try:
|
||||||
|
if pointer.is_symlink(): return result
|
||||||
|
protected = json.loads(pointer.read_text())['id'] if pointer.exists() else None
|
||||||
|
except (OSError, ValueError, KeyError, TypeError): return result
|
||||||
|
for folder in base.iterdir() if base.exists() else []:
|
||||||
|
if folder.name == protected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): continue
|
||||||
|
owner = folder / '.owner.json'
|
||||||
|
if owner.is_symlink(): continue
|
||||||
|
try:
|
||||||
|
value = json.loads(owner.read_text()); info = folder.stat()
|
||||||
|
if not isinstance(value, dict): continue
|
||||||
|
if (value.get('kind') != 'quality-backup' or value.get('root') != str(root) or
|
||||||
|
value.get('id') != folder.name or not isinstance(value.get('createdAt'), (int, float)) or
|
||||||
|
not math.isfinite(value['createdAt']) or now - max(value['createdAt'], info.st_mtime) < 86400): continue
|
||||||
|
if verified(folder, checksums=False) is not None: continue
|
||||||
|
if (value.get('formatVersion') != 2 or value.get('directoryIdentity') != [info.st_dev, info.st_ino] or
|
||||||
|
value.get('childPending') is not False or not isinstance(value.get('children'), list) or
|
||||||
|
not exited(value, 'ownerPid', 'ownerCreated') or
|
||||||
|
not all(isinstance(p, dict) and exited(p, 'pid', 'created') for p in value['children'])):
|
||||||
|
result['incompleteBackupsDeferred'] += 1; continue
|
||||||
|
contents = list(folder.rglob('*'))
|
||||||
|
if any(p.is_symlink() for p in contents): continue
|
||||||
|
size = sum(p.stat().st_size for p in contents if p.is_file())
|
||||||
|
if folder.stat().st_ino != info.st_ino or json.loads(owner.read_text()) != value: continue
|
||||||
|
retire_folder(root, folder, 'backup')
|
||||||
|
result['incompleteBackupsRemoved'] += 1; result['incompleteBackupBytesRemoved'] += size
|
||||||
|
except (OSError, ValueError, TypeError): continue
|
||||||
|
clean_retired(root)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def discard_failed_new(root, before):
|
def discard_failed_new(root, before):
|
||||||
"""Delete only this call's registered, incomplete candidate after it has stopped."""
|
"""Delete only this call's registered, incomplete candidate after it has stopped."""
|
||||||
root=Path(root).resolve();base=root/'output/backups'
|
root=Path(root).resolve();base=root/'output/backups'
|
||||||
|
|||||||
@@ -16,7 +16,17 @@ MAX_AGE = 7 * 86400
|
|||||||
def candidates(root, support):
|
def candidates(root, support):
|
||||||
root, support = Path(root).resolve(), Path(support).resolve()
|
root, support = Path(root).resolve(), Path(support).resolve()
|
||||||
paths = set()
|
paths = set()
|
||||||
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
|
native_logs = []
|
||||||
|
native = root / '.runtime/nao-users'
|
||||||
|
if native.is_dir() and not native.is_symlink():
|
||||||
|
for actor in native.iterdir():
|
||||||
|
if actor.is_symlink() or not actor.is_dir(): continue
|
||||||
|
native_logs.append((actor / 'logs', 1))
|
||||||
|
sessions = actor / 'sessions'
|
||||||
|
if sessions.is_dir() and not sessions.is_symlink():
|
||||||
|
native_logs.extend((session / 'logs', 1) for session in sessions.iterdir()
|
||||||
|
if session.is_dir() and not session.is_symlink())
|
||||||
|
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1), *native_logs):
|
||||||
if not base.is_dir() or base.is_symlink():
|
if not base.is_dir() or base.is_symlink():
|
||||||
continue
|
continue
|
||||||
for directory, dirs, files in os.walk(base, followlinks=False):
|
for directory, dirs, files in os.walk(base, followlinks=False):
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"""为 NAO 原生子进程捕获输出,写入有大小和时间限制的私有日志。"""
|
||||||
|
import argparse
|
||||||
|
import codecs
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import logging
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import signal
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
SEGMENT_BYTES = 10 * 1024**2
|
||||||
|
TOTAL_BYTES = 90 * 1024**2
|
||||||
|
MAX_AGE = 7 * 86400
|
||||||
|
BACKUPS = 8
|
||||||
|
|
||||||
|
|
||||||
|
def safe_file(path):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Unsafe log path')
|
||||||
|
if path.exists():
|
||||||
|
info = path.stat()
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe log file')
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def writer_lock(path):
|
||||||
|
safe_file(path)
|
||||||
|
fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600)
|
||||||
|
try:
|
||||||
|
info = os.fstat(fd)
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: raise ValueError('Unsafe writer lock')
|
||||||
|
if os.name == 'nt':
|
||||||
|
import msvcrt
|
||||||
|
if not info.st_size: os.write(fd, b'0')
|
||||||
|
os.lseek(fd, 0, 0); msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
yield
|
||||||
|
finally: os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
def prune(path, total_bytes=TOTAL_BYTES, now=None):
|
||||||
|
now = time.time() if now is None else now
|
||||||
|
archives = [path.with_name(path.name + '.' + str(i)) for i in range(1, BACKUPS + 1)]
|
||||||
|
for archive in archives: safe_file(archive)
|
||||||
|
existing = sorted((p for p in archives if p.exists()), key=lambda p: p.stat().st_mtime)
|
||||||
|
total = (path.stat().st_size if path.exists() else 0) + sum(p.stat().st_size for p in existing)
|
||||||
|
for archive in existing:
|
||||||
|
if now - archive.stat().st_mtime > MAX_AGE or total > total_bytes:
|
||||||
|
total -= archive.stat().st_size; archive.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
class PrivateHandler(RotatingFileHandler):
|
||||||
|
def _open(self):
|
||||||
|
path = Path(self.baseFilename); safe_file(path)
|
||||||
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, 'O_NOFOLLOW', 0), 0o600)
|
||||||
|
if hasattr(os, 'fchmod'): os.fchmod(fd, 0o600)
|
||||||
|
else: os.chmod(path, 0o600)
|
||||||
|
return os.fdopen(fd, 'a', encoding='utf-8', errors='replace')
|
||||||
|
|
||||||
|
def handleError(self, record):
|
||||||
|
# 交给捕获层限频重试;不能因诊断日志失败终止业务子进程。
|
||||||
|
raise RuntimeError('Native log write failed')
|
||||||
|
|
||||||
|
|
||||||
|
def capture(path, command, segment_bytes=SEGMENT_BYTES, total_bytes=TOTAL_BYTES):
|
||||||
|
path = Path(path).absolute()
|
||||||
|
if path.name not in ('server.log', 'fastapi.log') or path.parent.name != 'logs': raise ValueError('Unexpected log name')
|
||||||
|
safe_file(path); path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
with writer_lock(path.with_name('writer-' + path.stem + '.lock')):
|
||||||
|
prune(path, total_bytes)
|
||||||
|
handler = PrivateHandler(path, maxBytes=segment_bytes, backupCount=BACKUPS, encoding='utf-8')
|
||||||
|
handler.setFormatter(logging.Formatter('%(message)s')); handler.terminator = ''
|
||||||
|
if path.stat().st_size >= segment_bytes or time.time() - path.stat().st_mtime > MAX_AGE:
|
||||||
|
handler.doRollover(); prune(path, total_bytes)
|
||||||
|
logger = logging.Logger('native-output', level=logging.INFO); logger.addHandler(handler)
|
||||||
|
child = None; stopping = False
|
||||||
|
def stop(signum, frame):
|
||||||
|
nonlocal stopping
|
||||||
|
stopping = True
|
||||||
|
if child is not None and child.poll() is None:
|
||||||
|
try: child.terminate()
|
||||||
|
except OSError: pass
|
||||||
|
previous = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
||||||
|
try:
|
||||||
|
if stopping: return 0
|
||||||
|
child = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
if stopping: child.terminate()
|
||||||
|
decoder = codecs.getincrementaldecoder('utf-8')(errors='replace')
|
||||||
|
retry_at = 0
|
||||||
|
def write(text):
|
||||||
|
nonlocal retry_at
|
||||||
|
if time.monotonic() < retry_at: return
|
||||||
|
try:
|
||||||
|
logger.info(text); prune(path, total_bytes)
|
||||||
|
except (OSError, RuntimeError, ValueError):
|
||||||
|
# 继续排空管道,防止日志空间不足拖住正在进行的 AI 请求。
|
||||||
|
retry_at = time.monotonic() + 60
|
||||||
|
while chunk := child.stdout.read1(min(64 * 1024, max(1, segment_bytes // 8))):
|
||||||
|
write(decoder.decode(chunk))
|
||||||
|
tail = decoder.decode(b'', final=True)
|
||||||
|
if tail: write(tail)
|
||||||
|
code = child.wait()
|
||||||
|
return code if code >= 0 else 128 - code
|
||||||
|
finally:
|
||||||
|
for signum, old in previous.items(): signal.signal(signum, old)
|
||||||
|
try: handler.close()
|
||||||
|
except OSError: pass
|
||||||
|
if child is not None:
|
||||||
|
if child.poll() is None:
|
||||||
|
child.terminate()
|
||||||
|
try: child.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired: child.kill(); child.wait()
|
||||||
|
child.stdout.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
os.umask(0o077)
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--log', type=Path, required=True)
|
||||||
|
parser.add_argument('command', nargs=argparse.REMAINDER)
|
||||||
|
args = parser.parse_args(); command = args.command
|
||||||
|
if command[:1] == ['--']: command = command[1:]
|
||||||
|
if not command: parser.error('A child command is required')
|
||||||
|
return capture(args.log, command)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try: raise SystemExit(main())
|
||||||
|
except Exception:
|
||||||
|
print('Native log capture failed', file=sys.stderr); raise SystemExit(1)
|
||||||
@@ -991,7 +991,7 @@ def _model_settings(*, include_options=False):
|
|||||||
|
|
||||||
def _config_fingerprint(model_base: str, model_key: str, model_name: str) -> str:
|
def _config_fingerprint(model_base: str, model_key: str, model_name: str) -> str:
|
||||||
paths = [TEMPLATES / "RULES.md", TEMPLATES / "agent" / "prompts" / "system.md",
|
paths = [TEMPLATES / "RULES.md", TEMPLATES / "agent" / "prompts" / "system.md",
|
||||||
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py",
|
ROOT / "scripts" / "nao_quality_runtime.py", ROOT / "scripts" / "nao_quality_runtime_mcp.py", ROOT / "scripts" / "nao_log_stream.py",
|
||||||
ROOT / "scripts" / "nao_quality_tools.py", ROOT / "scripts" / "nao_quality_mcp.py",
|
ROOT / "scripts" / "nao_quality_tools.py", ROOT / "scripts" / "nao_quality_mcp.py",
|
||||||
ROOT / "scripts" / "nao_model_settings.py", ROOT / "scripts" / "nao_model_gate.py", ROOT / "scripts" / "nao_evidence_finalizer.py", ROOT / "scripts" / "nao_evidence_report.py", ROOT / "scripts" / "nao_weight_ranking.py",
|
ROOT / "scripts" / "nao_model_settings.py", ROOT / "scripts" / "nao_model_gate.py", ROOT / "scripts" / "nao_evidence_finalizer.py", ROOT / "scripts" / "nao_evidence_report.py", ROOT / "scripts" / "nao_weight_ranking.py",
|
||||||
ROOT / "scripts" / "nao_metric_request.py", ROOT / "scripts" / "nao_analysis_coverage.py", ROOT / "scripts" / "nao_combined_ranking.py",
|
ROOT / "scripts" / "nao_metric_request.py", ROOT / "scripts" / "nao_analysis_coverage.py", ROOT / "scripts" / "nao_combined_ranking.py",
|
||||||
@@ -1209,6 +1209,11 @@ def _process_matches(record: dict[str, Any], actor_root: Path) -> bool:
|
|||||||
if not start or start != record.get("lstart"):
|
if not start or start != record.get("lstart"):
|
||||||
return False
|
return False
|
||||||
kind = record.get("kind")
|
kind = record.get("kind")
|
||||||
|
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
|
||||||
|
if wrapper in command:
|
||||||
|
name = "fastapi.log" if kind == "fastapi" else "server.log"
|
||||||
|
if "--log " + str(actor_root / "logs" / name) not in command:
|
||||||
|
return False
|
||||||
if kind == "fastapi":
|
if kind == "fastapi":
|
||||||
return str(PYTHON) in command and str(FASTAPI_MAIN) in command
|
return str(PYTHON) in command and str(FASTAPI_MAIN) in command
|
||||||
if kind == "server":
|
if kind == "server":
|
||||||
@@ -1668,21 +1673,23 @@ class RuntimeManager:
|
|||||||
path.touch(mode=0o600, exist_ok=True)
|
path.touch(mode=0o600, exist_ok=True)
|
||||||
path.chmod(0o600)
|
path.chmod(0o600)
|
||||||
processes: list[dict[str, Any]] = []
|
processes: list[dict[str, Any]] = []
|
||||||
log_path = logs / "server.log"
|
wrapper = str(Path(__file__).with_name("nao_log_stream.py"))
|
||||||
fd = _open_server_log(log_path, actor_root)
|
for log_path in (logs / "fastapi.log", logs / "server.log"):
|
||||||
|
os.close(_open_server_log(log_path, actor_root))
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "ab") as log:
|
fastapi = subprocess.Popen(
|
||||||
fastapi = subprocess.Popen(
|
[str(PYTHON), wrapper, "--log", str(logs / "fastapi.log"), "--", str(PYTHON), str(FASTAPI_MAIN)],
|
||||||
[str(PYTHON), str(FASTAPI_MAIN)], cwd=project, env=env,
|
cwd=project, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||||
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True,
|
stderr=subprocess.DEVNULL, start_new_session=True,
|
||||||
)
|
)
|
||||||
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
|
processes.append(_process_record(fastapi.pid, "fastapi", fastapi_port))
|
||||||
server = subprocess.Popen(
|
server = subprocess.Popen(
|
||||||
[str(NAO_SERVER), "serve", "--port", str(server_port), "--host", "127.0.0.1"],
|
[str(PYTHON), wrapper, "--log", str(logs / "server.log"), "--", str(NAO_SERVER),
|
||||||
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL,
|
"serve", "--port", str(server_port), "--host", "127.0.0.1"],
|
||||||
stdout=log, stderr=log, start_new_session=True,
|
cwd=NAO_SERVER.parent, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||||
)
|
stderr=subprocess.DEVNULL, start_new_session=True,
|
||||||
processes.append(_process_record(server.pid, "server", server_port))
|
)
|
||||||
|
processes.append(_process_record(server.pid, "server", server_port))
|
||||||
except Exception:
|
except Exception:
|
||||||
for record in processes:
|
for record in processes:
|
||||||
if _process_matches(record, actor_root):
|
if _process_matches(record, actor_root):
|
||||||
|
|||||||
@@ -3,16 +3,54 @@ import gzip
|
|||||||
import hashlib
|
import hashlib
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tarfile
|
import tarfile
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import backup
|
import backup
|
||||||
|
|
||||||
|
|
||||||
class BackupRemediationTest(unittest.TestCase):
|
class BackupRemediationTest(unittest.TestCase):
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def private_verifier_fixture(self, root):
|
||||||
|
"""Exercise verify's Darwin/container path without starting a MySQL server."""
|
||||||
|
import backup_verifier
|
||||||
|
|
||||||
|
config = root / '.runtime/retention/verify-fixture/client.cnf'
|
||||||
|
config.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n')
|
||||||
|
registration = root / '.runtime/retention/scratch/fixture.json'
|
||||||
|
registered = []
|
||||||
|
child_pids = []
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def fake_instance(instance_root):
|
||||||
|
self.assertEqual(Path(instance_root), root)
|
||||||
|
yield config
|
||||||
|
|
||||||
|
def fake_scratch_register(instance_root, database, server):
|
||||||
|
self.assertEqual(Path(instance_root), root)
|
||||||
|
self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$')
|
||||||
|
self.assertEqual(server, 'synthetic-server-uuid')
|
||||||
|
registration.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
registration.write_text('{}')
|
||||||
|
registered.append((database, server))
|
||||||
|
return registration
|
||||||
|
|
||||||
|
with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \
|
||||||
|
patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \
|
||||||
|
patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \
|
||||||
|
patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \
|
||||||
|
patch.object(backup.backup_retention, 'scratch_child',
|
||||||
|
side_effect=lambda path, pid: child_pids.append((Path(path), pid))):
|
||||||
|
yield {'config': config, 'instance': instance, 'registration': registration,
|
||||||
|
'registered': registered, 'childPids': child_pids,
|
||||||
|
'scratchCreated': scratch_created}
|
||||||
|
|
||||||
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
|
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root = Path(tmp)
|
root = Path(tmp)
|
||||||
@@ -23,18 +61,46 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
|
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
|
||||||
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
|
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
|
||||||
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
|
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
|
||||||
|
# candidate_child() records process creation time via psutil.
|
||||||
|
# Reuse this live test process as a synthetic child; no command runs.
|
||||||
|
self.pid = os.getpid()
|
||||||
|
|
||||||
def wait(self): return 0
|
def wait(self): return 0
|
||||||
def poll(self): return 0
|
def poll(self): return 0
|
||||||
def kill(self): pass
|
def kill(self): pass
|
||||||
|
|
||||||
|
seen_client = []
|
||||||
|
|
||||||
|
def fake_client(statement, database=None):
|
||||||
|
seen_client.append((statement, database))
|
||||||
|
self.assertEqual(database, 'quality_fixture')
|
||||||
|
if statement == 'SHOW TABLES;':
|
||||||
|
return 'binary_fixture\n'
|
||||||
|
if statement == 'SELECT COUNT(*) FROM `binary_fixture`;':
|
||||||
|
return '1\n'
|
||||||
|
self.fail('Unexpected isolated export SQL: ' + statement)
|
||||||
|
|
||||||
with patch.object(backup, 'ROOT', root), \
|
with patch.object(backup, 'ROOT', root), \
|
||||||
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
|
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
|
||||||
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
|
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
|
||||||
patch.object(backup, 'verify_references', return_value={'verified': True}), \
|
patch.object(backup, 'verify_references', return_value={'verified': True}), \
|
||||||
patch.object(backup, 'query_evidence', return_value={}), \
|
patch.object(backup, 'query_evidence', return_value={}), \
|
||||||
|
patch.object(backup, 'client', side_effect=fake_client), \
|
||||||
patch.object(backup.subprocess, 'Popen', BinaryDump):
|
patch.object(backup.subprocess, 'Popen', BinaryDump):
|
||||||
folder = backup.create()
|
folder = backup.create()
|
||||||
|
self.assertEqual(seen_client, [
|
||||||
|
('SHOW TABLES;', 'quality_fixture'),
|
||||||
|
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
|
||||||
|
('SHOW TABLES;', 'quality_fixture'),
|
||||||
|
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
|
||||||
|
])
|
||||||
|
owner = json.loads((folder / '.owner.json').read_text())
|
||||||
|
self.assertEqual(owner['ownerPid'], os.getpid())
|
||||||
|
self.assertIsInstance(owner['ownerCreated'], (int, float))
|
||||||
|
self.assertFalse(owner['childPending'])
|
||||||
|
self.assertEqual(len(owner['children']), 1)
|
||||||
|
self.assertEqual(owner['children'][0]['pid'], os.getpid())
|
||||||
|
self.assertIsInstance(owner['children'][0]['created'], (int, float))
|
||||||
manifest = json.loads((folder / 'manifest.json').read_text())
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
||||||
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
|
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
|
||||||
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
|
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
|
||||||
@@ -189,6 +255,9 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
|
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
|
||||||
|
|
||||||
def fake_client(statement, database=None):
|
def fake_client(statement, database=None):
|
||||||
|
self.assertEqual(backup.client_config(), private['config'])
|
||||||
|
if statement == 'SELECT @@server_uuid;':
|
||||||
|
return 'synthetic-server-uuid\n'
|
||||||
if statement.startswith('CREATE DATABASE'):
|
if statement.startswith('CREATE DATABASE'):
|
||||||
state['database'] = statement.split('`')[1]
|
state['database'] = statement.split('`')[1]
|
||||||
return ''
|
return ''
|
||||||
@@ -221,6 +290,7 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
class FakeMysqlRestore:
|
class FakeMysqlRestore:
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
state['restoreCommand'] = list(args[0])
|
state['restoreCommand'] = list(args[0])
|
||||||
|
self.pid = 424242
|
||||||
self.stdin = RecordingInput()
|
self.stdin = RecordingInput()
|
||||||
self.returncode = 0
|
self.returncode = 0
|
||||||
def wait(self):
|
def wait(self):
|
||||||
@@ -230,7 +300,8 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
def kill(self):
|
def kill(self):
|
||||||
self.returncode = -9
|
self.returncode = -9
|
||||||
|
|
||||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
with self.private_verifier_fixture(root) as private, \
|
||||||
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||||
patch.object(backup, 'client', side_effect=fake_client), \
|
patch.object(backup, 'client', side_effect=fake_client), \
|
||||||
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
||||||
result = backup.verify(folder)
|
result = backup.verify(folder)
|
||||||
@@ -239,6 +310,12 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
|
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
|
||||||
self.assertTrue(state['updated'])
|
self.assertTrue(state['updated'])
|
||||||
self.assertTrue(state['dropped'])
|
self.assertTrue(state['dropped'])
|
||||||
|
self.assertTrue(private['instance'].called)
|
||||||
|
self.assertTrue(private['registered'])
|
||||||
|
self.assertEqual(private['childPids'], [(private['registration'], 424242)])
|
||||||
|
private['scratchCreated'].assert_called_once_with(private['registration'])
|
||||||
|
self.assertFalse(private['registration'].exists())
|
||||||
|
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
|
||||||
self.assertEqual(state['database'].split('_')[0], 'quality')
|
self.assertEqual(state['database'].split('_')[0], 'quality')
|
||||||
self.assertFalse(result['migrationReady'])
|
self.assertFalse(result['migrationReady'])
|
||||||
self.assertTrue(result['archiveReferencesRebound'])
|
self.assertTrue(result['archiveReferencesRebound'])
|
||||||
@@ -502,17 +579,23 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
'tableCounts': {}, 'queryEvidence': None,
|
'tableCounts': {}, 'queryEvidence': None,
|
||||||
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
|
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
|
||||||
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
|
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
|
||||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
with self.private_verifier_fixture(root) as private, \
|
||||||
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||||
self.assertRaises(backup.BackupFailure) as checksum:
|
self.assertRaises(backup.BackupFailure) as checksum:
|
||||||
backup.verify(folder)
|
backup.verify(folder)
|
||||||
self.assertEqual((checksum.exception.stage, checksum.exception.code),
|
self.assertEqual((checksum.exception.stage, checksum.exception.code),
|
||||||
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
|
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
|
||||||
|
private['instance'].assert_called_once_with(root)
|
||||||
|
self.assertEqual(private['registered'], [])
|
||||||
|
|
||||||
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
|
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
|
||||||
state = {'dropped': False}
|
state = {'dropped': False}
|
||||||
def fake_client(statement, database=None):
|
def fake_client(statement, database=None):
|
||||||
|
self.assertEqual(backup.client_config(), private['config'])
|
||||||
if statement.startswith('CREATE DATABASE'):
|
if statement.startswith('CREATE DATABASE'):
|
||||||
return ''
|
return ''
|
||||||
|
if statement == 'SELECT @@server_uuid;':
|
||||||
|
return 'synthetic-server-uuid\n'
|
||||||
if statement.startswith('DROP DATABASE'):
|
if statement.startswith('DROP DATABASE'):
|
||||||
state['dropped'] = True
|
state['dropped'] = True
|
||||||
return ''
|
return ''
|
||||||
@@ -522,6 +605,7 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
|
|
||||||
class FailedMysqlRestore:
|
class FailedMysqlRestore:
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
self.pid = 434343
|
||||||
self.stdin = io.BytesIO()
|
self.stdin = io.BytesIO()
|
||||||
self.returncode = 1
|
self.returncode = 1
|
||||||
def wait(self):
|
def wait(self):
|
||||||
@@ -531,7 +615,8 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
def kill(self):
|
def kill(self):
|
||||||
self.returncode = -9
|
self.returncode = -9
|
||||||
|
|
||||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
with self.private_verifier_fixture(root) as private, \
|
||||||
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||||
patch.object(backup, 'client', side_effect=fake_client), \
|
patch.object(backup, 'client', side_effect=fake_client), \
|
||||||
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
|
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
|
||||||
self.assertRaises(backup.BackupFailure) as restore:
|
self.assertRaises(backup.BackupFailure) as restore:
|
||||||
@@ -539,6 +624,11 @@ class BackupRemediationTest(unittest.TestCase):
|
|||||||
self.assertEqual((restore.exception.stage, restore.exception.code),
|
self.assertEqual((restore.exception.stage, restore.exception.code),
|
||||||
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
|
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
|
||||||
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
|
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
|
||||||
|
private['instance'].assert_called_once_with(root)
|
||||||
|
self.assertEqual(private['childPids'], [(private['registration'], 434343)])
|
||||||
|
private['scratchCreated'].assert_called_once_with(private['registration'])
|
||||||
|
self.assertFalse(private['registration'].exists())
|
||||||
|
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
|
||||||
|
|
||||||
old_root = (root / 'old/runtimedata/archive').resolve()
|
old_root = (root / 'old/runtimedata/archive').resolve()
|
||||||
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
|
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import nao_log_stream as stream
|
||||||
|
import log_retention
|
||||||
|
|
||||||
|
|
||||||
|
class NativeLogTests(unittest.TestCase):
|
||||||
|
def test_log_write_failure_does_not_interrupt_child_work(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory).resolve(); result = root / 'completed.txt'
|
||||||
|
command = [sys.executable, '-c', 'import sys,pathlib;sys.stdout.write("x"*300000);pathlib.Path(sys.argv[1]).write_text("done")', str(result)]
|
||||||
|
with patch.object(stream.PrivateHandler, 'emit', side_effect=OSError('Disk full')):
|
||||||
|
self.assertEqual(stream.capture(root / 'logs/server.log', command), 0)
|
||||||
|
self.assertEqual(result.read_text(), 'done')
|
||||||
|
|
||||||
|
def test_real_child_output_rolls_and_stays_bounded(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
path = Path(directory).resolve() / 'logs/server.log'
|
||||||
|
command = [sys.executable, '-c', 'import sys;sys.stdout.write("old line\\n"*6000+"LAST_MARKER\\n");sys.stderr.write("stderr marker\\n")']
|
||||||
|
self.assertEqual(stream.capture(path, command, segment_bytes=1024, total_bytes=9000), 0)
|
||||||
|
files = list(path.parent.glob('server.log*'))
|
||||||
|
self.assertGreater(len(files), 1); self.assertLessEqual(sum(p.stat().st_size for p in files), 9000)
|
||||||
|
contents = ''.join(p.read_text() for p in files)
|
||||||
|
self.assertIn('LAST_MARKER', contents); self.assertIn('stderr marker', contents)
|
||||||
|
for p in files: self.assertEqual(p.stat().st_mode & 0o077, 0)
|
||||||
|
|
||||||
|
def test_oversized_legacy_log_and_expired_archives_are_bounded_on_start(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
path = Path(directory).resolve() / 'logs/server.log'; path.parent.mkdir()
|
||||||
|
path.write_bytes(b'old' * 10000); old = path.with_name('server.log.8'); old.write_bytes(b'expired')
|
||||||
|
os.utime(old, (time.time() - 9 * 86400,) * 2)
|
||||||
|
stream.capture(path, [sys.executable, '-c', 'print("new")'], segment_bytes=1024, total_bytes=9000)
|
||||||
|
self.assertIn('new', path.read_text()); self.assertFalse(old.exists())
|
||||||
|
self.assertLessEqual(sum(p.stat().st_size for p in path.parent.glob('server.log*')), 9000)
|
||||||
|
|
||||||
|
def test_symlink_log_or_archive_is_rejected_before_child_launch(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory).resolve(); path = root / 'logs/server.log'; path.parent.mkdir()
|
||||||
|
outside = root / 'data.xlsx'; outside.write_bytes(b'business data')
|
||||||
|
path.with_name('server.log.1').symlink_to(outside)
|
||||||
|
with self.assertRaises(ValueError): stream.capture(path, [sys.executable, '-c', 'print("no")'])
|
||||||
|
self.assertEqual(outside.read_bytes(), b'business data')
|
||||||
|
|
||||||
|
@unittest.skipIf(os.name == 'nt', 'POSIX process-group shutdown contract')
|
||||||
|
def test_stop_keeps_wrapper_until_term_ignoring_child_can_be_killed_as_a_group(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory).resolve(); path = root / 'logs/fastapi.log'
|
||||||
|
command = [sys.executable, str(Path(stream.__file__).resolve()), '--log', str(path), '--', sys.executable, '-u', '-c',
|
||||||
|
'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);print("READY",flush=True);time.sleep(60)']
|
||||||
|
process = subprocess.Popen(command, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
try:
|
||||||
|
deadline = time.monotonic() + 5
|
||||||
|
while not path.exists() or 'READY' not in path.read_text():
|
||||||
|
if time.monotonic() > deadline: self.fail('Native wrapper did not start')
|
||||||
|
time.sleep(.02)
|
||||||
|
import nao_quality_runtime as runtime
|
||||||
|
with patch.object(runtime, 'PYTHON', Path(sys.executable)), patch.object(runtime, 'FASTAPI_MAIN', Path('-c')):
|
||||||
|
record=runtime._process_record(process.pid,'fastapi',5283)
|
||||||
|
self.assertTrue(runtime._process_matches(record,root))
|
||||||
|
self.assertFalse(runtime._process_matches(record,root/'another-session'))
|
||||||
|
os.killpg(process.pid, signal.SIGTERM); time.sleep(.1)
|
||||||
|
self.assertIsNone(process.poll())
|
||||||
|
os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
|
||||||
|
finally:
|
||||||
|
if process.poll() is None: os.killpg(process.pid, signal.SIGKILL); process.wait(timeout=5)
|
||||||
|
|
||||||
|
def test_native_actor_and_session_logs_are_cleaned_but_business_files_and_open_logs_are_kept(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory).resolve(); support = root / 'support'
|
||||||
|
actor = root / '.runtime/nao-users/actor'
|
||||||
|
paths = [actor / 'logs/server.log', actor / 'sessions/session/logs/fastapi.log.1']
|
||||||
|
for path in paths:
|
||||||
|
path.parent.mkdir(parents=True); path.write_bytes(b'old diagnostics\n'); os.utime(path, (time.time() - 9 * 86400,) * 2)
|
||||||
|
private = actor / 'sessions/session/private/nao.sqlite'; private.parent.mkdir(); private.write_bytes(b'keep')
|
||||||
|
self.assertTrue(set(paths).issubset(log_retention.candidates(root, support)))
|
||||||
|
result = log_retention.retain(root, support, open_paths={str(paths[0])})
|
||||||
|
self.assertEqual(result['logsRemoved'], 1); self.assertTrue(paths[0].exists()); self.assertTrue(private.exists())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': unittest.main()
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
|
||||||
|
import backup
|
||||||
|
import backup_retention as br
|
||||||
|
import backup_verifier
|
||||||
|
import operations
|
||||||
|
import retention
|
||||||
|
from test_retention import good_backup
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateRecoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temporary = tempfile.TemporaryDirectory(); self.root = Path(self.temporary.name).resolve()
|
||||||
|
def tearDown(self): self.temporary.cleanup()
|
||||||
|
|
||||||
|
def candidate(self, suffix='22222222', dead=True, age=3 * 86400):
|
||||||
|
folder = self.root / 'output/backups' / ('20261108T010000Z-' + suffix)
|
||||||
|
folder.mkdir(parents=True); (folder / 'database.sql.gz').write_bytes(b'partial dump')
|
||||||
|
br.register_candidate(folder, self.root)
|
||||||
|
owner = json.loads((folder / '.owner.json').read_text()); owner['createdAt'] -= age
|
||||||
|
if dead: owner.update({'ownerPid': 2000000000, 'ownerCreated': 1.0})
|
||||||
|
br.atomic(folder / '.owner.json', owner); os.utime(folder, (time.time() - age,) * 2)
|
||||||
|
return folder
|
||||||
|
|
||||||
|
def test_expired_dead_candidate_is_cleaned_without_a_new_backup(self):
|
||||||
|
current = good_backup(self.root, '11111111'); br.rotate(self.root, current)
|
||||||
|
orphan = self.candidate()
|
||||||
|
result = br.clean_incomplete(self.root)
|
||||||
|
self.assertEqual(result['incompleteBackupsRemoved'], 1)
|
||||||
|
self.assertGreater(result['incompleteBackupBytesRemoved'], 0)
|
||||||
|
self.assertFalse(orphan.exists()); self.assertEqual(br.latest(self.root)[0], current)
|
||||||
|
|
||||||
|
def test_live_parent_live_child_pending_and_unknown_records_are_preserved(self):
|
||||||
|
parent = self.candidate('11111111', dead=False)
|
||||||
|
child = self.candidate('22222222'); br.candidate_child(parent, os.getpid())
|
||||||
|
value = json.loads((parent / '.owner.json').read_text())
|
||||||
|
child_owner = json.loads((child / '.owner.json').read_text()); child_owner['children'] = value['children']
|
||||||
|
br.atomic(child / '.owner.json', child_owner); os.utime(child, (time.time() - 3 * 86400,) * 2)
|
||||||
|
pending = self.candidate('33333333'); owner = json.loads((pending / '.owner.json').read_text())
|
||||||
|
owner['childPending'] = True; br.atomic(pending / '.owner.json', owner); os.utime(pending, (time.time() - 3 * 86400,) * 2)
|
||||||
|
unknown = self.candidate('44444444'); owner = json.loads((unknown / '.owner.json').read_text())
|
||||||
|
owner.pop('ownerCreated'); br.atomic(unknown / '.owner.json', owner); os.utime(unknown, (time.time() - 3 * 86400,) * 2)
|
||||||
|
result = br.clean_incomplete(self.root)
|
||||||
|
self.assertEqual(result['incompleteBackupsRemoved'], 0)
|
||||||
|
self.assertGreaterEqual(result['incompleteBackupsDeferred'], 3)
|
||||||
|
for folder in (parent, child, pending, unknown): self.assertTrue(folder.exists())
|
||||||
|
|
||||||
|
def test_latest_pointer_valid_candidate_fresh_and_symlink_contents_are_protected(self):
|
||||||
|
pointed = self.candidate('11111111'); br.atomic(self.root / '.runtime/retention/latest.json', {'id': pointed.name})
|
||||||
|
valid = good_backup(self.root, '55555555')
|
||||||
|
fresh = self.candidate('22222222', age=10)
|
||||||
|
linked = self.candidate('33333333'); outside = self.root / 'business.xlsx'; outside.write_bytes(b'keep')
|
||||||
|
(linked / 'link').symlink_to(outside)
|
||||||
|
old = time.time() - 3 * 86400; os.utime(linked, (old, old))
|
||||||
|
result = br.clean_incomplete(self.root)
|
||||||
|
self.assertEqual(result['incompleteBackupsRemoved'], 0)
|
||||||
|
for p in (pointed, valid, fresh, linked, outside): self.assertTrue(p.exists())
|
||||||
|
|
||||||
|
def test_monitor_cleanup_precedes_database_access(self):
|
||||||
|
orphan = self.candidate()
|
||||||
|
with patch.object(retention, 'Database', side_effect=RuntimeError('Database unavailable')):
|
||||||
|
with self.assertRaises(RuntimeError): operations.janitor(self.root, self.root / 'support', operations.DEFAULTS)
|
||||||
|
self.assertFalse(orphan.exists())
|
||||||
|
|
||||||
|
def test_low_disk_recovery_removes_orphan_before_deferring_new_backup(self):
|
||||||
|
current = good_backup(self.root, '11111111'); br.rotate(self.root, current); orphan = self.candidate()
|
||||||
|
class Reply:
|
||||||
|
def __enter__(self): return self
|
||||||
|
def __exit__(self, *_): return False
|
||||||
|
def read(self, *_): return b'{"status":"UP"}'
|
||||||
|
class Opener:
|
||||||
|
def open(self, *_args, **_kwargs): return Reply()
|
||||||
|
class Database:
|
||||||
|
def __init__(self, root): self.root = root; self.name = 'synthetic'; self.conn = self
|
||||||
|
def close(self): pass
|
||||||
|
def clean_owned_scratch(self, *_): return 0
|
||||||
|
def current(self): return 'current'
|
||||||
|
def busy(self): return False
|
||||||
|
def rows(self, *_): return [{'size': 1}]
|
||||||
|
with patch.object(retention.urllib.request, 'build_opener', return_value=Opener()), \
|
||||||
|
patch.object(retention, 'Database', Database), patch.object(backup_verifier, 'clean_abandoned', return_value=0), \
|
||||||
|
patch.object(retention.shutil, 'disk_usage', return_value=shutil._ntuple_diskusage(100, 99, 1)), \
|
||||||
|
patch.object(backup, 'create') as create:
|
||||||
|
with self.assertRaises(retention.Busy): retention.run(self.root)
|
||||||
|
self.assertFalse(orphan.exists()); self.assertFalse(create.called)
|
||||||
|
self.assertEqual(br.latest(self.root)[0], current)
|
||||||
|
|
||||||
|
def test_unreadable_process_identity_defers_cleanup(self):
|
||||||
|
import psutil
|
||||||
|
orphan = self.candidate()
|
||||||
|
with patch.object(psutil, 'Process', side_effect=psutil.AccessDenied()):
|
||||||
|
result = br.clean_incomplete(self.root)
|
||||||
|
self.assertEqual(result['incompleteBackupsRemoved'], 0); self.assertTrue(orphan.exists())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': unittest.main()
|
||||||
Reference in New Issue
Block a user