新增:Linux Docker 部署与自动回收、备份及容量监测

This commit is contained in:
2026-11-10 14:53:07 +08:00
parent 3962521a5c
commit dc5b9b76ec
45 changed files with 5591 additions and 35 deletions
+213
View File
@@ -0,0 +1,213 @@
"""Reclaim only unreferenced, completed source revisions and their owned archives."""
import hashlib
import json
from contextlib import contextmanager
from pathlib import Path
import re
import time
import uuid
# These are measured business values, not documents that refer to source IDs.
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
class Cleaner:
def __init__(self, db, br, safe_file, fingerprint, defer):
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
self.root = db.root.resolve()
self.boundary = self.root / 'runtimedata/archive/source-revisions'
self.journals = self.root / '.runtime/retention/source-files'
self.reference_deadline = None
@contextmanager
def quick_checks(self):
self.reference_deadline = time.monotonic() + .25
try:
yield
finally:
self.reference_deadline = None
@staticmethod
def ident(value):
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
raise ValueError('Invalid SQL identifier')
return '`' + value + '`'
def exists(self, table, condition, args):
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
try:
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
return result
except Exception as error:
if error.args and error.args[0] in (1317, 3024):
raise self.defer('Source reference check exceeded its short query budget') from error
raise
def metadata_mentions(self, needles, excluded=None):
for table, columns in self.db.column_types.items():
for column, kind in columns.items():
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
continue
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
args = list(needles)
if excluded and table == excluded[0]:
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
args.append(excluded[1])
condition = ' AND '.join(conditions)
else:
condition = ' OR '.join(conditions)
if self.exists(table, condition, args):
return True
return False
def referenced(self, revision):
rid = revision['id']
for fk in self.db.fks:
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
return True
# Replacement, withdrawal and restore records include IDs without FKs.
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('revision_id'):
condition = self.ident(column) + '=%s'
args = [rid]
if table == 'source_revision':
condition += ' AND id<>%s'; args.append(rid)
if self.exists(table, condition, args):
return True
# A catalog source can be imported again. Retain its original even when
# its current source_path points to an external workbook rather than archive.
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
return True
return self.metadata_mentions([rid], ('source_revision', rid))
def file_referenced(self, path, exclude_revision=None):
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('_path'):
condition = self.ident(column) + '=%s'; args = [str(path)]
if table == 'source_revision' and exclude_revision:
condition += ' AND id<>%s'; args.append(exclude_revision)
if self.exists(table, condition, args):
return True
# Search both raw and JSON-escaped spelling, including saved snapshots.
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
def owned_file(self, revision):
path = Path(revision['archive_path'])
if not self.safe_file(path, self.boundary):
return None
relative = path.relative_to(self.boundary).parts
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
not re.fullmatch(r'\d{4}', relative[1]) or
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
relative[3] != revision['sha256'] + '.xlsx'):
return None
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
return None
return path
def safe_journals(self):
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
raise ValueError('Unsafe source file journal directory')
def drain(self, current, limit=5):
"""Resume only registered deletions after database commit; never scan stray files."""
self.safe_journals()
removed = size = 0
for record in sorted(self.journals.glob('*.json'))[:limit]:
if record.is_symlink():
continue
try:
value = json.loads(record.read_text())
except (OSError, ValueError):
continue
token = value.get('token', '')
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
value.get('database') != self.db.name or not isinstance(token, str) or
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
continue
revision = value.get('revision')
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
continue
path = Path(revision['archive_path'])
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
record.unlink() # An interrupted/rolled-back delete still owns its file.
continue
if not path.exists() and not path.is_symlink():
record.unlink(); continue
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
continue
if self.br.sha(path) != revision['sha256']:
continue
with self.db.frozen(current), self.quick_checks():
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
self.owned_file(revision) != path or
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
continue
size += path.stat().st_size
path.unlink(); removed += 1
record.unlink()
return removed, size
def collect(self, current, state, now, grace=1800, limit=3):
self.safe_journals()
seen = state.setdefault('sourceRevisions', {})
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
# All registered/current revisions remain readable until both ownership and
# dependency checks pass. Unknown states and missing/damaged files are kept.
revisions = self.db.rows('SELECT * FROM source_revision')
present = {row['id'] for row in revisions}
for stale in set(seen) - present:
seen.pop(stale, None)
deadline = time.monotonic() + 5
for revision in revisions:
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
break
rid = revision['id']
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
seen.pop(rid, None); continue
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
if seen.get(rid, {}).get('identity') != identity:
seen[rid] = {'identity': identity, 'since': now}
if now - seen[rid]['since'] < grace:
continue
path = self.owned_file(revision)
if path is None:
continue
stamp = self.fingerprint(path)
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
continue
# Durable intent first: a crash after row deletion must not leave an
# unregistered orphan. The journal is harmless if the row still exists.
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
'database': self.db.name, 'token': token, 'createdAt': now,
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
'fingerprint': stamp})
with self.db.frozen(current), self.quick_checks():
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
if (not live or live[0] != revision or self.referenced(revision) or
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
record.unlink(); seen.pop(rid, None); continue
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
files, size = self.drain(current)
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
return result