新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,213 @@
|
||||
"""Reclaim only unreferenced, completed source revisions and their owned archives."""
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
# These are measured business values, not documents that refer to source IDs.
|
||||
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
||||
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
|
||||
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
|
||||
|
||||
|
||||
class Cleaner:
|
||||
def __init__(self, db, br, safe_file, fingerprint, defer):
|
||||
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
|
||||
self.root = db.root.resolve()
|
||||
self.boundary = self.root / 'runtimedata/archive/source-revisions'
|
||||
self.journals = self.root / '.runtime/retention/source-files'
|
||||
self.reference_deadline = None
|
||||
|
||||
@contextmanager
|
||||
def quick_checks(self):
|
||||
self.reference_deadline = time.monotonic() + .25
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.reference_deadline = None
|
||||
|
||||
@staticmethod
|
||||
def ident(value):
|
||||
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
|
||||
raise ValueError('Invalid SQL identifier')
|
||||
return '`' + value + '`'
|
||||
|
||||
def exists(self, table, condition, args):
|
||||
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
||||
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
||||
try:
|
||||
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
|
||||
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
|
||||
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
||||
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
||||
return result
|
||||
except Exception as error:
|
||||
if error.args and error.args[0] in (1317, 3024):
|
||||
raise self.defer('Source reference check exceeded its short query budget') from error
|
||||
raise
|
||||
|
||||
def metadata_mentions(self, needles, excluded=None):
|
||||
for table, columns in self.db.column_types.items():
|
||||
for column, kind in columns.items():
|
||||
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
|
||||
continue
|
||||
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
|
||||
args = list(needles)
|
||||
if excluded and table == excluded[0]:
|
||||
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
|
||||
args.append(excluded[1])
|
||||
condition = ' AND '.join(conditions)
|
||||
else:
|
||||
condition = ' OR '.join(conditions)
|
||||
if self.exists(table, condition, args):
|
||||
return True
|
||||
return False
|
||||
|
||||
def referenced(self, revision):
|
||||
rid = revision['id']
|
||||
for fk in self.db.fks:
|
||||
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
|
||||
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
|
||||
return True
|
||||
# Replacement, withdrawal and restore records include IDs without FKs.
|
||||
for table, columns in self.db.columns.items():
|
||||
for column in columns:
|
||||
if column.endswith('revision_id'):
|
||||
condition = self.ident(column) + '=%s'
|
||||
args = [rid]
|
||||
if table == 'source_revision':
|
||||
condition += ' AND id<>%s'; args.append(rid)
|
||||
if self.exists(table, condition, args):
|
||||
return True
|
||||
# A catalog source can be imported again. Retain its original even when
|
||||
# its current source_path points to an external workbook rather than archive.
|
||||
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
|
||||
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
|
||||
return True
|
||||
return self.metadata_mentions([rid], ('source_revision', rid))
|
||||
|
||||
def file_referenced(self, path, exclude_revision=None):
|
||||
for table, columns in self.db.columns.items():
|
||||
for column in columns:
|
||||
if column.endswith('_path'):
|
||||
condition = self.ident(column) + '=%s'; args = [str(path)]
|
||||
if table == 'source_revision' and exclude_revision:
|
||||
condition += ' AND id<>%s'; args.append(exclude_revision)
|
||||
if self.exists(table, condition, args):
|
||||
return True
|
||||
# Search both raw and JSON-escaped spelling, including saved snapshots.
|
||||
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
|
||||
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
|
||||
|
||||
def owned_file(self, revision):
|
||||
path = Path(revision['archive_path'])
|
||||
if not self.safe_file(path, self.boundary):
|
||||
return None
|
||||
relative = path.relative_to(self.boundary).parts
|
||||
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
|
||||
not re.fullmatch(r'\d{4}', relative[1]) or
|
||||
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
|
||||
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
|
||||
relative[3] != revision['sha256'] + '.xlsx'):
|
||||
return None
|
||||
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
|
||||
return None
|
||||
return path
|
||||
|
||||
def safe_journals(self):
|
||||
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
|
||||
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
|
||||
raise ValueError('Unsafe source file journal directory')
|
||||
|
||||
def drain(self, current, limit=5):
|
||||
"""Resume only registered deletions after database commit; never scan stray files."""
|
||||
self.safe_journals()
|
||||
removed = size = 0
|
||||
for record in sorted(self.journals.glob('*.json'))[:limit]:
|
||||
if record.is_symlink():
|
||||
continue
|
||||
try:
|
||||
value = json.loads(record.read_text())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
token = value.get('token', '')
|
||||
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
|
||||
value.get('database') != self.db.name or not isinstance(token, str) or
|
||||
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
|
||||
continue
|
||||
revision = value.get('revision')
|
||||
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
|
||||
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
|
||||
continue
|
||||
path = Path(revision['archive_path'])
|
||||
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
|
||||
record.unlink() # An interrupted/rolled-back delete still owns its file.
|
||||
continue
|
||||
if not path.exists() and not path.is_symlink():
|
||||
record.unlink(); continue
|
||||
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
|
||||
continue
|
||||
if self.br.sha(path) != revision['sha256']:
|
||||
continue
|
||||
with self.db.frozen(current), self.quick_checks():
|
||||
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
|
||||
self.owned_file(revision) != path or
|
||||
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
|
||||
continue
|
||||
size += path.stat().st_size
|
||||
path.unlink(); removed += 1
|
||||
record.unlink()
|
||||
return removed, size
|
||||
|
||||
def collect(self, current, state, now, grace=1800, limit=3):
|
||||
self.safe_journals()
|
||||
seen = state.setdefault('sourceRevisions', {})
|
||||
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
|
||||
# All registered/current revisions remain readable until both ownership and
|
||||
# dependency checks pass. Unknown states and missing/damaged files are kept.
|
||||
revisions = self.db.rows('SELECT * FROM source_revision')
|
||||
present = {row['id'] for row in revisions}
|
||||
for stale in set(seen) - present:
|
||||
seen.pop(stale, None)
|
||||
deadline = time.monotonic() + 5
|
||||
for revision in revisions:
|
||||
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
|
||||
break
|
||||
rid = revision['id']
|
||||
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
|
||||
seen.pop(rid, None); continue
|
||||
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
|
||||
if seen.get(rid, {}).get('identity') != identity:
|
||||
seen[rid] = {'identity': identity, 'since': now}
|
||||
if now - seen[rid]['since'] < grace:
|
||||
continue
|
||||
path = self.owned_file(revision)
|
||||
if path is None:
|
||||
continue
|
||||
stamp = self.fingerprint(path)
|
||||
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
|
||||
continue
|
||||
# Durable intent first: a crash after row deletion must not leave an
|
||||
# unregistered orphan. The journal is harmless if the row still exists.
|
||||
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
|
||||
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
|
||||
'database': self.db.name, 'token': token, 'createdAt': now,
|
||||
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
|
||||
'fingerprint': stamp})
|
||||
with self.db.frozen(current), self.quick_checks():
|
||||
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
|
||||
if (not live or live[0] != revision or self.referenced(revision) or
|
||||
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
|
||||
record.unlink(); seen.pop(rid, None); continue
|
||||
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
|
||||
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
|
||||
files, size = self.drain(current)
|
||||
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
|
||||
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
|
||||
return result
|
||||
Reference in New Issue
Block a user