216 lines
11 KiB
Python
216 lines
11 KiB
Python
"""Reclaim only unreferenced, completed source revisions and their owned archives."""
|
|
import hashlib
|
|
import json
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import uuid
|
|
|
|
|
|
# These are measured business values, not documents that refer to source IDs.
|
|
BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
|
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
|
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
|
|
('source_filter_field', 'column_name'),
|
|
('source_filter_value', 'value')}
|
|
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
|
|
|
|
|
|
class Cleaner:
|
|
def __init__(self, db, br, safe_file, fingerprint, defer):
|
|
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
|
|
self.root = db.root.resolve()
|
|
self.boundary = self.root / 'runtimedata/archive/source-revisions'
|
|
self.journals = self.root / '.runtime/retention/source-files'
|
|
self.reference_deadline = None
|
|
|
|
@contextmanager
|
|
def quick_checks(self):
|
|
self.reference_deadline = time.monotonic() + .25
|
|
try:
|
|
yield
|
|
finally:
|
|
self.reference_deadline = None
|
|
|
|
@staticmethod
|
|
def ident(value):
|
|
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
|
|
raise ValueError('Invalid SQL identifier')
|
|
return '`' + value + '`'
|
|
|
|
def exists(self, table, condition, args):
|
|
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
|
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
|
try:
|
|
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
|
|
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
|
|
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
|
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
|
return result
|
|
except Exception as error:
|
|
if error.args and error.args[0] in (1317, 3024):
|
|
raise self.defer('Source reference check exceeded its short query budget') from error
|
|
raise
|
|
|
|
def metadata_mentions(self, needles, excluded=None):
|
|
for table, columns in self.db.column_types.items():
|
|
for column, kind in columns.items():
|
|
if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
|
|
continue
|
|
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
|
|
args = list(needles)
|
|
if excluded and table == excluded[0]:
|
|
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
|
|
args.append(excluded[1])
|
|
condition = ' AND '.join(conditions)
|
|
else:
|
|
condition = ' OR '.join(conditions)
|
|
if self.exists(table, condition, args):
|
|
return True
|
|
return False
|
|
|
|
def referenced(self, revision):
|
|
rid = revision['id']
|
|
for fk in self.db.fks:
|
|
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
|
|
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
|
|
return True
|
|
# Replacement, withdrawal and restore records include IDs without FKs.
|
|
for table, columns in self.db.columns.items():
|
|
for column in columns:
|
|
if column.endswith('revision_id'):
|
|
condition = self.ident(column) + '=%s'
|
|
args = [rid]
|
|
if table == 'source_revision':
|
|
condition += ' AND id<>%s'; args.append(rid)
|
|
if self.exists(table, condition, args):
|
|
return True
|
|
# A catalog source can be imported again. Retain its original even when
|
|
# its current source_path points to an external workbook rather than archive.
|
|
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
|
|
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
|
|
return True
|
|
return self.metadata_mentions([rid], ('source_revision', rid))
|
|
|
|
def file_referenced(self, path, exclude_revision=None):
|
|
for table, columns in self.db.columns.items():
|
|
for column in columns:
|
|
if column.endswith('_path'):
|
|
condition = self.ident(column) + '=%s'; args = [str(path)]
|
|
if table == 'source_revision' and exclude_revision:
|
|
condition += ' AND id<>%s'; args.append(exclude_revision)
|
|
if self.exists(table, condition, args):
|
|
return True
|
|
# Search both raw and JSON-escaped spelling, including saved snapshots.
|
|
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
|
|
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
|
|
|
|
def owned_file(self, revision):
|
|
path = Path(revision['archive_path'])
|
|
if not self.safe_file(path, self.boundary):
|
|
return None
|
|
relative = path.relative_to(self.boundary).parts
|
|
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
|
|
not re.fullmatch(r'\d{4}', relative[1]) or
|
|
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
|
|
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
|
|
relative[3] != revision['sha256'] + '.xlsx'):
|
|
return None
|
|
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
|
|
return None
|
|
return path
|
|
|
|
def safe_journals(self):
|
|
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
|
|
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
|
|
raise ValueError('Unsafe source file journal directory')
|
|
|
|
def drain(self, current, limit=5):
|
|
"""Resume only registered deletions after database commit; never scan stray files."""
|
|
self.safe_journals()
|
|
removed = size = 0
|
|
for record in sorted(self.journals.glob('*.json'))[:limit]:
|
|
if record.is_symlink():
|
|
continue
|
|
try:
|
|
value = json.loads(record.read_text())
|
|
except (OSError, ValueError):
|
|
continue
|
|
token = value.get('token', '')
|
|
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
|
|
value.get('database') != self.db.name or not isinstance(token, str) or
|
|
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
|
|
continue
|
|
revision = value.get('revision')
|
|
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
|
|
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
|
|
continue
|
|
path = Path(revision['archive_path'])
|
|
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
|
|
record.unlink() # An interrupted/rolled-back delete still owns its file.
|
|
continue
|
|
if not path.exists() and not path.is_symlink():
|
|
record.unlink(); continue
|
|
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
|
|
continue
|
|
if self.br.sha(path) != revision['sha256']:
|
|
continue
|
|
with self.db.frozen(current), self.quick_checks():
|
|
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
|
|
self.owned_file(revision) != path or
|
|
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
|
|
continue
|
|
size += path.stat().st_size
|
|
path.unlink(); removed += 1
|
|
record.unlink()
|
|
return removed, size
|
|
|
|
def collect(self, current, state, now, grace=1800, limit=3):
|
|
self.safe_journals()
|
|
seen = state.setdefault('sourceRevisions', {})
|
|
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
|
|
# All registered/current revisions remain readable until both ownership and
|
|
# dependency checks pass. Unknown states and missing/damaged files are kept.
|
|
revisions = self.db.rows('SELECT * FROM source_revision')
|
|
present = {row['id'] for row in revisions}
|
|
for stale in set(seen) - present:
|
|
seen.pop(stale, None)
|
|
deadline = time.monotonic() + 5
|
|
for revision in revisions:
|
|
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
|
|
break
|
|
rid = revision['id']
|
|
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
|
|
seen.pop(rid, None); continue
|
|
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
|
|
if seen.get(rid, {}).get('identity') != identity:
|
|
seen[rid] = {'identity': identity, 'since': now}
|
|
if now - seen[rid]['since'] < grace:
|
|
continue
|
|
path = self.owned_file(revision)
|
|
if path is None:
|
|
continue
|
|
stamp = self.fingerprint(path)
|
|
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
|
|
continue
|
|
# Durable intent first: a crash after row deletion must not leave an
|
|
# unregistered orphan. The journal is harmless if the row still exists.
|
|
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
|
|
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
|
|
'database': self.db.name, 'token': token, 'createdAt': now,
|
|
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
|
|
'fingerprint': stamp})
|
|
with self.db.frozen(current), self.quick_checks():
|
|
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
|
|
if (not live or live[0] != revision or self.referenced(revision) or
|
|
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
|
|
record.unlink(); seen.pop(rid, None); continue
|
|
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
|
|
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
|
|
files, size = self.drain(current)
|
|
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
|
|
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
|
|
return result
|