Files
quality_analyse/scripts/revision_retention.py
T

216 lines
11 KiB
Python

"""Reclaim only unreferenced, completed source revisions and their owned archives."""
import hashlib
import json
from contextlib import contextmanager
from pathlib import Path
import re
import time
import uuid
# These are measured business values, not documents that refer to source IDs.
BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
('source_filter_field', 'column_name'),
('source_filter_value', 'value')}
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
class Cleaner:
def __init__(self, db, br, safe_file, fingerprint, defer):
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
self.root = db.root.resolve()
self.boundary = self.root / 'runtimedata/archive/source-revisions'
self.journals = self.root / '.runtime/retention/source-files'
self.reference_deadline = None
@contextmanager
def quick_checks(self):
self.reference_deadline = time.monotonic() + .25
try:
yield
finally:
self.reference_deadline = None
@staticmethod
def ident(value):
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
raise ValueError('Invalid SQL identifier')
return '`' + value + '`'
def exists(self, table, condition, args):
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
try:
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
if self.reference_deadline and time.monotonic() > self.reference_deadline:
raise self.defer('Source reference checks exceeded the brief exclusion budget')
return result
except Exception as error:
if error.args and error.args[0] in (1317, 3024):
raise self.defer('Source reference check exceeded its short query budget') from error
raise
def metadata_mentions(self, needles, excluded=None):
for table, columns in self.db.column_types.items():
for column, kind in columns.items():
if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
continue
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
args = list(needles)
if excluded and table == excluded[0]:
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
args.append(excluded[1])
condition = ' AND '.join(conditions)
else:
condition = ' OR '.join(conditions)
if self.exists(table, condition, args):
return True
return False
def referenced(self, revision):
rid = revision['id']
for fk in self.db.fks:
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
return True
# Replacement, withdrawal and restore records include IDs without FKs.
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('revision_id'):
condition = self.ident(column) + '=%s'
args = [rid]
if table == 'source_revision':
condition += ' AND id<>%s'; args.append(rid)
if self.exists(table, condition, args):
return True
# A catalog source can be imported again. Retain its original even when
# its current source_path points to an external workbook rather than archive.
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
return True
return self.metadata_mentions([rid], ('source_revision', rid))
def file_referenced(self, path, exclude_revision=None):
for table, columns in self.db.columns.items():
for column in columns:
if column.endswith('_path'):
condition = self.ident(column) + '=%s'; args = [str(path)]
if table == 'source_revision' and exclude_revision:
condition += ' AND id<>%s'; args.append(exclude_revision)
if self.exists(table, condition, args):
return True
# Search both raw and JSON-escaped spelling, including saved snapshots.
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
def owned_file(self, revision):
path = Path(revision['archive_path'])
if not self.safe_file(path, self.boundary):
return None
relative = path.relative_to(self.boundary).parts
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
not re.fullmatch(r'\d{4}', relative[1]) or
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
relative[3] != revision['sha256'] + '.xlsx'):
return None
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
return None
return path
def safe_journals(self):
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
raise ValueError('Unsafe source file journal directory')
def drain(self, current, limit=5):
"""Resume only registered deletions after database commit; never scan stray files."""
self.safe_journals()
removed = size = 0
for record in sorted(self.journals.glob('*.json'))[:limit]:
if record.is_symlink():
continue
try:
value = json.loads(record.read_text())
except (OSError, ValueError):
continue
token = value.get('token', '')
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
value.get('database') != self.db.name or not isinstance(token, str) or
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
continue
revision = value.get('revision')
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
continue
path = Path(revision['archive_path'])
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
record.unlink() # An interrupted/rolled-back delete still owns its file.
continue
if not path.exists() and not path.is_symlink():
record.unlink(); continue
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
continue
if self.br.sha(path) != revision['sha256']:
continue
with self.db.frozen(current), self.quick_checks():
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
self.owned_file(revision) != path or
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
continue
size += path.stat().st_size
path.unlink(); removed += 1
record.unlink()
return removed, size
def collect(self, current, state, now, grace=1800, limit=3):
self.safe_journals()
seen = state.setdefault('sourceRevisions', {})
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
# All registered/current revisions remain readable until both ownership and
# dependency checks pass. Unknown states and missing/damaged files are kept.
revisions = self.db.rows('SELECT * FROM source_revision')
present = {row['id'] for row in revisions}
for stale in set(seen) - present:
seen.pop(stale, None)
deadline = time.monotonic() + 5
for revision in revisions:
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
break
rid = revision['id']
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
seen.pop(rid, None); continue
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
if seen.get(rid, {}).get('identity') != identity:
seen[rid] = {'identity': identity, 'since': now}
if now - seen[rid]['since'] < grace:
continue
path = self.owned_file(revision)
if path is None:
continue
stamp = self.fingerprint(path)
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
continue
# Durable intent first: a crash after row deletion must not leave an
# unregistered orphan. The journal is harmless if the row still exists.
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
'database': self.db.name, 'token': token, 'createdAt': now,
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
'fingerprint': stamp})
with self.db.frozen(current), self.quick_checks():
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
if (not live or live[0] != revision or self.referenced(revision) or
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
record.unlink(); seen.pop(rid, None); continue
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
files, size = self.drain(current)
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
return result