"""Reclaim only unreferenced, completed source revisions and their owned archives.""" import hashlib import json from contextlib import contextmanager from pathlib import Path import re import time import uuid # These are measured business values, not documents that refer to source IDs. VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'), ('fact_bright', 'raw_values'), ('fact_bright', 'components'), ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')} TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'} class Cleaner: def __init__(self, db, br, safe_file, fingerprint, defer): self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer self.root = db.root.resolve() self.boundary = self.root / 'runtimedata/archive/source-revisions' self.journals = self.root / '.runtime/retention/source-files' self.reference_deadline = None @contextmanager def quick_checks(self): self.reference_deadline = time.monotonic() + .25 try: yield finally: self.reference_deadline = None @staticmethod def ident(value): if not re.fullmatch(r'[A-Za-z0-9_]+', value): raise ValueError('Invalid SQL identifier') return '`' + value + '`' def exists(self, table, condition, args): if self.reference_deadline and time.monotonic() > self.reference_deadline: raise self.defer('Source reference checks exceeded the brief exclusion budget') try: result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' + self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args)) if self.reference_deadline and time.monotonic() > self.reference_deadline: raise self.defer('Source reference checks exceeded the brief exclusion budget') return result except Exception as error: if error.args and error.args[0] in (1317, 3024): raise self.defer('Source reference check exceeded its short query budget') from error raise def metadata_mentions(self, needles, excluded=None): for table, columns in self.db.column_types.items(): for column, kind in columns.items(): if kind not in TEXT_TYPES or (table, column) in VALUE_JSON: continue conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles] args = list(needles) if excluded and table == excluded[0]: conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s'] args.append(excluded[1]) condition = ' AND '.join(conditions) else: condition = ' OR '.join(conditions) if self.exists(table, condition, args): return True return False def referenced(self, revision): rid = revision['id'] for fk in self.db.fks: if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists( fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)): return True # Replacement, withdrawal and restore records include IDs without FKs. for table, columns in self.db.columns.items(): for column in columns: if column.endswith('revision_id'): condition = self.ident(column) + '=%s' args = [rid] if table == 'source_revision': condition += ' AND id<>%s'; args.append(rid) if self.exists(table, condition, args): return True # A catalog source can be imported again. Retain its original even when # its current source_path points to an external workbook rather than archive. if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']: if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)): return True return self.metadata_mentions([rid], ('source_revision', rid)) def file_referenced(self, path, exclude_revision=None): for table, columns in self.db.columns.items(): for column in columns: if column.endswith('_path'): condition = self.ident(column) + '=%s'; args = [str(path)] if table == 'source_revision' and exclude_revision: condition += ' AND id<>%s'; args.append(exclude_revision) if self.exists(table, condition, args): return True # Search both raw and JSON-escaped spelling, including saved snapshots. needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]])) return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None) def owned_file(self, revision): path = Path(revision['archive_path']) if not self.safe_file(path, self.boundary): return None relative = path.relative_to(self.boundary).parts if (len(relative) != 4 or relative[0] not in ('black', 'bright') or not re.fullmatch(r'\d{4}', relative[1]) or not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or relative[3] != revision['sha256'] + '.xlsx'): return None if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1: return None return path def safe_journals(self): for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals): if p.is_symlink() or not p.resolve().is_relative_to(self.root): raise ValueError('Unsafe source file journal directory') def drain(self, current, limit=5): """Resume only registered deletions after database commit; never scan stray files.""" self.safe_journals() removed = size = 0 for record in sorted(self.journals.glob('*.json'))[:limit]: if record.is_symlink(): continue try: value = json.loads(record.read_text()) except (OSError, ValueError): continue token = value.get('token', '') if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or value.get('database') != self.db.name or not isinstance(token, str) or not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name): continue revision = value.get('revision') if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)): continue path = Path(revision['archive_path']) if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)): record.unlink() # An interrupted/rolled-back delete still owns its file. continue if not path.exists() and not path.is_symlink(): record.unlink(); continue if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']: continue if self.br.sha(path) != revision['sha256']: continue with self.db.frozen(current), self.quick_checks(): if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)): continue size += path.stat().st_size path.unlink(); removed += 1 record.unlink() return removed, size def collect(self, current, state, now, grace=1800, limit=3): self.safe_journals() seen = state.setdefault('sourceRevisions', {}) result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0} # All registered/current revisions remain readable until both ownership and # dependency checks pass. Unknown states and missing/damaged files are kept. revisions = self.db.rows('SELECT * FROM source_revision') present = {row['id'] for row in revisions} for stale in set(seen) - present: seen.pop(stale, None) deadline = time.monotonic() + 5 for revision in revisions: if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit: break rid = revision['id'] if revision.get('status') != 'TRANSFORMED' or self.referenced(revision): seen.pop(rid, None); continue identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest() if seen.get(rid, {}).get('identity') != identity: seen[rid] = {'identity': identity, 'since': now} if now - seen[rid]['since'] < grace: continue path = self.owned_file(revision) if path is None: continue stamp = self.fingerprint(path) if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp: continue # Durable intent first: a crash after row deletion must not leave an # unregistered orphan. The journal is harmless if the row still exists. self.journals.mkdir(parents=True, exist_ok=True, mode=0o700) token = uuid.uuid4().hex; record = self.journals / (token + '.json') self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root), 'database': self.db.name, 'token': token, 'createdAt': now, 'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')}, 'fingerprint': stamp}) with self.db.frozen(current), self.quick_checks(): live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,)) if (not live or live[0] != revision or self.referenced(revision) or self.owned_file(revision) != path or self.fingerprint(path) != stamp): record.unlink(); seen.pop(rid, None); continue self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,)) result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None) files, size = self.drain(current) result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json')) return result