526 lines
30 KiB
Python
526 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Conservative, online retention for this Mac's accepted Quality Center runtime."""
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from datetime import datetime
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
import os
|
|
from pathlib import Path
|
|
import plistlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
import pymysql
|
|
from pymysql.cursors import DictCursor
|
|
|
|
LABEL = 'com.lyunlab.quality-center-retention'
|
|
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
|
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
|
|
BUILD_CHILDREN = ('defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
|
|
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
|
|
'fact_black', 'fact_bright')
|
|
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
|
|
'release_action', 'candidate_reconciliation')
|
|
MUTABLE_RELEASE_REFS = ('data_release', 'import_run', 'release_action', 'candidate_reconciliation')
|
|
|
|
|
|
def ident(value):
|
|
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
|
|
raise ValueError('Invalid SQL identifier')
|
|
return '`' + value + '`'
|
|
|
|
|
|
class Busy(RuntimeError):
|
|
pass
|
|
|
|
|
|
def safe_file(path, boundary):
|
|
path, boundary = Path(path), Path(boundary).absolute()
|
|
if not path.is_absolute():
|
|
return False
|
|
if not path.is_relative_to(boundary) or not path.resolve().is_relative_to(boundary.resolve()):
|
|
return False
|
|
cursor = path
|
|
while cursor != boundary:
|
|
if cursor.is_symlink():
|
|
return False
|
|
cursor = cursor.parent
|
|
return not boundary.is_symlink() and path.is_file()
|
|
|
|
|
|
def fingerprint(path):
|
|
s = Path(path).stat()
|
|
return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns)
|
|
|
|
|
|
class Database:
|
|
def __init__(self, root, database=None):
|
|
self.root = Path(root).resolve()
|
|
self.config = json.loads((self.root / '.runtime/local-config.json').read_text())
|
|
self.name = database or self.config['mysqlDatabase']
|
|
ident(self.name)
|
|
endpoint = ({'host': self.config['mysqlHost'], 'port': int(self.config['mysqlPort'])}
|
|
if self.config.get('mysqlHost') else {'unix_socket': str(self.root / '.runtime/mysql.sock')})
|
|
self.conn = pymysql.connect(**endpoint, user='root',
|
|
password=self.config['rootPassword'], database=self.name, charset='utf8mb4',
|
|
autocommit=True, cursorclass=DictCursor, connect_timeout=3, read_timeout=10, write_timeout=5)
|
|
self.execute("SET SESSION time_zone='+00:00'")
|
|
self.execute('SET SESSION lock_wait_timeout=1')
|
|
self.execute('SET SESSION innodb_lock_wait_timeout=1')
|
|
rows = self.rows('SELECT TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA=%s', (self.name,))
|
|
self.columns = {}
|
|
self.column_types = {}
|
|
for row in rows:
|
|
self.columns.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['IS_NULLABLE'] == 'YES'
|
|
self.column_types.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['DATA_TYPE']
|
|
self.tables = sorted(self.columns)
|
|
self.fks = self.rows('SELECT TABLE_NAME,COLUMN_NAME,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME FROM information_schema.key_column_usage WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL', (self.name,))
|
|
self.schema_signature=hashlib.sha256(json.dumps({'columns':self.columns,'fks':sorted(self.fks,key=lambda r:json.dumps(r,sort_keys=True))},sort_keys=True).encode()).hexdigest()
|
|
self.longest_exclusion=0.0
|
|
if not {'current_release','data_release','release_member','transform_build','source_upload_draft'}.issubset(self.columns):
|
|
raise ValueError('Unexpected application schema')
|
|
|
|
def rows(self, sql, args=()):
|
|
with self.conn.cursor() as cursor:
|
|
cursor.execute(sql, args)
|
|
return list(cursor.fetchall())
|
|
|
|
def execute(self, sql, args=()):
|
|
with self.conn.cursor() as cursor:
|
|
return cursor.execute(sql, args)
|
|
|
|
def current(self):
|
|
rows = self.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'")
|
|
if len(rows) != 1 or not rows[0]['release_id']:
|
|
raise Busy('Current release not confirmed')
|
|
fields='status,complete' if 'complete' in self.columns['data_release'] else 'status'
|
|
release=self.rows('SELECT '+fields+' FROM data_release WHERE id=%s',(rows[0]['release_id'],))
|
|
if len(release)!=1 or release[0]['status']!='PUBLISHED' or release[0].get('complete',True) not in (True,1):
|
|
raise Busy('Current release is not a complete publication')
|
|
return rows[0]['release_id']
|
|
|
|
def busy(self):
|
|
for table in ('task','import_run','data_workflow_operation','analysis_turn','analysis_protocol_execution','quality_inspection_run'):
|
|
if table not in self.columns or 'status' not in self.columns[table]:
|
|
continue
|
|
sql = 'SELECT 1 FROM ' + ident(table) + ' WHERE (status IS NULL OR status NOT IN (' + ','.join(['%s']*len(TERMINAL)) + '))'
|
|
if table == 'task' and {'next_retry_at','retry_count','max_retries'}.issubset(self.columns[table]):
|
|
sql += " OR (status='FAILED' AND next_retry_at IS NOT NULL AND retry_count<max_retries)"
|
|
if self.rows(sql + ' LIMIT 1', TERMINAL):
|
|
return True
|
|
if 'analysis_report' in self.columns:
|
|
# Report rows keep QUEUED after handing work to analysis_turn. The
|
|
# linked turn, not this stored label, is the execution state.
|
|
for report in self.rows('SELECT status,turn_id FROM analysis_report'):
|
|
if report['turn_id']:
|
|
turns=self.rows('SELECT status FROM analysis_turn WHERE id=%s',(report['turn_id'],))
|
|
if not turns or turns[0]['status'] not in TERMINAL:return True
|
|
elif report['status'] not in (*TERMINAL,'BLOCKED'):
|
|
return True
|
|
if 'source_candidate' in self.columns:
|
|
return bool(self.rows("SELECT 1 FROM source_candidate WHERE status IN ('QUEUED','RUNNING','IMPORTING') LIMIT 1"))
|
|
return False
|
|
|
|
@contextmanager
|
|
def frozen(self, expected):
|
|
"""Short exclusion against writers; never START TRANSACTION after LOCK TABLES.
|
|
|
|
All aliases are intentionally avoided while these MySQL locks are held.
|
|
Busy jobs and a changed release cause rollback without deleting anything.
|
|
"""
|
|
if (self.root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
|
|
if self.busy():
|
|
raise Busy('An application job is active')
|
|
self.conn.autocommit(False)
|
|
locked = False
|
|
acquired=None
|
|
try:
|
|
self.execute('LOCK TABLES ' + ','.join(ident(t) + ' WRITE' for t in self.tables))
|
|
locked = True
|
|
acquired=time.monotonic()
|
|
if sorted(next(iter(r.values())) for r in self.rows('SHOW TABLES'))!=self.tables:
|
|
raise Busy('Application schema changed')
|
|
if self.current() != expected or self.busy():
|
|
raise Busy('The publication or job state changed')
|
|
yield
|
|
self.conn.commit()
|
|
except BaseException as error:
|
|
self.conn.rollback()
|
|
if isinstance(error,pymysql.err.OperationalError) and error.args[0] in (1205,3572):
|
|
raise Busy('Application tables are in use') from error
|
|
raise
|
|
finally:
|
|
if locked:
|
|
self.execute('UNLOCK TABLES')
|
|
self.longest_exclusion=max(self.longest_exclusion,time.monotonic()-acquired)
|
|
self.conn.autocommit(True)
|
|
|
|
def protected_releases(self, current):
|
|
releases = self.rows('SELECT id,status,parent_release_id,expected_current_release_id FROM data_release')
|
|
protected = {current}
|
|
pending = {r['id'] for r in releases if r['status'] != 'PUBLISHED'}
|
|
protected.update(pending)
|
|
# Keep candidate baselines and all their ancestors: review/retry must work.
|
|
while True:
|
|
more = {r[c] for r in releases if r['id'] in pending for c in ('parent_release_id','expected_current_release_id') if r[c]}
|
|
if more.issubset(pending):
|
|
break
|
|
pending.update(more); protected.update(more)
|
|
for fk in self.fks:
|
|
t, c = fk['TABLE_NAME'], fk['COLUMN_NAME']
|
|
if fk['REFERENCED_TABLE_NAME'] != 'data_release' or t in MUTABLE_RELEASE_REFS or t in OWNED_RELEASE:
|
|
continue
|
|
protected.update(r[c] for r in self.rows('SELECT DISTINCT ' + ident(c) + ' FROM ' + ident(t) + ' WHERE ' + ident(c) + ' IS NOT NULL'))
|
|
# This field is intentionally not a database FK, but is a real user reference.
|
|
if 'analysis_session' in self.columns:
|
|
protected.update(r['release_id'] for r in self.rows('SELECT DISTINCT release_id FROM analysis_session'))
|
|
if 'source_candidate' in self.columns and 'expected_current_release_id' in self.columns['source_candidate']:
|
|
protected.update(r['expected_current_release_id'] for r in self.rows("SELECT DISTINCT expected_current_release_id FROM source_candidate WHERE status IN ('DISCOVERED','QUEUED','RUNNING') AND expected_current_release_id IS NOT NULL"))
|
|
if 'source_change_request' in self.columns:
|
|
cols = [c for c in ('requested_from_release_id','result_release_id') if c in self.columns['source_change_request']]
|
|
for row in self.rows('SELECT ' + ','.join(map(ident, cols)) + " FROM source_change_request WHERE status IN ('REQUESTED','APPROVED')") if cols else []:
|
|
protected.update(v for v in row.values() if v)
|
|
if 'candidate_reconciliation' in self.columns:
|
|
for row in self.rows('SELECT release_id,baseline_release_id FROM candidate_reconciliation'):
|
|
if row['release_id'] in pending and row['baseline_release_id']:
|
|
protected.add(row['baseline_release_id'])
|
|
return releases, protected
|
|
|
|
def release_step(self, current, state, grace, now):
|
|
releases, protected = self.protected_releases(current)
|
|
seen = state.setdefault('releases', {})
|
|
eligible = []
|
|
ids = {r['id'] for r in releases}
|
|
for stale in set(seen) - ids:
|
|
seen.pop(stale)
|
|
for row in releases:
|
|
rid = row['id']
|
|
if rid in protected:
|
|
seen.pop(rid, None)
|
|
elif row['status'] == 'PUBLISHED':
|
|
seen.setdefault(rid, now)
|
|
if now - seen[rid] >= grace:
|
|
eligible.append(rid)
|
|
if not eligible:
|
|
return 0
|
|
rid = sorted(eligible, key=lambda x: seen[x])[0]
|
|
# Clear optional historical links only. Unknown/business references were
|
|
# included in protected above and are never cascaded away.
|
|
for fk in self.fks:
|
|
t,c = fk['TABLE_NAME'],fk['COLUMN_NAME']
|
|
if fk['REFERENCED_TABLE_NAME'] == 'data_release' and t in MUTABLE_RELEASE_REFS and self.columns[t][c]:
|
|
self.execute('UPDATE ' + ident(t) + ' SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
|
|
if 'data_workflow_operation' in self.columns:
|
|
for c in ('release_id','expected_current_release_id'):
|
|
if c in self.columns['data_workflow_operation']:
|
|
self.execute('UPDATE data_workflow_operation SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
|
|
for t in OWNED_RELEASE:
|
|
if t in self.columns:
|
|
self.execute('DELETE FROM ' + ident(t) + ' WHERE release_id=%s', (rid,))
|
|
self.execute('DELETE FROM data_release WHERE id=%s', (rid,))
|
|
seen.pop(rid, None)
|
|
return 1
|
|
|
|
def build_step(self, state, grace, now, batch):
|
|
referenced = {r['transform_build_id'] for r in self.rows('SELECT DISTINCT transform_build_id FROM release_member')}
|
|
builds = self.rows('SELECT id,status,build_key FROM transform_build')
|
|
seen = state.setdefault('builds', {})
|
|
ids = {r['id'] for r in builds}
|
|
for stale in set(seen) - ids:
|
|
seen.pop(stale)
|
|
for row in builds:
|
|
bid = row['id']
|
|
unknown_reference = any(self.rows('SELECT 1 FROM ' + ident(f['TABLE_NAME']) + ' WHERE ' + ident(f['COLUMN_NAME']) + '=%s LIMIT 1', (bid,))
|
|
for f in self.fks if f['REFERENCED_TABLE_NAME'] == 'transform_build' and f['TABLE_NAME'] not in (*BUILD_CHILDREN, 'release_member'))
|
|
if bid in referenced or unknown_reference or row['status'] not in ('COMPLETED','FAILED','CANCELLED','RETIRED'):
|
|
seen.pop(bid, None)
|
|
continue
|
|
seen.setdefault(bid, now)
|
|
if row['status'] != 'RETIRED' and now - seen[bid] < grace:
|
|
continue
|
|
if row['status'] != 'RETIRED':
|
|
# Free the unique reuse key before deleting chunks. A new import
|
|
# can build a fresh result, never reuse this partially emptied one.
|
|
tombstone = hashlib.sha256(('retired:'+bid+':'+row['build_key']).encode()).hexdigest()
|
|
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
|
|
for t in BUILD_CHILDREN:
|
|
if t in self.columns:
|
|
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
|
|
if count:
|
|
return count, 0
|
|
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
|
|
seen.pop(bid, None)
|
|
return 0, 1
|
|
return 0, 0
|
|
|
|
def file_referenced(self, path):
|
|
for table, cols in self.columns.items():
|
|
for col in cols:
|
|
if col.endswith('_path') and self.rows('SELECT 1 FROM ' + ident(table) + ' WHERE ' + ident(col) + '=%s LIMIT 1', (str(path),)):
|
|
return True
|
|
return False
|
|
|
|
def expired(self, limit=10):
|
|
return self.rows('SELECT * FROM source_upload_draft WHERE expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6)) ORDER BY expires_at LIMIT ' + str(int(limit)))
|
|
|
|
def promotion(self, draft):
|
|
if not draft.get('candidate_id') or not {'source_revision','source_document','source_candidate'}.issubset(self.columns):
|
|
return None
|
|
rows = self.rows("SELECT * FROM source_candidate WHERE id=%s AND status='IMPORTED'", (draft['candidate_id'],))
|
|
if not rows:
|
|
return None
|
|
c = rows[0]
|
|
docs = {d['id']:d for d in self.rows('SELECT id,source_slot FROM source_document')}
|
|
for rev in self.rows('SELECT source_document_id,archive_path,sha256,parser_config_hash,file_size FROM source_revision WHERE sha256=%s AND parser_config_hash=%s', (c['sha256'],c['parser_config_hash'])):
|
|
if docs.get(rev['source_document_id'],{}).get('source_slot') == c['source_slot']:
|
|
p=Path(rev['archive_path'])
|
|
if safe_file(p,self.root/'runtimedata/archive/source-revisions') and p.stat().st_size==rev['file_size']:
|
|
return c,rev,p
|
|
return None
|
|
|
|
def clean_owned_scratch(self, br):
|
|
import psutil
|
|
server=self.rows('SELECT @@server_uuid AS value')[0]['value']
|
|
directory=self.root/'.runtime/retention/scratch'
|
|
removed=0
|
|
for path in directory.glob('*.json'):
|
|
if path.is_symlink():continue
|
|
value=json.loads(path.read_text());name=value.get('database','')
|
|
if value.get('kind')!='quality-restore-check' or value.get('server')!=server or not value.get('created') or name==self.name or not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',name):continue
|
|
if time.time()-value['createdAt']<300:continue
|
|
alive=False
|
|
for pid_key,created_key in [('ownerPid','ownerCreated'),('childPid','childCreated')]:
|
|
if pid_key not in value:continue
|
|
try:
|
|
process=psutil.Process(value[pid_key])
|
|
if abs(process.create_time()-value[created_key])<.01 and process.is_running() and process.status()!=psutil.STATUS_ZOMBIE:alive=True
|
|
except psutil.NoSuchProcess:pass
|
|
except psutil.Error:alive=True
|
|
if alive or self.rows('SELECT 1 FROM information_schema.processlist WHERE DB=%s LIMIT 1',(name,)):continue
|
|
self.execute('DROP DATABASE IF EXISTS '+ident(name));path.unlink();removed+=1
|
|
return removed
|
|
|
|
|
|
def cleanup_drafts(db, current, state, now, br):
|
|
removed=0; bytes_removed=0
|
|
# Checksum expensive files outside the brief database exclusion window.
|
|
for draft in db.expired():
|
|
path=Path(draft['source_path']); boundary=db.root/'runtimedata/archive/upload-drafts'
|
|
info=None
|
|
if safe_file(path,boundary):
|
|
info=fingerprint(path)
|
|
if info[2]!=draft['file_size'] or br.sha(path)!=draft['sha256']:
|
|
continue
|
|
promotion=db.promotion(draft)
|
|
if promotion:
|
|
c,rev,target=promotion
|
|
target_stamp=fingerprint(target)
|
|
if br.sha(target)!=rev['sha256']:
|
|
continue
|
|
with db.frozen(current):
|
|
live=db.rows('SELECT * FROM source_upload_draft WHERE id=%s AND expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6))',(draft['id'],))
|
|
if not live or live[0]!=draft:
|
|
continue
|
|
if info and (not safe_file(path,boundary) or fingerprint(path)!=info):
|
|
continue
|
|
if promotion and safe_file(target,db.root/'runtimedata/archive/source-revisions') and fingerprint(target)==target_stamp:
|
|
# The WHERE identity prevents overwriting a source reconfigured
|
|
# between preflight hashing and acquiring the database exclusion.
|
|
db.execute("UPDATE source_candidate SET source_path=%s,archive_path=%s WHERE id=%s AND status='IMPORTED' AND source_path=%s AND sha256=%s AND parser_config_hash=%s",
|
|
(str(target),str(target),c['id'],c['source_path'],c['sha256'],c['parser_config_hash']))
|
|
db.execute('DELETE FROM source_upload_draft WHERE id=%s',(draft['id'],))
|
|
if info:
|
|
with db.frozen(current):
|
|
if safe_file(path,boundary) and fingerprint(path)==info and not db.file_referenced(path):
|
|
bytes_removed+=path.stat().st_size
|
|
path.unlink()
|
|
removed+=1
|
|
# Interrupted uploads can leave no database row at all. Their age and every
|
|
# direct path reference are rechecked under the same exclusion before unlink.
|
|
boundary=db.root/'runtimedata/archive/upload-drafts'
|
|
for path in list(boundary.glob('upload-*.xlsx'))[:100]:
|
|
if not safe_file(path,boundary) or now-path.stat().st_mtime<86400:
|
|
continue
|
|
info=fingerprint(path)
|
|
with db.frozen(current):
|
|
if fingerprint(path)==info and not db.file_referenced(path):
|
|
size=path.stat().st_size;path.unlink();bytes_removed+=size
|
|
return removed,bytes_removed
|
|
|
|
|
|
def run(root, max_batches=30, grace=1800):
|
|
root=Path(root).resolve();sys.path.append(str(root/'scripts'))
|
|
import backup_retention as br
|
|
import backup
|
|
import backup_verifier
|
|
import revision_retention
|
|
if (root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
|
|
readiness=urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
with readiness.open('http://127.0.0.1:5281/actuator/health',timeout=2) as response:
|
|
if json.load(response).get('status')!='UP':raise Busy('Backend is not ready')
|
|
except (OSError,ValueError) as error:raise Busy('Backend is not ready') from error
|
|
state_path=root/'.runtime/retention/state.json'
|
|
state=json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
|
|
with br.lock(root,timeout=0):
|
|
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
|
|
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
|
|
db=Database(root)
|
|
try:
|
|
policy_path=root/'.runtime/retention/config.json'
|
|
if policy_path.exists():
|
|
policy=json.loads(policy_path.read_text());accepted=br.accepted_backend(root)
|
|
if policy['schemaSignature']!=db.schema_signature or not accepted or policy['acceptedJarSha256']!=accepted[1]:
|
|
raise Busy('Application version changed; retention compatibility review required')
|
|
result['scratchRemoved']=db.clean_owned_scratch(br)
|
|
current=db.current()
|
|
if db.busy():
|
|
raise Busy('An application job is active')
|
|
latest=br.latest(root)
|
|
# Recover a restore-verified candidate left between verification and
|
|
# pointer publication by a previous crash, without creating a duplicate.
|
|
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
|
|
candidates=[]
|
|
for folder in (root/'output/backups').iterdir() if (root/'output/backups').exists() else []:
|
|
candidate=br.verified(folder,checksums=False)
|
|
if candidate and candidate.get('queryEvidence',{}).get('releaseId')==current:candidates.append((candidate.get('createdAt',''),folder))
|
|
if candidates:
|
|
selected=max(candidates)[1]
|
|
if br.verified(selected):br.rotate(root,selected);latest=br.latest(root)
|
|
if latest:
|
|
state['lastBackupAt']=max(state.get('lastBackupAt',0),datetime.fromisoformat(latest[1]['createdAt']).timestamp())
|
|
due=not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current
|
|
due=due or time.time()-state.get('lastBackupAt',0)>86400
|
|
due=due or (state.get('dirtySince') and not state.get('builds') and time.time()-state['dirtySince']>300)
|
|
if due:
|
|
database_bytes=int(db.rows('SELECT COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) AS size FROM information_schema.tables WHERE TABLE_SCHEMA=%s',(db.name,))[0]['size'])
|
|
archive_bytes=sum(p.stat().st_size for p in (root/'runtimedata/archive').rglob('*') if p.is_file())
|
|
required_free=max(10*1024**3,3*database_bytes+2*archive_bytes+5*1024**3)
|
|
reserves = [root / '.runtime', root / 'output/backups']
|
|
for reserve in reserves: reserve.mkdir(parents=True, exist_ok=True)
|
|
if min(shutil.disk_usage(p).free for p in reserves)<required_free:raise Busy('Insufficient reserve for a verified backup; existing backup preserved')
|
|
if state.get('lastBackupAttemptRelease')==current and time.time()-state.get('lastBackupAttempt',0)<3600:
|
|
raise Busy('Waiting before retrying a failed backup')
|
|
state['lastBackupAttempt']=time.time();state['lastBackupAttemptRelease']=current;br.atomic(state_path,state)
|
|
logging.info('creating consistent backup; previous verified backup remains available')
|
|
before=br.backup_ids(root)
|
|
try:
|
|
folder=backup.create();backup.verify(folder)
|
|
except BaseException:
|
|
br.discard_failed_new(root,before)
|
|
raise
|
|
if db.current()!=current:
|
|
raise Busy('Publication changed while backing up; cleanup deferred')
|
|
latest=br.latest(root) # rotate is shared with manual/UI backup operations
|
|
if latest is None or latest[0]!=folder:
|
|
br.rotate(root,folder);latest=br.latest(root)
|
|
state['lastBackupAt']=time.time();state.pop('dirtySince',None)
|
|
br.atomic(state_path,state)
|
|
logging.info('isolated restore verified; latest backup published')
|
|
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
|
|
raise Busy('No verified backup for the current publication')
|
|
# Every data-deletion batch rechecks both job state and current pointer.
|
|
deadline=time.monotonic()+20
|
|
for _ in range(max_batches):
|
|
with db.frozen(current):
|
|
releases=db.release_step(current,state,grace,time.time())
|
|
rows,builds=db.build_step(state,grace,time.time(),1000)
|
|
result['releasesRemoved']+=releases;result['rowsRemoved']+=rows;result['buildsRemoved']+=builds
|
|
if not (releases or rows or builds) or time.monotonic()>deadline or db.longest_exclusion>.5:
|
|
break
|
|
time.sleep(.05)
|
|
drafts,size=cleanup_drafts(db,current,state,time.time(),br)
|
|
result['draftsRemoved']=drafts;result['bytesRemoved']=size
|
|
sources=revision_retention.Cleaner(db,br,safe_file,fingerprint,Busy)
|
|
result.update(sources.collect(current,state,time.time(),grace))
|
|
result['maxExclusiveSeconds']=round(db.longest_exclusion,4)
|
|
if any(result[k] for k in ('releasesRemoved','rowsRemoved','buildsRemoved','draftsRemoved','bytesRemoved','sourceRevisionsRemoved','sourceArchiveBytesRemoved')):
|
|
state.setdefault('dirtySince',time.time())
|
|
state['lastResult']=result;state.pop('lastError',None)
|
|
state['consecutiveFailures']=0
|
|
br.atomic(state_path,state)
|
|
return result
|
|
except Busy as error:
|
|
state['lastDeferred']={'time':time.time(),'reason':str(error)}
|
|
state['lastResult']=result;br.atomic(state_path,state)
|
|
raise
|
|
finally:
|
|
db.conn.close()
|
|
|
|
|
|
def install(root):
|
|
root=Path(root).resolve();sys.path.insert(0,str(root/'scripts'));import backup_retention as br
|
|
python=root/'.runtime/retention-env/bin/python3'
|
|
if not python.is_file():
|
|
raise ValueError('Private maintenance runtime missing')
|
|
files=[Path(__file__).resolve(),root/'scripts/backup_retention.py',root/'scripts/revision_retention.py']
|
|
db=Database(root)
|
|
try:
|
|
accepted=br.accepted_backend(root)
|
|
if not accepted:raise ValueError('Accepted backend runtime is not registered')
|
|
br.atomic(root/'.runtime/retention/config.json',{'schemaSignature':db.schema_signature,
|
|
'acceptedJarSha256':accepted[1],'retirementGraceSeconds':1800,'intervalSeconds':300,
|
|
'backupIntervalSeconds':86400,'draftExpiryGraceSeconds':1800,'sourceRevisionGraceSeconds':1800})
|
|
finally:db.conn.close()
|
|
digest=hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
|
|
version=SUPPORT/'retention'/digest;version.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
for p in files:
|
|
shutil.copy2(p,version/p.name);(version/p.name).chmod(0o600)
|
|
legacy=root/'.runtime/current-snapshot-20261109'
|
|
if legacy.is_dir() and not legacy.is_symlink() and (legacy/'manifest.json').is_file():
|
|
br.atomic(root/'.runtime/retention/legacy.json',{'path':'.runtime/current-snapshot-20261109','manifestSha256':br.sha(legacy/'manifest.json')})
|
|
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
plist=Path.home()/'Library/LaunchAgents'/(LABEL+'.plist')
|
|
value={'Label':LABEL,'ProgramArguments':[str(python),str(version/'retention.py'),'run','--root',str(root)],
|
|
'WorkingDirectory':str(root),'RunAtLoad':True,'StartInterval':300,'ProcessType':'Background','Nice':10,
|
|
'LowPriorityIO':True,'ExitTimeOut':300,'EnvironmentVariables':{'PYTHONDONTWRITEBYTECODE':'1'},
|
|
'StandardOutPath':str(logs/'retention-launchd.log'),'StandardErrorPath':str(logs/'retention-launchd-error.log')}
|
|
subprocess.run(['/bin/launchctl','bootout','gui/'+str(os.getuid())+'/'+LABEL],capture_output=True)
|
|
with plist.open('wb') as out:plistlib.dump(value,out)
|
|
plist.chmod(0o600)
|
|
subprocess.run(['/bin/launchctl','enable','gui/'+str(os.getuid())+'/'+LABEL],check=True)
|
|
for attempt in range(50):
|
|
loaded=subprocess.run(['/bin/launchctl','bootstrap','gui/'+str(os.getuid()),str(plist)],capture_output=True)
|
|
if loaded.returncode==0:break
|
|
time.sleep(.2)
|
|
else:raise RuntimeError('Maintenance registration did not complete')
|
|
print('Installed five-minute idle maintenance and daily/publication backup checks')
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action',choices=['run','install','status'])
|
|
parser.add_argument('--root',required=True)
|
|
args=parser.parse_args();root=Path(args.root).resolve();sys.path.append(str(root/'scripts'));import backup_retention as br
|
|
if args.action=='install':
|
|
with br.lock(root,timeout=0):install(root)
|
|
return
|
|
if args.action=='status':
|
|
p=root/'.runtime/retention/state.json';print(p.read_text() if p.exists() else '{}');return
|
|
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
handler=RotatingFileHandler(logs/'retention.log',maxBytes=2*1024*1024,backupCount=2)
|
|
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
|
logging.getLogger().addHandler(handler);logging.getLogger().setLevel(logging.INFO)
|
|
try:
|
|
result=run(root);logging.info('maintenance complete %s',json.dumps(result))
|
|
except (Busy,br.Busy) as error:
|
|
logging.info('maintenance deferred: %s',str(error))
|
|
except Exception as error:
|
|
p=root/'.runtime/retention/state.json';state=json.loads(p.read_text()) if p.exists() else {}
|
|
state['lastError']={'time':time.time(),'type':type(error).__name__};br.atomic(p,state)
|
|
state['consecutiveFailures']=min(1000,state.get('consecutiveFailures',0)+1);br.atomic(p,state)
|
|
logging.error('maintenance failed safely: %s',type(error).__name__)
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':main()
|