Files

545 lines
31 KiB
Python

#!/usr/bin/env python3
"""Conservative, online retention for this Mac's accepted Quality Center runtime."""
import argparse
from contextlib import contextmanager
from datetime import datetime
import hashlib
import importlib
import json
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import plistlib
import re
import shutil
import subprocess
import sys
import time
import urllib.request
import pymysql
from pymysql.cursors import DictCursor
LABEL = 'com.lyunlab.quality-center-retention'
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
'fact_black', 'fact_bright')
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
'release_action', 'candidate_reconciliation')
MUTABLE_RELEASE_REFS = ('data_release', 'import_run', 'release_action', 'candidate_reconciliation')
def ident(value):
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
raise ValueError('Invalid SQL identifier')
return '`' + value + '`'
class Busy(RuntimeError):
pass
def safe_file(path, boundary):
path, boundary = Path(path), Path(boundary).absolute()
if not path.is_absolute():
return False
if not path.is_relative_to(boundary) or not path.resolve().is_relative_to(boundary.resolve()):
return False
cursor = path
while cursor != boundary:
if cursor.is_symlink():
return False
cursor = cursor.parent
return not boundary.is_symlink() and path.is_file()
def fingerprint(path):
s = Path(path).stat()
return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns)
class Database:
def __init__(self, root, database=None):
self.root = Path(root).resolve()
self.config = json.loads((self.root / '.runtime/local-config.json').read_text())
self.name = database or self.config['mysqlDatabase']
ident(self.name)
endpoint = ({'host': self.config['mysqlHost'], 'port': int(self.config['mysqlPort'])}
if self.config.get('mysqlHost') else {'unix_socket': str(self.root / '.runtime/mysql.sock')})
self.conn = pymysql.connect(**endpoint, user='root',
password=self.config['rootPassword'], database=self.name, charset='utf8mb4',
autocommit=True, cursorclass=DictCursor, connect_timeout=3, read_timeout=10, write_timeout=5)
self.execute("SET SESSION time_zone='+00:00'")
self.execute('SET SESSION lock_wait_timeout=1')
self.execute('SET SESSION innodb_lock_wait_timeout=1')
rows = self.rows('SELECT TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA=%s', (self.name,))
self.columns = {}
self.column_types = {}
for row in rows:
self.columns.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['IS_NULLABLE'] == 'YES'
self.column_types.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['DATA_TYPE']
self.tables = sorted(self.columns)
self.fks = self.rows('SELECT TABLE_NAME,COLUMN_NAME,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME FROM information_schema.key_column_usage WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL', (self.name,))
self.schema_signature=hashlib.sha256(json.dumps({'columns':self.columns,'fks':sorted(self.fks,key=lambda r:json.dumps(r,sort_keys=True))},sort_keys=True).encode()).hexdigest()
self.longest_exclusion=0.0
if not {'current_release','data_release','release_member','transform_build','source_upload_draft'}.issubset(self.columns):
raise ValueError('Unexpected application schema')
def rows(self, sql, args=()):
with self.conn.cursor() as cursor:
cursor.execute(sql, args)
return list(cursor.fetchall())
def execute(self, sql, args=()):
with self.conn.cursor() as cursor:
return cursor.execute(sql, args)
def current(self):
rows = self.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'")
if len(rows) != 1 or not rows[0]['release_id']:
raise Busy('Current release not confirmed')
fields='status,complete' if 'complete' in self.columns['data_release'] else 'status'
release=self.rows('SELECT '+fields+' FROM data_release WHERE id=%s',(rows[0]['release_id'],))
if len(release)!=1 or release[0]['status']!='PUBLISHED' or release[0].get('complete',True) not in (True,1):
raise Busy('Current release is not a complete publication')
return rows[0]['release_id']
def busy(self):
for table in ('task','import_run','data_workflow_operation','analysis_turn','analysis_protocol_execution','quality_inspection_run'):
if table not in self.columns or 'status' not in self.columns[table]:
continue
sql = 'SELECT 1 FROM ' + ident(table) + ' WHERE (status IS NULL OR status NOT IN (' + ','.join(['%s']*len(TERMINAL)) + '))'
if table == 'task' and {'next_retry_at','retry_count','max_retries'}.issubset(self.columns[table]):
sql += " OR (status='FAILED' AND next_retry_at IS NOT NULL AND retry_count<max_retries)"
if self.rows(sql + ' LIMIT 1', TERMINAL):
return True
if 'analysis_report' in self.columns:
# Report rows keep QUEUED after handing work to analysis_turn. The
# linked turn, not this stored label, is the execution state.
for report in self.rows('SELECT status,turn_id FROM analysis_report'):
if report['turn_id']:
turns=self.rows('SELECT status FROM analysis_turn WHERE id=%s',(report['turn_id'],))
if not turns or turns[0]['status'] not in TERMINAL:return True
elif report['status'] not in (*TERMINAL,'BLOCKED'):
return True
if 'source_candidate' in self.columns:
return bool(self.rows("SELECT 1 FROM source_candidate WHERE status IN ('QUEUED','RUNNING','IMPORTING') LIMIT 1"))
return False
@contextmanager
def frozen(self, expected):
"""Short exclusion against writers; never START TRANSACTION after LOCK TABLES.
All aliases are intentionally avoided while these MySQL locks are held.
Busy jobs and a changed release cause rollback without deleting anything.
"""
if (self.root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
if self.busy():
raise Busy('An application job is active')
self.conn.autocommit(False)
locked = False
acquired=None
try:
self.execute('LOCK TABLES ' + ','.join(ident(t) + ' WRITE' for t in self.tables))
locked = True
acquired=time.monotonic()
if sorted(next(iter(r.values())) for r in self.rows('SHOW TABLES'))!=self.tables:
raise Busy('Application schema changed')
if self.current() != expected or self.busy():
raise Busy('The publication or job state changed')
yield
self.conn.commit()
except BaseException as error:
self.conn.rollback()
if isinstance(error,pymysql.err.OperationalError) and error.args[0] in (1205,3572):
raise Busy('Application tables are in use') from error
raise
finally:
if locked:
self.execute('UNLOCK TABLES')
self.longest_exclusion=max(self.longest_exclusion,time.monotonic()-acquired)
self.conn.autocommit(True)
def protected_releases(self, current):
releases = self.rows('SELECT id,status,parent_release_id,expected_current_release_id FROM data_release')
protected = {current}
pending = {r['id'] for r in releases if r['status'] != 'PUBLISHED'}
protected.update(pending)
# Keep candidate baselines and all their ancestors: review/retry must work.
while True:
more = {r[c] for r in releases if r['id'] in pending for c in ('parent_release_id','expected_current_release_id') if r[c]}
if more.issubset(pending):
break
pending.update(more); protected.update(more)
for fk in self.fks:
t, c = fk['TABLE_NAME'], fk['COLUMN_NAME']
if fk['REFERENCED_TABLE_NAME'] != 'data_release' or t in MUTABLE_RELEASE_REFS or t in OWNED_RELEASE:
continue
protected.update(r[c] for r in self.rows('SELECT DISTINCT ' + ident(c) + ' FROM ' + ident(t) + ' WHERE ' + ident(c) + ' IS NOT NULL'))
# This field is intentionally not a database FK, but is a real user reference.
if 'analysis_session' in self.columns:
protected.update(r['release_id'] for r in self.rows('SELECT DISTINCT release_id FROM analysis_session'))
if 'source_candidate' in self.columns and 'expected_current_release_id' in self.columns['source_candidate']:
protected.update(r['expected_current_release_id'] for r in self.rows("SELECT DISTINCT expected_current_release_id FROM source_candidate WHERE status IN ('DISCOVERED','QUEUED','RUNNING') AND expected_current_release_id IS NOT NULL"))
if 'source_change_request' in self.columns:
cols = [c for c in ('requested_from_release_id','result_release_id') if c in self.columns['source_change_request']]
for row in self.rows('SELECT ' + ','.join(map(ident, cols)) + " FROM source_change_request WHERE status IN ('REQUESTED','APPROVED')") if cols else []:
protected.update(v for v in row.values() if v)
if 'candidate_reconciliation' in self.columns:
for row in self.rows('SELECT release_id,baseline_release_id FROM candidate_reconciliation'):
if row['release_id'] in pending and row['baseline_release_id']:
protected.add(row['baseline_release_id'])
return releases, protected
def release_step(self, current, state, grace, now):
releases, protected = self.protected_releases(current)
seen = state.setdefault('releases', {})
eligible = []
ids = {r['id'] for r in releases}
for stale in set(seen) - ids:
seen.pop(stale)
for row in releases:
rid = row['id']
if rid in protected:
seen.pop(rid, None)
elif row['status'] == 'PUBLISHED':
seen.setdefault(rid, now)
if now - seen[rid] >= grace:
eligible.append(rid)
if not eligible:
return 0
rid = sorted(eligible, key=lambda x: seen[x])[0]
# Clear optional historical links only. Unknown/business references were
# included in protected above and are never cascaded away.
for fk in self.fks:
t,c = fk['TABLE_NAME'],fk['COLUMN_NAME']
if fk['REFERENCED_TABLE_NAME'] == 'data_release' and t in MUTABLE_RELEASE_REFS and self.columns[t][c]:
self.execute('UPDATE ' + ident(t) + ' SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
if 'data_workflow_operation' in self.columns:
for c in ('release_id','expected_current_release_id'):
if c in self.columns['data_workflow_operation']:
self.execute('UPDATE data_workflow_operation SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
for t in OWNED_RELEASE:
if t in self.columns:
self.execute('DELETE FROM ' + ident(t) + ' WHERE release_id=%s', (rid,))
self.execute('DELETE FROM data_release WHERE id=%s', (rid,))
seen.pop(rid, None)
return 1
def build_step(self, state, grace, now, batch):
referenced = {r['transform_build_id'] for r in self.rows('SELECT DISTINCT transform_build_id FROM release_member')}
builds = self.rows('SELECT id,status,build_key FROM transform_build')
seen = state.setdefault('builds', {})
ids = {r['id'] for r in builds}
for stale in set(seen) - ids:
seen.pop(stale)
for row in builds:
bid = row['id']
unknown_reference = any(self.rows('SELECT 1 FROM ' + ident(f['TABLE_NAME']) + ' WHERE ' + ident(f['COLUMN_NAME']) + '=%s LIMIT 1', (bid,))
for f in self.fks if f['REFERENCED_TABLE_NAME'] == 'transform_build' and f['TABLE_NAME'] not in (*BUILD_CHILDREN, 'release_member'))
if bid in referenced or unknown_reference or row['status'] not in ('COMPLETED','FAILED','CANCELLED','RETIRED'):
seen.pop(bid, None)
continue
seen.setdefault(bid, now)
if row['status'] != 'RETIRED' and now - seen[bid] < grace:
continue
if row['status'] != 'RETIRED':
# Free the unique reuse key before deleting chunks. A new import
# can build a fresh result, never reuse this partially emptied one.
tombstone = hashlib.sha256(('retired:'+bid+':'+row['build_key']).encode()).hexdigest()
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
for t in BUILD_CHILDREN:
if t in self.columns:
if t == 'source_filter_member':
# Members have no build_id; drain each owned field in
# order without aliases because the exclusion uses LOCK TABLES.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
elif t == 'source_filter_value':
# Values are likewise owned through fields and are deleted
# only after all field memberships have been drained.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
else:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if count:
return count, 0
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
seen.pop(bid, None)
return 0, 1
return 0, 0
def file_referenced(self, path):
for table, cols in self.columns.items():
for col in cols:
if col.endswith('_path') and self.rows('SELECT 1 FROM ' + ident(table) + ' WHERE ' + ident(col) + '=%s LIMIT 1', (str(path),)):
return True
return False
def expired(self, limit=10):
return self.rows('SELECT * FROM source_upload_draft WHERE expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6)) ORDER BY expires_at LIMIT ' + str(int(limit)))
def promotion(self, draft):
if not draft.get('candidate_id') or not {'source_revision','source_document','source_candidate'}.issubset(self.columns):
return None
rows = self.rows("SELECT * FROM source_candidate WHERE id=%s AND status='IMPORTED'", (draft['candidate_id'],))
if not rows:
return None
c = rows[0]
docs = {d['id']:d for d in self.rows('SELECT id,source_slot FROM source_document')}
for rev in self.rows('SELECT source_document_id,archive_path,sha256,parser_config_hash,file_size FROM source_revision WHERE sha256=%s AND parser_config_hash=%s', (c['sha256'],c['parser_config_hash'])):
if docs.get(rev['source_document_id'],{}).get('source_slot') == c['source_slot']:
p=Path(rev['archive_path'])
if safe_file(p,self.root/'runtimedata/archive/source-revisions') and p.stat().st_size==rev['file_size']:
return c,rev,p
return None
def clean_owned_scratch(self, br):
import psutil
server=self.rows('SELECT @@server_uuid AS value')[0]['value']
directory=self.root/'.runtime/retention/scratch'
removed=0
for path in directory.glob('*.json'):
if path.is_symlink():continue
value=json.loads(path.read_text());name=value.get('database','')
if value.get('kind')!='quality-restore-check' or value.get('server')!=server or not value.get('created') or name==self.name or not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',name):continue
if time.time()-value['createdAt']<300:continue
alive=False
for pid_key,created_key in [('ownerPid','ownerCreated'),('childPid','childCreated')]:
if pid_key not in value:continue
try:
process=psutil.Process(value[pid_key])
if abs(process.create_time()-value[created_key])<.01 and process.is_running() and process.status()!=psutil.STATUS_ZOMBIE:alive=True
except psutil.NoSuchProcess:pass
except psutil.Error:alive=True
if alive or self.rows('SELECT 1 FROM information_schema.processlist WHERE DB=%s LIMIT 1',(name,)):continue
self.execute('DROP DATABASE IF EXISTS '+ident(name));path.unlink();removed+=1
return removed
def cleanup_drafts(db, current, state, now, br):
removed=0; bytes_removed=0
# Checksum expensive files outside the brief database exclusion window.
for draft in db.expired():
path=Path(draft['source_path']); boundary=db.root/'runtimedata/archive/upload-drafts'
info=None
if safe_file(path,boundary):
info=fingerprint(path)
if info[2]!=draft['file_size'] or br.sha(path)!=draft['sha256']:
continue
promotion=db.promotion(draft)
if promotion:
c,rev,target=promotion
target_stamp=fingerprint(target)
if br.sha(target)!=rev['sha256']:
continue
with db.frozen(current):
live=db.rows('SELECT * FROM source_upload_draft WHERE id=%s AND expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6))',(draft['id'],))
if not live or live[0]!=draft:
continue
if info and (not safe_file(path,boundary) or fingerprint(path)!=info):
continue
if promotion and safe_file(target,db.root/'runtimedata/archive/source-revisions') and fingerprint(target)==target_stamp:
# The WHERE identity prevents overwriting a source reconfigured
# between preflight hashing and acquiring the database exclusion.
db.execute("UPDATE source_candidate SET source_path=%s,archive_path=%s WHERE id=%s AND status='IMPORTED' AND source_path=%s AND sha256=%s AND parser_config_hash=%s",
(str(target),str(target),c['id'],c['source_path'],c['sha256'],c['parser_config_hash']))
db.execute('DELETE FROM source_upload_draft WHERE id=%s',(draft['id'],))
if info:
with db.frozen(current):
if safe_file(path,boundary) and fingerprint(path)==info and not db.file_referenced(path):
bytes_removed+=path.stat().st_size
path.unlink()
removed+=1
# Interrupted uploads can leave no database row at all. Their age and every
# direct path reference are rechecked under the same exclusion before unlink.
boundary=db.root/'runtimedata/archive/upload-drafts'
for path in list(boundary.glob('upload-*.xlsx'))[:100]:
if not safe_file(path,boundary) or now-path.stat().st_mtime<86400:
continue
info=fingerprint(path)
with db.frozen(current):
if fingerprint(path)==info and not db.file_referenced(path):
size=path.stat().st_size;path.unlink();bytes_removed+=size
return removed,bytes_removed
def run(root, max_batches=30, grace=1800):
root=Path(root).resolve();sys.path.append(str(root/'scripts'))
import backup_retention as br
import backup
import backup_verifier
import revision_retention
if (root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
readiness=urllib.request.build_opener(urllib.request.ProxyHandler({}))
try:
with readiness.open('http://127.0.0.1:5281/actuator/health',timeout=2) as response:
if json.load(response).get('status')!='UP':raise Busy('Backend is not ready')
except (OSError,ValueError) as error:raise Busy('Backend is not ready') from error
state_path=root/'.runtime/retention/state.json'
state=json.loads(state_path.read_text()) if state_path.exists() else {}
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
with br.lock(root,timeout=0):
result.update(br.clean_incomplete(root))
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
db=Database(root)
try:
policy_path=root/'.runtime/retention/config.json'
if policy_path.exists():
policy=json.loads(policy_path.read_text());accepted=br.accepted_backend(root)
if policy['schemaSignature']!=db.schema_signature or not accepted or policy['acceptedJarSha256']!=accepted[1]:
raise Busy('Application version changed; retention compatibility review required')
result['scratchRemoved']=db.clean_owned_scratch(br)
current=db.current()
if db.busy():
raise Busy('An application job is active')
latest=br.latest(root)
# Recover a restore-verified candidate left between verification and
# pointer publication by a previous crash, without creating a duplicate.
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
candidates=[]
for folder in (root/'output/backups').iterdir() if (root/'output/backups').exists() else []:
candidate=br.verified(folder,checksums=False)
if candidate and candidate.get('queryEvidence',{}).get('releaseId')==current:candidates.append((candidate.get('createdAt',''),folder))
if candidates:
selected=max(candidates)[1]
if br.verified(selected):br.rotate(root,selected);latest=br.latest(root)
if latest:
state['lastBackupAt']=max(state.get('lastBackupAt',0),datetime.fromisoformat(latest[1]['createdAt']).timestamp())
due=not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current
due=due or time.time()-state.get('lastBackupAt',0)>86400
due=due or (state.get('dirtySince') and not state.get('builds') and time.time()-state['dirtySince']>300)
if due:
database_bytes=int(db.rows('SELECT COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) AS size FROM information_schema.tables WHERE TABLE_SCHEMA=%s',(db.name,))[0]['size'])
archive_bytes=sum(p.stat().st_size for p in (root/'runtimedata/archive').rglob('*') if p.is_file())
required_free=max(10*1024**3,3*database_bytes+2*archive_bytes+5*1024**3)
reserves = [root / '.runtime', root / 'output/backups']
for reserve in reserves: reserve.mkdir(parents=True, exist_ok=True)
if min(shutil.disk_usage(p).free for p in reserves)<required_free:raise Busy('Insufficient reserve for a verified backup; existing backup preserved')
if state.get('lastBackupAttemptRelease')==current and time.time()-state.get('lastBackupAttempt',0)<3600:
raise Busy('Waiting before retrying a failed backup')
state['lastBackupAttempt']=time.time();state['lastBackupAttemptRelease']=current;br.atomic(state_path,state)
logging.info('creating consistent backup; previous verified backup remains available')
before=br.backup_ids(root)
try:
folder=backup.create();backup.verify(folder)
except BaseException:
br.discard_failed_new(root,before)
raise
if db.current()!=current:
raise Busy('Publication changed while backing up; cleanup deferred')
latest=br.latest(root) # rotate is shared with manual/UI backup operations
if latest is None or latest[0]!=folder:
br.rotate(root,folder);latest=br.latest(root)
state['lastBackupAt']=time.time();state.pop('dirtySince',None)
br.atomic(state_path,state)
logging.info('isolated restore verified; latest backup published')
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
raise Busy('No verified backup for the current publication')
# Every data-deletion batch rechecks both job state and current pointer.
deadline=time.monotonic()+20
for _ in range(max_batches):
with db.frozen(current):
releases=db.release_step(current,state,grace,time.time())
rows,builds=db.build_step(state,grace,time.time(),1000)
result['releasesRemoved']+=releases;result['rowsRemoved']+=rows;result['buildsRemoved']+=builds
if not (releases or rows or builds) or time.monotonic()>deadline or db.longest_exclusion>.5:
break
time.sleep(.05)
drafts,size=cleanup_drafts(db,current,state,time.time(),br)
result['draftsRemoved']=drafts;result['bytesRemoved']=size
sources=revision_retention.Cleaner(db,br,safe_file,fingerprint,Busy)
result.update(sources.collect(current,state,time.time(),grace))
result['maxExclusiveSeconds']=round(db.longest_exclusion,4)
if any(result[k] for k in ('releasesRemoved','rowsRemoved','buildsRemoved','draftsRemoved','bytesRemoved','sourceRevisionsRemoved','sourceArchiveBytesRemoved')):
state.setdefault('dirtySince',time.time())
state['lastResult']=result;state.pop('lastError',None)
state['consecutiveFailures']=0
br.atomic(state_path,state)
return result
except Busy as error:
state['lastDeferred']={'time':time.time(),'reason':str(error)}
state['lastResult']=result;br.atomic(state_path,state)
raise
finally:
db.conn.close()
def install(root):
root=Path(root).resolve();sys.path.insert(0,str(root/'scripts'));import backup_retention as br
python=root/'.runtime/retention-env/bin/python3'
if not python.is_file():
raise ValueError('Private maintenance runtime missing')
files=[Path(__file__).resolve(),root/'scripts/backup_retention.py',root/'scripts/revision_retention.py']
db=Database(root)
try:
accepted=br.accepted_backend(root)
if not accepted:raise ValueError('Accepted backend runtime is not registered')
br.atomic(root/'.runtime/retention/config.json',{'schemaSignature':db.schema_signature,
'acceptedJarSha256':accepted[1],'retirementGraceSeconds':1800,'intervalSeconds':300,
'backupIntervalSeconds':86400,'draftExpiryGraceSeconds':1800,'sourceRevisionGraceSeconds':1800})
finally:db.conn.close()
digest=hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
version=SUPPORT/'retention'/digest;version.mkdir(parents=True,exist_ok=True,mode=0o700)
for p in files:
shutil.copy2(p,version/p.name);(version/p.name).chmod(0o600)
legacy=root/'.runtime/current-snapshot-20261109'
if legacy.is_dir() and not legacy.is_symlink() and (legacy/'manifest.json').is_file():
br.atomic(root/'.runtime/retention/legacy.json',{'path':'.runtime/current-snapshot-20261109','manifestSha256':br.sha(legacy/'manifest.json')})
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
plist=Path.home()/'Library/LaunchAgents'/(LABEL+'.plist')
value={'Label':LABEL,'ProgramArguments':[str(python),str(version/'retention.py'),'run','--root',str(root)],
'WorkingDirectory':str(root),'RunAtLoad':True,'StartInterval':300,'ProcessType':'Background','Nice':10,
'LowPriorityIO':True,'ExitTimeOut':300,'EnvironmentVariables':{'PYTHONDONTWRITEBYTECODE':'1'},
'StandardOutPath':str(logs/'retention-launchd.log'),'StandardErrorPath':str(logs/'retention-launchd-error.log')}
subprocess.run(['/bin/launchctl','bootout','gui/'+str(os.getuid())+'/'+LABEL],capture_output=True)
with plist.open('wb') as out:plistlib.dump(value,out)
plist.chmod(0o600)
subprocess.run(['/bin/launchctl','enable','gui/'+str(os.getuid())+'/'+LABEL],check=True)
for attempt in range(50):
loaded=subprocess.run(['/bin/launchctl','bootstrap','gui/'+str(os.getuid()),str(plist)],capture_output=True)
if loaded.returncode==0:break
time.sleep(.2)
else:raise RuntimeError('Maintenance registration did not complete')
print('Installed five-minute idle maintenance and daily/publication backup checks')
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('action',choices=['run','install','status'])
parser.add_argument('--root',required=True)
args=parser.parse_args();root=Path(args.root).resolve();sys.path.append(str(root/'scripts'));import backup_retention as br
if args.action=='install':
with br.lock(root,timeout=0):install(root)
return
if args.action=='status':
p=root/'.runtime/retention/state.json';print(p.read_text() if p.exists() else '{}');return
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
handler=RotatingFileHandler(logs/'retention.log',maxBytes=2*1024*1024,backupCount=2)
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logging.getLogger().addHandler(handler);logging.getLogger().setLevel(logging.INFO)
try:
result=run(root);logging.info('maintenance complete %s',json.dumps(result))
except (Busy,br.Busy) as error:
logging.info('maintenance deferred: %s',str(error))
except Exception as error:
p=root/'.runtime/retention/state.json';state=json.loads(p.read_text()) if p.exists() else {}
state['lastError']={'time':time.time(),'type':type(error).__name__};br.atomic(p,state)
state['consecutiveFailures']=min(1000,state.get('consecutiveFailures',0)+1);br.atomic(p,state)
logging.error('maintenance failed safely: %s',type(error).__name__)
raise SystemExit(1)
if __name__=='__main__':main()