#!/usr/bin/env python3 """Conservative, online retention for this Mac's accepted Quality Center runtime.""" import argparse from contextlib import contextmanager from datetime import datetime import hashlib import importlib import json import logging from logging.handlers import RotatingFileHandler import os from pathlib import Path import plistlib import re import shutil import subprocess import sys import time import urllib.request import pymysql from pymysql.cursors import DictCursor LABEL = 'com.lyunlab.quality-center-retention' SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') BUILD_CHILDREN = ('defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', 'fact_black', 'fact_bright') OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', 'release_action', 'candidate_reconciliation') MUTABLE_RELEASE_REFS = ('data_release', 'import_run', 'release_action', 'candidate_reconciliation') def ident(value): if not re.fullmatch(r'[A-Za-z0-9_]+', value): raise ValueError('Invalid SQL identifier') return '`' + value + '`' class Busy(RuntimeError): pass def safe_file(path, boundary): path, boundary = Path(path), Path(boundary).absolute() if not path.is_absolute(): return False if not path.is_relative_to(boundary) or not path.resolve().is_relative_to(boundary.resolve()): return False cursor = path while cursor != boundary: if cursor.is_symlink(): return False cursor = cursor.parent return not boundary.is_symlink() and path.is_file() def fingerprint(path): s = Path(path).stat() return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns) class Database: def __init__(self, root, database=None): self.root = Path(root).resolve() self.config = json.loads((self.root / '.runtime/local-config.json').read_text()) self.name = database or self.config['mysqlDatabase'] ident(self.name) endpoint = ({'host': self.config['mysqlHost'], 'port': int(self.config['mysqlPort'])} if self.config.get('mysqlHost') else {'unix_socket': str(self.root / '.runtime/mysql.sock')}) self.conn = pymysql.connect(**endpoint, user='root', password=self.config['rootPassword'], database=self.name, charset='utf8mb4', autocommit=True, cursorclass=DictCursor, connect_timeout=3, read_timeout=10, write_timeout=5) self.execute("SET SESSION time_zone='+00:00'") self.execute('SET SESSION lock_wait_timeout=1') self.execute('SET SESSION innodb_lock_wait_timeout=1') rows = self.rows('SELECT TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA=%s', (self.name,)) self.columns = {} self.column_types = {} for row in rows: self.columns.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['IS_NULLABLE'] == 'YES' self.column_types.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['DATA_TYPE'] self.tables = sorted(self.columns) self.fks = self.rows('SELECT TABLE_NAME,COLUMN_NAME,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME FROM information_schema.key_column_usage WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL', (self.name,)) self.schema_signature=hashlib.sha256(json.dumps({'columns':self.columns,'fks':sorted(self.fks,key=lambda r:json.dumps(r,sort_keys=True))},sort_keys=True).encode()).hexdigest() self.longest_exclusion=0.0 if not {'current_release','data_release','release_member','transform_build','source_upload_draft'}.issubset(self.columns): raise ValueError('Unexpected application schema') def rows(self, sql, args=()): with self.conn.cursor() as cursor: cursor.execute(sql, args) return list(cursor.fetchall()) def execute(self, sql, args=()): with self.conn.cursor() as cursor: return cursor.execute(sql, args) def current(self): rows = self.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'") if len(rows) != 1 or not rows[0]['release_id']: raise Busy('Current release not confirmed') fields='status,complete' if 'complete' in self.columns['data_release'] else 'status' release=self.rows('SELECT '+fields+' FROM data_release WHERE id=%s',(rows[0]['release_id'],)) if len(release)!=1 or release[0]['status']!='PUBLISHED' or release[0].get('complete',True) not in (True,1): raise Busy('Current release is not a complete publication') return rows[0]['release_id'] def busy(self): for table in ('task','import_run','data_workflow_operation','analysis_turn','analysis_protocol_execution','quality_inspection_run'): if table not in self.columns or 'status' not in self.columns[table]: continue sql = 'SELECT 1 FROM ' + ident(table) + ' WHERE (status IS NULL OR status NOT IN (' + ','.join(['%s']*len(TERMINAL)) + '))' if table == 'task' and {'next_retry_at','retry_count','max_retries'}.issubset(self.columns[table]): sql += " OR (status='FAILED' AND next_retry_at IS NOT NULL AND retry_count= grace: eligible.append(rid) if not eligible: return 0 rid = sorted(eligible, key=lambda x: seen[x])[0] # Clear optional historical links only. Unknown/business references were # included in protected above and are never cascaded away. for fk in self.fks: t,c = fk['TABLE_NAME'],fk['COLUMN_NAME'] if fk['REFERENCED_TABLE_NAME'] == 'data_release' and t in MUTABLE_RELEASE_REFS and self.columns[t][c]: self.execute('UPDATE ' + ident(t) + ' SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,)) if 'data_workflow_operation' in self.columns: for c in ('release_id','expected_current_release_id'): if c in self.columns['data_workflow_operation']: self.execute('UPDATE data_workflow_operation SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,)) for t in OWNED_RELEASE: if t in self.columns: self.execute('DELETE FROM ' + ident(t) + ' WHERE release_id=%s', (rid,)) self.execute('DELETE FROM data_release WHERE id=%s', (rid,)) seen.pop(rid, None) return 1 def build_step(self, state, grace, now, batch): referenced = {r['transform_build_id'] for r in self.rows('SELECT DISTINCT transform_build_id FROM release_member')} builds = self.rows('SELECT id,status,build_key FROM transform_build') seen = state.setdefault('builds', {}) ids = {r['id'] for r in builds} for stale in set(seen) - ids: seen.pop(stale) for row in builds: bid = row['id'] unknown_reference = any(self.rows('SELECT 1 FROM ' + ident(f['TABLE_NAME']) + ' WHERE ' + ident(f['COLUMN_NAME']) + '=%s LIMIT 1', (bid,)) for f in self.fks if f['REFERENCED_TABLE_NAME'] == 'transform_build' and f['TABLE_NAME'] not in (*BUILD_CHILDREN, 'release_member')) if bid in referenced or unknown_reference or row['status'] not in ('COMPLETED','FAILED','CANCELLED','RETIRED'): seen.pop(bid, None) continue seen.setdefault(bid, now) if row['status'] != 'RETIRED' and now - seen[bid] < grace: continue if row['status'] != 'RETIRED': # Free the unique reuse key before deleting chunks. A new import # can build a fresh result, never reuse this partially emptied one. tombstone = hashlib.sha256(('retired:'+bid+':'+row['build_key']).encode()).hexdigest() self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid)) for t in BUILD_CHILDREN: if t in self.columns: count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) if count: return count, 0 self.execute('DELETE FROM transform_build WHERE id=%s', (bid,)) seen.pop(bid, None) return 0, 1 return 0, 0 def file_referenced(self, path): for table, cols in self.columns.items(): for col in cols: if col.endswith('_path') and self.rows('SELECT 1 FROM ' + ident(table) + ' WHERE ' + ident(col) + '=%s LIMIT 1', (str(path),)): return True return False def expired(self, limit=10): return self.rows('SELECT * FROM source_upload_draft WHERE expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6)) ORDER BY expires_at LIMIT ' + str(int(limit))) def promotion(self, draft): if not draft.get('candidate_id') or not {'source_revision','source_document','source_candidate'}.issubset(self.columns): return None rows = self.rows("SELECT * FROM source_candidate WHERE id=%s AND status='IMPORTED'", (draft['candidate_id'],)) if not rows: return None c = rows[0] docs = {d['id']:d for d in self.rows('SELECT id,source_slot FROM source_document')} for rev in self.rows('SELECT source_document_id,archive_path,sha256,parser_config_hash,file_size FROM source_revision WHERE sha256=%s AND parser_config_hash=%s', (c['sha256'],c['parser_config_hash'])): if docs.get(rev['source_document_id'],{}).get('source_slot') == c['source_slot']: p=Path(rev['archive_path']) if safe_file(p,self.root/'runtimedata/archive/source-revisions') and p.stat().st_size==rev['file_size']: return c,rev,p return None def clean_owned_scratch(self, br): import psutil server=self.rows('SELECT @@server_uuid AS value')[0]['value'] directory=self.root/'.runtime/retention/scratch' removed=0 for path in directory.glob('*.json'): if path.is_symlink():continue value=json.loads(path.read_text());name=value.get('database','') if value.get('kind')!='quality-restore-check' or value.get('server')!=server or not value.get('created') or name==self.name or not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',name):continue if time.time()-value['createdAt']<300:continue alive=False for pid_key,created_key in [('ownerPid','ownerCreated'),('childPid','childCreated')]: if pid_key not in value:continue try: process=psutil.Process(value[pid_key]) if abs(process.create_time()-value[created_key])<.01 and process.is_running() and process.status()!=psutil.STATUS_ZOMBIE:alive=True except psutil.NoSuchProcess:pass except psutil.Error:alive=True if alive or self.rows('SELECT 1 FROM information_schema.processlist WHERE DB=%s LIMIT 1',(name,)):continue self.execute('DROP DATABASE IF EXISTS '+ident(name));path.unlink();removed+=1 return removed def cleanup_drafts(db, current, state, now, br): removed=0; bytes_removed=0 # Checksum expensive files outside the brief database exclusion window. for draft in db.expired(): path=Path(draft['source_path']); boundary=db.root/'runtimedata/archive/upload-drafts' info=None if safe_file(path,boundary): info=fingerprint(path) if info[2]!=draft['file_size'] or br.sha(path)!=draft['sha256']: continue promotion=db.promotion(draft) if promotion: c,rev,target=promotion target_stamp=fingerprint(target) if br.sha(target)!=rev['sha256']: continue with db.frozen(current): live=db.rows('SELECT * FROM source_upload_draft WHERE id=%s AND expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6))',(draft['id'],)) if not live or live[0]!=draft: continue if info and (not safe_file(path,boundary) or fingerprint(path)!=info): continue if promotion and safe_file(target,db.root/'runtimedata/archive/source-revisions') and fingerprint(target)==target_stamp: # The WHERE identity prevents overwriting a source reconfigured # between preflight hashing and acquiring the database exclusion. db.execute("UPDATE source_candidate SET source_path=%s,archive_path=%s WHERE id=%s AND status='IMPORTED' AND source_path=%s AND sha256=%s AND parser_config_hash=%s", (str(target),str(target),c['id'],c['source_path'],c['sha256'],c['parser_config_hash'])) db.execute('DELETE FROM source_upload_draft WHERE id=%s',(draft['id'],)) if info: with db.frozen(current): if safe_file(path,boundary) and fingerprint(path)==info and not db.file_referenced(path): bytes_removed+=path.stat().st_size path.unlink() removed+=1 # Interrupted uploads can leave no database row at all. Their age and every # direct path reference are rechecked under the same exclusion before unlink. boundary=db.root/'runtimedata/archive/upload-drafts' for path in list(boundary.glob('upload-*.xlsx'))[:100]: if not safe_file(path,boundary) or now-path.stat().st_mtime<86400: continue info=fingerprint(path) with db.frozen(current): if fingerprint(path)==info and not db.file_referenced(path): size=path.stat().st_size;path.unlink();bytes_removed+=size return removed,bytes_removed def run(root, max_batches=30, grace=1800): root=Path(root).resolve();sys.path.append(str(root/'scripts')) import backup_retention as br import backup import backup_verifier import revision_retention if (root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused') readiness=urllib.request.build_opener(urllib.request.ProxyHandler({})) try: with readiness.open('http://127.0.0.1:5281/actuator/health',timeout=2) as response: if json.load(response).get('status')!='UP':raise Busy('Backend is not ready') except (OSError,ValueError) as error:raise Busy('Backend is not ready') from error state_path=root/'.runtime/retention/state.json' state=json.loads(state_path.read_text()) if state_path.exists() else {} result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0} with br.lock(root,timeout=0): result['retiredBackupFoldersRemoved']=br.clean_retired(root) result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root) db=Database(root) try: policy_path=root/'.runtime/retention/config.json' if policy_path.exists(): policy=json.loads(policy_path.read_text());accepted=br.accepted_backend(root) if policy['schemaSignature']!=db.schema_signature or not accepted or policy['acceptedJarSha256']!=accepted[1]: raise Busy('Application version changed; retention compatibility review required') result['scratchRemoved']=db.clean_owned_scratch(br) current=db.current() if db.busy(): raise Busy('An application job is active') latest=br.latest(root) # Recover a restore-verified candidate left between verification and # pointer publication by a previous crash, without creating a duplicate. if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current: candidates=[] for folder in (root/'output/backups').iterdir() if (root/'output/backups').exists() else []: candidate=br.verified(folder,checksums=False) if candidate and candidate.get('queryEvidence',{}).get('releaseId')==current:candidates.append((candidate.get('createdAt',''),folder)) if candidates: selected=max(candidates)[1] if br.verified(selected):br.rotate(root,selected);latest=br.latest(root) if latest: state['lastBackupAt']=max(state.get('lastBackupAt',0),datetime.fromisoformat(latest[1]['createdAt']).timestamp()) due=not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current due=due or time.time()-state.get('lastBackupAt',0)>86400 due=due or (state.get('dirtySince') and not state.get('builds') and time.time()-state['dirtySince']>300) if due: database_bytes=int(db.rows('SELECT COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) AS size FROM information_schema.tables WHERE TABLE_SCHEMA=%s',(db.name,))[0]['size']) archive_bytes=sum(p.stat().st_size for p in (root/'runtimedata/archive').rglob('*') if p.is_file()) required_free=max(10*1024**3,3*database_bytes+2*archive_bytes+5*1024**3) reserves = [root / '.runtime', root / 'output/backups'] for reserve in reserves: reserve.mkdir(parents=True, exist_ok=True) if min(shutil.disk_usage(p).free for p in reserves)deadline or db.longest_exclusion>.5: break time.sleep(.05) drafts,size=cleanup_drafts(db,current,state,time.time(),br) result['draftsRemoved']=drafts;result['bytesRemoved']=size sources=revision_retention.Cleaner(db,br,safe_file,fingerprint,Busy) result.update(sources.collect(current,state,time.time(),grace)) result['maxExclusiveSeconds']=round(db.longest_exclusion,4) if any(result[k] for k in ('releasesRemoved','rowsRemoved','buildsRemoved','draftsRemoved','bytesRemoved','sourceRevisionsRemoved','sourceArchiveBytesRemoved')): state.setdefault('dirtySince',time.time()) state['lastResult']=result;state.pop('lastError',None) state['consecutiveFailures']=0 br.atomic(state_path,state) return result except Busy as error: state['lastDeferred']={'time':time.time(),'reason':str(error)} state['lastResult']=result;br.atomic(state_path,state) raise finally: db.conn.close() def install(root): root=Path(root).resolve();sys.path.insert(0,str(root/'scripts'));import backup_retention as br python=root/'.runtime/retention-env/bin/python3' if not python.is_file(): raise ValueError('Private maintenance runtime missing') files=[Path(__file__).resolve(),root/'scripts/backup_retention.py',root/'scripts/revision_retention.py'] db=Database(root) try: accepted=br.accepted_backend(root) if not accepted:raise ValueError('Accepted backend runtime is not registered') br.atomic(root/'.runtime/retention/config.json',{'schemaSignature':db.schema_signature, 'acceptedJarSha256':accepted[1],'retirementGraceSeconds':1800,'intervalSeconds':300, 'backupIntervalSeconds':86400,'draftExpiryGraceSeconds':1800,'sourceRevisionGraceSeconds':1800}) finally:db.conn.close() digest=hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest() version=SUPPORT/'retention'/digest;version.mkdir(parents=True,exist_ok=True,mode=0o700) for p in files: shutil.copy2(p,version/p.name);(version/p.name).chmod(0o600) legacy=root/'.runtime/current-snapshot-20261109' if legacy.is_dir() and not legacy.is_symlink() and (legacy/'manifest.json').is_file(): br.atomic(root/'.runtime/retention/legacy.json',{'path':'.runtime/current-snapshot-20261109','manifestSha256':br.sha(legacy/'manifest.json')}) logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700) plist=Path.home()/'Library/LaunchAgents'/(LABEL+'.plist') value={'Label':LABEL,'ProgramArguments':[str(python),str(version/'retention.py'),'run','--root',str(root)], 'WorkingDirectory':str(root),'RunAtLoad':True,'StartInterval':300,'ProcessType':'Background','Nice':10, 'LowPriorityIO':True,'ExitTimeOut':300,'EnvironmentVariables':{'PYTHONDONTWRITEBYTECODE':'1'}, 'StandardOutPath':str(logs/'retention-launchd.log'),'StandardErrorPath':str(logs/'retention-launchd-error.log')} subprocess.run(['/bin/launchctl','bootout','gui/'+str(os.getuid())+'/'+LABEL],capture_output=True) with plist.open('wb') as out:plistlib.dump(value,out) plist.chmod(0o600) subprocess.run(['/bin/launchctl','enable','gui/'+str(os.getuid())+'/'+LABEL],check=True) for attempt in range(50): loaded=subprocess.run(['/bin/launchctl','bootstrap','gui/'+str(os.getuid()),str(plist)],capture_output=True) if loaded.returncode==0:break time.sleep(.2) else:raise RuntimeError('Maintenance registration did not complete') print('Installed five-minute idle maintenance and daily/publication backup checks') def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('action',choices=['run','install','status']) parser.add_argument('--root',required=True) args=parser.parse_args();root=Path(args.root).resolve();sys.path.append(str(root/'scripts'));import backup_retention as br if args.action=='install': with br.lock(root,timeout=0):install(root) return if args.action=='status': p=root/'.runtime/retention/state.json';print(p.read_text() if p.exists() else '{}');return logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700) handler=RotatingFileHandler(logs/'retention.log',maxBytes=2*1024*1024,backupCount=2) handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logging.getLogger().addHandler(handler);logging.getLogger().setLevel(logging.INFO) try: result=run(root);logging.info('maintenance complete %s',json.dumps(result)) except (Busy,br.Busy) as error: logging.info('maintenance deferred: %s',str(error)) except Exception as error: p=root/'.runtime/retention/state.json';state=json.loads(p.read_text()) if p.exists() else {} state['lastError']={'time':time.time(),'type':type(error).__name__};br.atomic(p,state) state['consecutiveFailures']=min(1000,state.get('consecutiveFailures',0)+1);br.atomic(p,state) logging.error('maintenance failed safely: %s',type(error).__name__) raise SystemExit(1) if __name__=='__main__':main()