Files
quality_analyse/scripts/backup_verifier.py
T

113 lines
5.7 KiB
Python

"""Private, socket-only MySQL instance for low-priority restore verification."""
from contextlib import contextmanager
import json
import os
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import uuid
import psutil
import backup_retention as br
def stop_owned(value):
if value.get('initPid'):
try:
init=psutil.Process(value['initPid'])
if abs(init.create_time()-value['initCreated'])<.01 and init.is_running() and init.status()!=psutil.STATUS_ZOMBIE:return False
except psutil.NoSuchProcess:pass
except psutil.Error:return False
pid=value.get('serverPid')
if not pid:
p=Path(value['directory'])/'server.pid'
if not p.exists():return True
pid=int(p.read_text())
try:
process=psutil.Process(pid)
if value.get('serverCreated') and abs(process.create_time()-value['serverCreated'])>.01:return True
command=process.cmdline()
if process.uids().real!=os.getuid() or Path(process.exe()).resolve()!=Path(value['binary']).resolve() or '--datadir='+value['directory']+'/data' not in command or '--skip-networking' not in command:
return False
process.terminate()
try:process.wait(timeout=180)
except psutil.TimeoutExpired:return False
return True
except psutil.NoSuchProcess:return True
except psutil.Error:return False
def remove_owned(root, directory, value):
directory=Path(directory);base=Path(root)/'.runtime/retention'
if directory.parent!=base or not directory.name.startswith('verify-') or directory.is_symlink():
return False
if not stop_owned(value):return False
socket=Path(value['socketDirectory'])
if socket.parent.resolve()!=Path('/tmp').resolve() or not socket.name.startswith('qret-') or socket.is_symlink():return False
shutil.rmtree(socket,ignore_errors=True)
shutil.rmtree(directory)
return True
def clean_abandoned(root):
base=Path(root)/'.runtime/retention';removed=0
for directory in base.glob('verify-*'):
path=directory/'owner.json'
if directory.is_symlink() or not path.is_file() or path.is_symlink():continue
value=json.loads(path.read_text())
if value.get('kind')!='private-quality-restore' or value.get('directory')!=str(directory):continue
try:
p=psutil.Process(value['ownerPid'])
if abs(p.create_time()-value['ownerCreated'])<.01 and p.is_running() and p.status()!=psutil.STATUS_ZOMBIE:continue
except psutil.NoSuchProcess:pass
except psutil.Error:continue
if time.time()-value['createdAt']<300:continue
if remove_owned(root,directory,value):removed+=1
return removed
@contextmanager
def instance(root):
root=Path(root).resolve();base=root/'.runtime/retention';base.mkdir(parents=True,exist_ok=True,mode=0o700)
directory=base/('verify-'+uuid.uuid4().hex);directory.mkdir(mode=0o700)
socket=Path(tempfile.mkdtemp(prefix='qret-',dir='/tmp')).resolve();socket.chmod(0o700)
binary=root/'.runtime/mysql/bin/mysqld'
value={'kind':'private-quality-restore','directory':str(directory),'socketDirectory':str(socket),
'binary':str(binary),'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time()}
br.atomic(directory/'owner.json',value)
process=None
try:
with (directory/'initialize.log').open('ab') as output:
init=subprocess.Popen([str(binary),'--no-defaults','--initialize-insecure','--basedir='+str(binary.parent.parent),
'--datadir='+str(directory/'data'),'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--skip-log-bin'],stdout=output,stderr=output)
value.update({'initPid':init.pid,'initCreated':psutil.Process(init.pid).create_time()});br.atomic(directory/'owner.json',value)
try:code=init.wait(timeout=180)
except subprocess.TimeoutExpired:
init.kill();init.wait();raise
if code:raise RuntimeError('Private restore server initialization failed')
args=[str(binary),'--no-defaults','--basedir='+str(binary.parent.parent),'--datadir='+str(directory/'data'),
'--socket='+str(socket/'mysql.sock'),'--pid-file='+str(directory/'server.pid'),
'--log-error='+str(directory/'server.log'),'--skip-networking','--mysqlx=OFF','--skip-log-bin',
'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--innodb-buffer-pool-size=128M','--innodb-redo-log-capacity=64M','--max-connections=8']
with (directory/'stdout.log').open('ab') as output:
process=subprocess.Popen(args,stdout=output,stderr=output,stdin=subprocess.DEVNULL)
value.update({'serverPid':process.pid,'serverCreated':psutil.Process(process.pid).create_time()});br.atomic(directory/'owner.json',value)
config=directory/'client.cnf'
config.write_text('[client]\nuser=root\nsocket='+str(socket/'mysql.sock')+'\n');config.chmod(0o600)
deadline=time.monotonic()+90
while time.monotonic()<deadline:
if process.poll() is not None:raise RuntimeError('Private restore server exited')
ready=subprocess.run([str(binary.parent/'mysql'),'--defaults-file='+str(config),'--batch','--skip-column-names','-e','SELECT 1'],capture_output=True)
if ready.returncode==0:break
time.sleep(.2)
else:raise RuntimeError('Private restore server not ready')
yield config
finally:
if not remove_owned(root,directory,value):
raise RuntimeError('Private restore server cleanup requires inspection')
if process is not None:process.wait(timeout=5)