Files
quality_analyse/scripts/managed_test_resources.py

208 lines
9.4 KiB
Python

"""Ownership records for disposable test schemas; never infer ownership from a prefix."""
from pathlib import Path
import json
import os
import re
import time
import uuid
import psutil
import backup_retention as br
PREFIXES = ('quality_verify_', 'quality_scale_', 'quality_retention_test_',
'quality_classification_source_', 'quality_classification_preview_',
'quality_sales_type_test_', 'quality_unclassified_delta_', 'quality_sales_metric_test_',
'quality_source_column_test_', 'quality_protocol_test_', 'quality_fine_rollup_')
MARKER = '__quality_test_owner'
def valid_name(name):
return (isinstance(name, str) and len(name) <= 64 and
re.fullmatch(r'[a-z0-9_]+_[a-f0-9]{8,32}', name) is not None and
name.startswith(PREFIXES))
def registry(root):
root = Path(root).resolve(); base = root / '.runtime/test-resources'
for path in (root / '.runtime', base):
if path.is_symlink() or not path.resolve().is_relative_to(root):
raise ValueError('Unsafe test registry path')
return base
def protect(root, name):
cfg = json.loads((Path(root) / '.runtime/local-config.json').read_text())
if not valid_name(name) or name == cfg['mysqlDatabase']:
raise ValueError('Not a disposable test database')
def active_process(pid, created):
try:
p = psutil.Process(int(pid))
return (abs(p.create_time() - float(created)) < .02 and p.is_running() and
p.status() != psutil.STATUS_ZOMBIE)
except psutil.NoSuchProcess:
return False
except (psutil.Error, TypeError, ValueError):
return True
def load(root, name):
path = registry(root) / (name + '.json')
if not path.is_file() or path.is_symlink():
return None
try:
value = json.loads(path.read_text())
if (not isinstance(value, dict) or value.get('kind') != 'quality-owned-test-db' or
value.get('root') != str(Path(root).resolve()) or value.get('database') != name or
not valid_name(name) or not re.fullmatch(r'[a-f0-9]{32}', value.get('token', ''))):
return None
return value
except (OSError, ValueError, TypeError):
return None
def create(root, name, execute, ddl=None, empty=False):
root = Path(root).resolve(); protect(root, name)
base = registry(root); base.mkdir(parents=True, exist_ok=True, mode=0o700)
path = base / (name + '.json')
# Exclusive registration also prevents a second process claiming an existing name.
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd)
value = {'kind': 'quality-owned-test-db', 'formatVersion': 1, 'root': str(root),
'database': name, 'token': uuid.uuid4().hex, 'createdAt': time.time(),
'expiresAt': time.time() + 86400, 'ownerPid': os.getpid(),
'ownerCreated': psutil.Process().create_time(), 'children': [], 'users': [],
'serverUuid': execute('SELECT @@server_uuid;', None).strip(), 'created': False}
br.atomic(path, value)
try:
result = execute(ddl or 'CREATE DATABASE `' + name + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', None)
except BaseException:
path.unlink(missing_ok=True); raise
value['created'] = True; br.atomic(path, value)
directory = root / '.runtime/mysql-data' / name
if directory.is_dir() and not directory.is_symlink():
stat = directory.stat()
value['directoryIdentity'] = [stat.st_dev, stat.st_ino]
br.atomic(path, value)
if not empty:
seal(root, name, execute)
return result
def seal(root, name, execute):
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
execute('CREATE TABLE `' + name + '`.`' + MARKER + '` (token CHAR(32) PRIMARY KEY);'
'INSERT INTO `' + name + '`.`' + MARKER + "` VALUES ('" + value['token'] + "');", None)
value['ready'] = True; br.atomic(registry(root) / (name + '.json'), value)
def same_directory(root, value):
p = Path(root) / '.runtime/mysql-data' / value['database']
if p.is_symlink() or not p.is_dir(): return False
stat = p.stat()
return value.get('directoryIdentity') == [stat.st_dev, stat.st_ino]
def attach_child(root, name, pid):
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
value.setdefault('children', []).append({'pid': pid, 'created': psutil.Process(pid).create_time()})
br.atomic(registry(root) / (name + '.json'), value)
def attach_user(root, name, user):
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
raise ValueError('Unsafe test account')
value = load(root, name)
if value is None or value['ownerPid'] != os.getpid():
raise ValueError('Test resource owner not confirmed')
value.setdefault('users', []).append(user)
br.atomic(registry(root) / (name + '.json'), value)
def marker_matches(value, execute):
if not value.get('ready') or execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
return False
name = value['database']
found = execute("SELECT COUNT(*) FROM information_schema.tables WHERE TABLE_SCHEMA='" + name +
"' AND TABLE_NAME='" + MARKER + "';", None).strip()
return found == '1' and execute('SELECT token FROM `' + name + '`.`' + MARKER + '`;', None).strip() == value['token']
def drop(root, name, execute):
protect(root, name); value = load(root, name)
if value is None:
raise ValueError('Registered test schema identity not confirmed')
same_server = execute('SELECT @@server_uuid;', None).strip() == value['serverUuid']
own_process = (value['ownerPid'] == os.getpid() and active_process(value['ownerPid'], value['ownerCreated']))
if not same_server or not (marker_matches(value, execute) or
(value.get('created') and not value.get('ready') and (own_process or same_directory(root, value)))):
raise ValueError('Registered test schema identity not confirmed')
result = execute('DROP DATABASE `' + name + '`;', None)
value['databaseDropped'] = True
path = registry(root) / (name + '.json')
if value.get('users'):
br.atomic(path, value)
else:
path.unlink()
return result
def user_dropped(root, user):
for path in registry(root).glob('*.json'):
value = load(root, path.stem)
if value is None or value['ownerPid'] != os.getpid() or user not in value.get('users', []):
continue
value['users'].remove(user)
if value.get('databaseDropped') and not value['users']:
path.unlink()
else:
br.atomic(path, value)
def cleanup(root, execute, now=None, limit=3):
"""Called while maintenance owns its shared lock and the business application is idle."""
now = time.time() if now is None else now
result = {'testDatabasesRemoved': 0, 'testAccountsRemoved': 0, 'testResourcesDeferred': 0}
for path in sorted(registry(root).glob('*.json')):
if result['testDatabasesRemoved'] >= limit:
break
value = load(root, path.stem)
if value is None or not value.get('created') or now < value.get('expiresAt', now + 1):
continue
protect(root, value['database'])
if (active_process(value.get('ownerPid'), value.get('ownerCreated')) or
any(active_process(p.get('pid'), p.get('created')) for p in value.get('children', []))):
result['testResourcesDeferred'] += 1; continue
if execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
result['testResourcesDeferred'] += 1; continue
name = value['database']
if execute("SELECT COUNT(*) FROM information_schema.processlist WHERE DB='" + name + "';", None).strip() != '0':
result['testResourcesDeferred'] += 1; continue
if not value.get('databaseDropped'):
if not marker_matches(value, execute) and not (not value.get('ready') and same_directory(root, value)):
result['testResourcesDeferred'] += 1; continue
drop(root, name, execute); result['testDatabasesRemoved'] += 1
value['databaseDropped'] = True
for user in list(value.get('users', [])):
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
continue
grantee = "'" + user + "'@'127.0.0.1'"
literal = "CONVERT(0x" + grantee.encode().hex() + " USING utf8mb4)"
unsafe = execute("SELECT (SELECT COUNT(*) FROM information_schema.USER_PRIVILEGES WHERE GRANTEE=" + literal +
" AND PRIVILEGE_TYPE<>'USAGE')+(SELECT COUNT(*) FROM information_schema.SCHEMA_PRIVILEGES WHERE GRANTEE=" + literal +
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.TABLE_PRIVILEGES WHERE GRANTEE=" + literal +
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.processlist WHERE USER='" + user + "');", None).strip()
if unsafe != '0':
result['testResourcesDeferred'] += 1; continue
execute("DROP USER IF EXISTS '" + user + "'@'127.0.0.1';", None)
value['users'].remove(user); result['testAccountsRemoved'] += 1
if not value.get('users'):
path.unlink(missing_ok=True)
else:
br.atomic(path, value)
return result