新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,207 @@
|
||||
"""Ownership records for disposable test schemas; never infer ownership from a prefix."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psutil
|
||||
import backup_retention as br
|
||||
|
||||
PREFIXES = ('quality_verify_', 'quality_scale_', 'quality_retention_test_',
|
||||
'quality_classification_source_', 'quality_classification_preview_',
|
||||
'quality_sales_type_test_', 'quality_unclassified_delta_', 'quality_sales_metric_test_',
|
||||
'quality_source_column_test_', 'quality_protocol_test_', 'quality_fine_rollup_')
|
||||
MARKER = '__quality_test_owner'
|
||||
|
||||
|
||||
def valid_name(name):
|
||||
return (isinstance(name, str) and len(name) <= 64 and
|
||||
re.fullmatch(r'[a-z0-9_]+_[a-f0-9]{8,32}', name) is not None and
|
||||
name.startswith(PREFIXES))
|
||||
|
||||
|
||||
def registry(root):
|
||||
root = Path(root).resolve(); base = root / '.runtime/test-resources'
|
||||
for path in (root / '.runtime', base):
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(root):
|
||||
raise ValueError('Unsafe test registry path')
|
||||
return base
|
||||
|
||||
|
||||
def protect(root, name):
|
||||
cfg = json.loads((Path(root) / '.runtime/local-config.json').read_text())
|
||||
if not valid_name(name) or name == cfg['mysqlDatabase']:
|
||||
raise ValueError('Not a disposable test database')
|
||||
|
||||
|
||||
def active_process(pid, created):
|
||||
try:
|
||||
p = psutil.Process(int(pid))
|
||||
return (abs(p.create_time() - float(created)) < .02 and p.is_running() and
|
||||
p.status() != psutil.STATUS_ZOMBIE)
|
||||
except psutil.NoSuchProcess:
|
||||
return False
|
||||
except (psutil.Error, TypeError, ValueError):
|
||||
return True
|
||||
|
||||
|
||||
def load(root, name):
|
||||
path = registry(root) / (name + '.json')
|
||||
if not path.is_file() or path.is_symlink():
|
||||
return None
|
||||
try:
|
||||
value = json.loads(path.read_text())
|
||||
if (not isinstance(value, dict) or value.get('kind') != 'quality-owned-test-db' or
|
||||
value.get('root') != str(Path(root).resolve()) or value.get('database') != name or
|
||||
not valid_name(name) or not re.fullmatch(r'[a-f0-9]{32}', value.get('token', ''))):
|
||||
return None
|
||||
return value
|
||||
except (OSError, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def create(root, name, execute, ddl=None, empty=False):
|
||||
root = Path(root).resolve(); protect(root, name)
|
||||
base = registry(root); base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
path = base / (name + '.json')
|
||||
# Exclusive registration also prevents a second process claiming an existing name.
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd)
|
||||
value = {'kind': 'quality-owned-test-db', 'formatVersion': 1, 'root': str(root),
|
||||
'database': name, 'token': uuid.uuid4().hex, 'createdAt': time.time(),
|
||||
'expiresAt': time.time() + 86400, 'ownerPid': os.getpid(),
|
||||
'ownerCreated': psutil.Process().create_time(), 'children': [], 'users': [],
|
||||
'serverUuid': execute('SELECT @@server_uuid;', None).strip(), 'created': False}
|
||||
br.atomic(path, value)
|
||||
try:
|
||||
result = execute(ddl or 'CREATE DATABASE `' + name + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', None)
|
||||
except BaseException:
|
||||
path.unlink(missing_ok=True); raise
|
||||
value['created'] = True; br.atomic(path, value)
|
||||
directory = root / '.runtime/mysql-data' / name
|
||||
if directory.is_dir() and not directory.is_symlink():
|
||||
stat = directory.stat()
|
||||
value['directoryIdentity'] = [stat.st_dev, stat.st_ino]
|
||||
br.atomic(path, value)
|
||||
if not empty:
|
||||
seal(root, name, execute)
|
||||
return result
|
||||
|
||||
|
||||
def seal(root, name, execute):
|
||||
value = load(root, name)
|
||||
if value is None or value['ownerPid'] != os.getpid():
|
||||
raise ValueError('Test resource owner not confirmed')
|
||||
execute('CREATE TABLE `' + name + '`.`' + MARKER + '` (token CHAR(32) PRIMARY KEY);'
|
||||
'INSERT INTO `' + name + '`.`' + MARKER + "` VALUES ('" + value['token'] + "');", None)
|
||||
value['ready'] = True; br.atomic(registry(root) / (name + '.json'), value)
|
||||
|
||||
|
||||
def same_directory(root, value):
|
||||
p = Path(root) / '.runtime/mysql-data' / value['database']
|
||||
if p.is_symlink() or not p.is_dir(): return False
|
||||
stat = p.stat()
|
||||
return value.get('directoryIdentity') == [stat.st_dev, stat.st_ino]
|
||||
|
||||
|
||||
def attach_child(root, name, pid):
|
||||
value = load(root, name)
|
||||
if value is None or value['ownerPid'] != os.getpid():
|
||||
raise ValueError('Test resource owner not confirmed')
|
||||
value.setdefault('children', []).append({'pid': pid, 'created': psutil.Process(pid).create_time()})
|
||||
br.atomic(registry(root) / (name + '.json'), value)
|
||||
|
||||
|
||||
def attach_user(root, name, user):
|
||||
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
|
||||
raise ValueError('Unsafe test account')
|
||||
value = load(root, name)
|
||||
if value is None or value['ownerPid'] != os.getpid():
|
||||
raise ValueError('Test resource owner not confirmed')
|
||||
value.setdefault('users', []).append(user)
|
||||
br.atomic(registry(root) / (name + '.json'), value)
|
||||
|
||||
|
||||
def marker_matches(value, execute):
|
||||
if not value.get('ready') or execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
|
||||
return False
|
||||
name = value['database']
|
||||
found = execute("SELECT COUNT(*) FROM information_schema.tables WHERE TABLE_SCHEMA='" + name +
|
||||
"' AND TABLE_NAME='" + MARKER + "';", None).strip()
|
||||
return found == '1' and execute('SELECT token FROM `' + name + '`.`' + MARKER + '`;', None).strip() == value['token']
|
||||
|
||||
|
||||
def drop(root, name, execute):
|
||||
protect(root, name); value = load(root, name)
|
||||
if value is None:
|
||||
raise ValueError('Registered test schema identity not confirmed')
|
||||
same_server = execute('SELECT @@server_uuid;', None).strip() == value['serverUuid']
|
||||
own_process = (value['ownerPid'] == os.getpid() and active_process(value['ownerPid'], value['ownerCreated']))
|
||||
if not same_server or not (marker_matches(value, execute) or
|
||||
(value.get('created') and not value.get('ready') and (own_process or same_directory(root, value)))):
|
||||
raise ValueError('Registered test schema identity not confirmed')
|
||||
result = execute('DROP DATABASE `' + name + '`;', None)
|
||||
value['databaseDropped'] = True
|
||||
path = registry(root) / (name + '.json')
|
||||
if value.get('users'):
|
||||
br.atomic(path, value)
|
||||
else:
|
||||
path.unlink()
|
||||
return result
|
||||
|
||||
|
||||
def user_dropped(root, user):
|
||||
for path in registry(root).glob('*.json'):
|
||||
value = load(root, path.stem)
|
||||
if value is None or value['ownerPid'] != os.getpid() or user not in value.get('users', []):
|
||||
continue
|
||||
value['users'].remove(user)
|
||||
if value.get('databaseDropped') and not value['users']:
|
||||
path.unlink()
|
||||
else:
|
||||
br.atomic(path, value)
|
||||
|
||||
|
||||
def cleanup(root, execute, now=None, limit=3):
|
||||
"""Called while maintenance owns its shared lock and the business application is idle."""
|
||||
now = time.time() if now is None else now
|
||||
result = {'testDatabasesRemoved': 0, 'testAccountsRemoved': 0, 'testResourcesDeferred': 0}
|
||||
for path in sorted(registry(root).glob('*.json')):
|
||||
if result['testDatabasesRemoved'] >= limit:
|
||||
break
|
||||
value = load(root, path.stem)
|
||||
if value is None or not value.get('created') or now < value.get('expiresAt', now + 1):
|
||||
continue
|
||||
protect(root, value['database'])
|
||||
if (active_process(value.get('ownerPid'), value.get('ownerCreated')) or
|
||||
any(active_process(p.get('pid'), p.get('created')) for p in value.get('children', []))):
|
||||
result['testResourcesDeferred'] += 1; continue
|
||||
if execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
|
||||
result['testResourcesDeferred'] += 1; continue
|
||||
name = value['database']
|
||||
if execute("SELECT COUNT(*) FROM information_schema.processlist WHERE DB='" + name + "';", None).strip() != '0':
|
||||
result['testResourcesDeferred'] += 1; continue
|
||||
if not value.get('databaseDropped'):
|
||||
if not marker_matches(value, execute) and not (not value.get('ready') and same_directory(root, value)):
|
||||
result['testResourcesDeferred'] += 1; continue
|
||||
drop(root, name, execute); result['testDatabasesRemoved'] += 1
|
||||
value['databaseDropped'] = True
|
||||
for user in list(value.get('users', [])):
|
||||
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
|
||||
continue
|
||||
grantee = "'" + user + "'@'127.0.0.1'"
|
||||
literal = "CONVERT(0x" + grantee.encode().hex() + " USING utf8mb4)"
|
||||
unsafe = execute("SELECT (SELECT COUNT(*) FROM information_schema.USER_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||
" AND PRIVILEGE_TYPE<>'USAGE')+(SELECT COUNT(*) FROM information_schema.SCHEMA_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.TABLE_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.processlist WHERE USER='" + user + "');", None).strip()
|
||||
if unsafe != '0':
|
||||
result['testResourcesDeferred'] += 1; continue
|
||||
execute("DROP USER IF EXISTS '" + user + "'@'127.0.0.1';", None)
|
||||
value['users'].remove(user); result['testAccountsRemoved'] += 1
|
||||
if not value.get('users'):
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
br.atomic(path, value)
|
||||
return result
|
||||
Reference in New Issue
Block a user