352 lines
16 KiB
Python
352 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Pinned macOS service guardian. Never initializes or imports a database."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
import os
|
|
from pathlib import Path
|
|
import plistlib
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
import psutil
|
|
|
|
LABEL = 'com.lyunlab.quality-center-service'
|
|
HOME = Path.home() / 'Library/Application Support/QualityCenterService'
|
|
CONFIG = HOME / 'service.json'
|
|
PLIST = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
|
DOMAIN = 'gui/' + str(os.getuid())
|
|
|
|
|
|
def atomic(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temporary = path.with_name(path.name + '.tmp-' + str(os.getpid()))
|
|
with temporary.open('w') as stream:
|
|
os.chmod(temporary, 0o600)
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
temporary.replace(path)
|
|
|
|
|
|
def stamp(pid):
|
|
return subprocess.run(['/bin/ps', '-p', str(pid), '-o', 'lstart='],
|
|
capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def matches(record, command=None):
|
|
"""True=confirmed identity, False=gone/reused, None=unknown identity."""
|
|
if not record:
|
|
return False
|
|
try:
|
|
p = psutil.Process(record['pid'])
|
|
if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
|
|
return False
|
|
if 'created' in record and abs(p.create_time() - record['created']) > .01:
|
|
return False
|
|
if 'started' in record and stamp(p.pid) != record['started']:
|
|
return False
|
|
expected = command or record['command']
|
|
if p.uids().real != os.getuid():
|
|
return None
|
|
if Path(p.exe()).resolve() != Path(expected[0]).resolve() or p.cmdline() != expected:
|
|
return None
|
|
return True
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
except (psutil.Error, KeyError, ValueError, OSError):
|
|
return None
|
|
|
|
|
|
def record(pid, command):
|
|
return {'pid': pid, 'created': psutil.Process(pid).create_time(),
|
|
'command': command, 'started': stamp(pid)}
|
|
|
|
|
|
class Guardian:
|
|
def __init__(self, config, home=HOME):
|
|
self.config = config
|
|
self.home = Path(home)
|
|
self.root = Path(config['root']).resolve()
|
|
self.runtime = self.root / '.runtime'
|
|
self.records = {}
|
|
self.children = {}
|
|
self.next_start = {'mysql': 0, 'backend': 0}
|
|
self.restarts = {'mysql': 0, 'backend': 0}
|
|
self.stopping = False
|
|
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
def preflight(self):
|
|
self.value = json.loads((self.runtime / 'local-config.json').read_text())
|
|
if not self.value.get('databaseReady') or not (self.runtime / 'mysql-data/auto.cnf').is_file():
|
|
raise RuntimeError('Existing initialized database not confirmed; refusing bootstrap')
|
|
self.jar = self.runtime / 'backend-builds' / (self.config['jarSha256'] + '.jar')
|
|
if hashlib.sha256(self.jar.read_bytes()).hexdigest() != self.config['jarSha256']:
|
|
raise RuntimeError('Accepted backend digest mismatch')
|
|
for binary in [self.runtime / 'mysql/bin/mysqld', self.runtime / 'jdk/Contents/Home/bin/java']:
|
|
if not os.access(binary, os.X_OK):
|
|
raise RuntimeError('Required runtime missing')
|
|
|
|
def command(self, role):
|
|
r = self.runtime
|
|
if role == 'backend':
|
|
return [str(r / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx2048m', '-jar', str(self.jar)]
|
|
return [str(r / 'mysql/bin/mysqld'), '--no-defaults', '--basedir=' + str(r / 'mysql'),
|
|
'--datadir=' + str(r / 'mysql-data'), '--port=' + str(self.value['mysqlPort']),
|
|
'--bind-address=127.0.0.1', '--socket=' + str(r / 'mysql.sock'),
|
|
'--pid-file=' + str(r / 'mysql-server.pid'),
|
|
'--log-error=' + str(self.root / 'output/logs/mysql-server.log'), '--mysqlx=OFF',
|
|
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M',
|
|
*(['--skip-log-bin'] if self.value.get('binaryLogEnabled', True) is False else [])]
|
|
|
|
def environment(self):
|
|
v, r = self.value, self.runtime
|
|
env = os.environ.copy()
|
|
env.update({
|
|
'JAVA_HOME': str(r / 'jdk/Contents/Home'),
|
|
'PATH': str(r / 'jdk/Contents/Home/bin') + ':' + self.config['path'],
|
|
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{v["mysqlPort"]}/{v["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
|
|
'MYSQL_USER': v['mysqlUser'], 'MYSQL_PASSWORD': v['mysqlPassword'],
|
|
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': str(v.get('legacyTimestampOffsetSeconds', 'UNCONFIRMED')),
|
|
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': v['adminUsername'],
|
|
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': v['adminPassword'],
|
|
'QUALITY_ARCHIVE_ROOT': str(self.root / 'runtimedata/archive'),
|
|
'QUALITY_ALLOWED_ROOTS': str(self.root / '质量数据/系统原始数据示例'),
|
|
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
|
|
'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED': str(v.get('analysisProtocolV1Enabled', False)).lower(),
|
|
'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED': str(v.get('analysisAutonomousEnabled', False)).lower(),
|
|
'QUALITY_PROJECT_ROOT': str(self.root), 'QUALITY_DEFAULT_SOURCE_YEAR': '2026',
|
|
'SERVER_ADDRESS': '127.0.0.1', 'PORT': '5281', 'PYTHONDONTWRITEBYTECODE': '1',
|
|
})
|
|
log_config = r / 'operations/logback-quality.xml'
|
|
if log_config.is_file() and not log_config.is_symlink():
|
|
env['LOGGING_CONFIG'] = str(log_config)
|
|
env['QUALITY_LOG_DIR'] = str(self.root / 'output/logs')
|
|
return env
|
|
|
|
def port(self, role):
|
|
return self.value['mysqlPort'] if role == 'mysql' else 5281
|
|
|
|
def occupied(self, role):
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', self.port(role)), timeout=.5):
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
def db_ready(self):
|
|
result = subprocess.run([str(self.runtime / 'mysql/bin/mysql'),
|
|
'--defaults-file=' + str(self.runtime / 'mysql-root.cnf'), '--connect-timeout=2',
|
|
'--batch', '--skip-column-names', '-e', 'SELECT 1;'],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
|
|
return result.returncode == 0 and result.stdout.strip() == b'1'
|
|
|
|
def backend_ready(self):
|
|
try:
|
|
with self.opener.open('http://127.0.0.1:5281/actuator/health', timeout=2) as response:
|
|
return json.load(response).get('status') == 'UP'
|
|
except (OSError, ValueError):
|
|
return False
|
|
|
|
def adopt(self, role):
|
|
path = self.runtime / (role + '.process.json')
|
|
if not path.exists():
|
|
return False
|
|
info = json.loads(path.read_text())
|
|
identity = matches(info, self.command(role))
|
|
if identity is None:
|
|
raise RuntimeError(role + ': existing process identity cannot be confirmed')
|
|
if identity:
|
|
self.records[role] = record(info['pid'], self.command(role))
|
|
logging.info('%s adopted pid=%s', role, info['pid'])
|
|
return bool(identity)
|
|
|
|
def ensure(self, role):
|
|
child = self.children.get(role)
|
|
if child is not None:
|
|
child.poll() # Reap exited children before checking PID identity.
|
|
info = self.records.get(role)
|
|
state = matches(info, self.command(role))
|
|
if state is None:
|
|
raise RuntimeError(role + ': process identity is unknown; no replacement started')
|
|
if state:
|
|
return True
|
|
if info:
|
|
self.records.pop(role)
|
|
self.next_start[role] = time.monotonic() + 10
|
|
logging.warning('%s exited; restart scheduled in 10 seconds', role)
|
|
if time.monotonic() < self.next_start[role]:
|
|
return False
|
|
if self.adopt(role):
|
|
return True
|
|
if self.occupied(role):
|
|
raise RuntimeError(role + ': port occupied by an unconfirmed process')
|
|
command = self.command(role)
|
|
logs = self.root / 'output/logs'
|
|
logs.mkdir(parents=True, exist_ok=True)
|
|
log_name = ('backend-console.log' if role == 'backend' and
|
|
(self.runtime / 'operations/logback-quality.xml').is_file() else role + '.log')
|
|
with (logs / log_name).open('ab') as log:
|
|
child = subprocess.Popen(command, cwd=self.root, env=self.environment(),
|
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
|
self.children[role] = child
|
|
self.records[role] = info = record(child.pid, command)
|
|
(self.runtime / (role + '.pid')).write_text(str(child.pid))
|
|
atomic(self.runtime / (role + '.process.json'), {'pid': child.pid, 'started': info['started']})
|
|
self.restarts[role] += 1
|
|
logging.info('%s started pid=%s', role, child.pid)
|
|
return True
|
|
|
|
def tick(self):
|
|
mysql_alive = self.ensure('mysql')
|
|
ready = mysql_alive and self.db_ready()
|
|
if ready:
|
|
self.ensure('backend')
|
|
atomic(self.home / 'state.json', {'updatedAt': time.time(), 'supervisorPid': os.getpid(),
|
|
'processes': self.records, 'starts': self.restarts,
|
|
'databaseReady': bool(ready), 'backendReady': self.backend_ready()})
|
|
|
|
def run(self):
|
|
self.preflight()
|
|
with (self.runtime / 'quality-service.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
signal.signal(sig, lambda *_: setattr(self, 'stopping', True))
|
|
logging.info('guardian started; accepted backend=%s', self.config['jarSha256'])
|
|
while not self.stopping:
|
|
try:
|
|
self.tick()
|
|
except Exception as error:
|
|
# Never log exception arguments: database credentials may be embedded.
|
|
detail = str(error) if type(error) is RuntimeError else type(error).__name__
|
|
logging.error('check failed: %s; leaving existing processes intact', detail)
|
|
atomic(self.home / 'error.json', {'time': time.time(), 'type': type(error).__name__})
|
|
for _ in range(30):
|
|
if self.stopping:
|
|
break
|
|
time.sleep(.1)
|
|
logging.info('guardian stopped; service processes remain available for maintenance')
|
|
|
|
|
|
def ctl(*args, check=True):
|
|
return subprocess.run(['/bin/launchctl', *args], check=check, capture_output=True, text=True)
|
|
|
|
|
|
def install(root, jar, clear_maintenance_pause=True):
|
|
root, jar = Path(root).resolve(), Path(jar).resolve()
|
|
cfg = {'root': str(root), 'jarSha256': hashlib.sha256(jar.read_bytes()).hexdigest(),
|
|
'path': '/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.13/bin:/usr/bin:/bin:/usr/sbin:/sbin'}
|
|
Guardian(cfg).preflight()
|
|
HOME.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
HOME.chmod(0o700)
|
|
script = Path(__file__).resolve()
|
|
digest = hashlib.sha256(script.read_bytes()).hexdigest()
|
|
target = HOME / 'versions' / digest / 'service.py'
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
shutil.copyfile(script, target)
|
|
target.chmod(0o600)
|
|
# Remove only the previous guardian; adopted MySQL/backend processes keep running.
|
|
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
|
atomic(CONFIG, cfg)
|
|
logs = HOME / 'logs'
|
|
logs.mkdir(mode=0o700, exist_ok=True)
|
|
plist = {'Label': LABEL, 'ProgramArguments': [str(Path(sys.executable).resolve()), str(target), 'run'],
|
|
'WorkingDirectory': str(HOME), 'RunAtLoad': True, 'KeepAlive': True,
|
|
'ThrottleInterval': 10, 'ProcessType': 'Background', 'ExitTimeOut': 30,
|
|
'AbandonProcessGroup': True,
|
|
'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
|
'StandardOutPath': str(logs / 'launchd.log'), 'StandardErrorPath': str(logs / 'launchd-error.log')}
|
|
PLIST.parent.mkdir(exist_ok=True)
|
|
with PLIST.open('wb') as stream:
|
|
plistlib.dump(plist, stream)
|
|
PLIST.chmod(0o600)
|
|
ctl('enable', DOMAIN + '/' + LABEL)
|
|
for attempt in range(50):
|
|
loaded=ctl('bootstrap', DOMAIN, str(PLIST), check=False)
|
|
if loaded.returncode==0:break
|
|
time.sleep(.2)
|
|
else:raise RuntimeError('Service registration did not complete; existing processes preserved')
|
|
if clear_maintenance_pause:
|
|
(root/'.runtime/retention/paused').unlink(missing_ok=True)
|
|
print('Installed login startup and continuous guardian:', PLIST)
|
|
|
|
|
|
def status():
|
|
loaded = ctl('print', DOMAIN + '/' + LABEL, check=False)
|
|
state_path = HOME / 'state.json'
|
|
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
output = {'launchdLoaded': loaded.returncode == 0, 'stateFresh': time.time() - state.get('updatedAt', 0) < 20,
|
|
'databaseReady': state.get('databaseReady'), 'backendReady': state.get('backendReady'),
|
|
'supervisorPid': state.get('supervisorPid'), 'starts': state.get('starts')}
|
|
output['processes'] = {k: {'pid': v['pid'], 'confirmed': matches(v)} for k, v in state.get('processes', {}).items()}
|
|
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def stop_services():
|
|
cfg = json.loads(CONFIG.read_text())
|
|
guardian = Guardian(cfg)
|
|
guardian.preflight()
|
|
for role in ('backend', 'mysql'):
|
|
if not guardian.adopt(role):
|
|
continue
|
|
info = guardian.records[role]
|
|
if matches(info) is not True:
|
|
raise RuntimeError('Cannot confirm process identity')
|
|
psutil.Process(info['pid']).terminate()
|
|
deadline = time.monotonic() + 300
|
|
while matches(info) is True and time.monotonic() < deadline:
|
|
time.sleep(.2)
|
|
if matches(info) is not False:
|
|
raise RuntimeError('Graceful shutdown incomplete; no forced termination performed')
|
|
print(role + ' stopped gracefully')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=['install', 'run', 'status', 'start', 'pause', 'stop'])
|
|
parser.add_argument('--root')
|
|
parser.add_argument('--jar')
|
|
args = parser.parse_args()
|
|
if args.action == 'install':
|
|
if not args.root or not args.jar:
|
|
parser.error('install requires --root and --jar')
|
|
install(args.root, args.jar)
|
|
elif args.action == 'status':
|
|
status()
|
|
elif args.action in ('pause', 'stop'):
|
|
cfg = json.loads(CONFIG.read_text())
|
|
atomic(Path(cfg['root'])/'.runtime/retention/paused', {'since':time.time()})
|
|
ctl('disable', DOMAIN + '/' + LABEL)
|
|
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
|
if args.action == 'stop':
|
|
stop_services()
|
|
print('Automatic restart disabled until the start command is used')
|
|
elif args.action == 'start':
|
|
cfg = json.loads(CONFIG.read_text())
|
|
(Path(cfg['root'])/'.runtime/retention/paused').unlink(missing_ok=True)
|
|
ctl('enable', DOMAIN + '/' + LABEL)
|
|
if ctl('print', DOMAIN + '/' + LABEL, check=False).returncode:
|
|
ctl('bootstrap', DOMAIN, str(PLIST))
|
|
else:
|
|
ctl('kickstart', DOMAIN + '/' + LABEL)
|
|
status()
|
|
else:
|
|
logs = HOME / 'logs'
|
|
logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
handler = RotatingFileHandler(logs / 'guardian.log', maxBytes=5*1024*1024, backupCount=4)
|
|
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
|
logging.getLogger().addHandler(handler)
|
|
logging.getLogger().setLevel(logging.INFO)
|
|
Guardian(json.loads(CONFIG.read_text())).run()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|