新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pinned macOS service guardian. Never initializes or imports a database."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import plistlib
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
import psutil
|
||||
|
||||
LABEL = 'com.lyunlab.quality-center-service'
|
||||
HOME = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||
CONFIG = HOME / 'service.json'
|
||||
PLIST = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
||||
DOMAIN = 'gui/' + str(os.getuid())
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = path.with_name(path.name + '.tmp-' + str(os.getpid()))
|
||||
with temporary.open('w') as stream:
|
||||
os.chmod(temporary, 0o600)
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def stamp(pid):
|
||||
return subprocess.run(['/bin/ps', '-p', str(pid), '-o', 'lstart='],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def matches(record, command=None):
|
||||
"""True=confirmed identity, False=gone/reused, None=unknown identity."""
|
||||
if not record:
|
||||
return False
|
||||
try:
|
||||
p = psutil.Process(record['pid'])
|
||||
if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
|
||||
return False
|
||||
if 'created' in record and abs(p.create_time() - record['created']) > .01:
|
||||
return False
|
||||
if 'started' in record and stamp(p.pid) != record['started']:
|
||||
return False
|
||||
expected = command or record['command']
|
||||
if p.uids().real != os.getuid():
|
||||
return None
|
||||
if Path(p.exe()).resolve() != Path(expected[0]).resolve() or p.cmdline() != expected:
|
||||
return None
|
||||
return True
|
||||
except psutil.NoSuchProcess:
|
||||
return False
|
||||
except (psutil.Error, KeyError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def record(pid, command):
|
||||
return {'pid': pid, 'created': psutil.Process(pid).create_time(),
|
||||
'command': command, 'started': stamp(pid)}
|
||||
|
||||
|
||||
class Guardian:
|
||||
def __init__(self, config, home=HOME):
|
||||
self.config = config
|
||||
self.home = Path(home)
|
||||
self.root = Path(config['root']).resolve()
|
||||
self.runtime = self.root / '.runtime'
|
||||
self.records = {}
|
||||
self.children = {}
|
||||
self.next_start = {'mysql': 0, 'backend': 0}
|
||||
self.restarts = {'mysql': 0, 'backend': 0}
|
||||
self.stopping = False
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def preflight(self):
|
||||
self.value = json.loads((self.runtime / 'local-config.json').read_text())
|
||||
if not self.value.get('databaseReady') or not (self.runtime / 'mysql-data/auto.cnf').is_file():
|
||||
raise RuntimeError('Existing initialized database not confirmed; refusing bootstrap')
|
||||
self.jar = self.runtime / 'backend-builds' / (self.config['jarSha256'] + '.jar')
|
||||
if hashlib.sha256(self.jar.read_bytes()).hexdigest() != self.config['jarSha256']:
|
||||
raise RuntimeError('Accepted backend digest mismatch')
|
||||
for binary in [self.runtime / 'mysql/bin/mysqld', self.runtime / 'jdk/Contents/Home/bin/java']:
|
||||
if not os.access(binary, os.X_OK):
|
||||
raise RuntimeError('Required runtime missing')
|
||||
|
||||
def command(self, role):
|
||||
r = self.runtime
|
||||
if role == 'backend':
|
||||
return [str(r / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx2048m', '-jar', str(self.jar)]
|
||||
return [str(r / 'mysql/bin/mysqld'), '--no-defaults', '--basedir=' + str(r / 'mysql'),
|
||||
'--datadir=' + str(r / 'mysql-data'), '--port=' + str(self.value['mysqlPort']),
|
||||
'--bind-address=127.0.0.1', '--socket=' + str(r / 'mysql.sock'),
|
||||
'--pid-file=' + str(r / 'mysql-server.pid'),
|
||||
'--log-error=' + str(self.root / 'output/logs/mysql-server.log'), '--mysqlx=OFF',
|
||||
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M',
|
||||
*(['--skip-log-bin'] if self.value.get('binaryLogEnabled', True) is False else [])]
|
||||
|
||||
def environment(self):
|
||||
v, r = self.value, self.runtime
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
'JAVA_HOME': str(r / 'jdk/Contents/Home'),
|
||||
'PATH': str(r / 'jdk/Contents/Home/bin') + ':' + self.config['path'],
|
||||
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{v["mysqlPort"]}/{v["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
|
||||
'MYSQL_USER': v['mysqlUser'], 'MYSQL_PASSWORD': v['mysqlPassword'],
|
||||
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': str(v.get('legacyTimestampOffsetSeconds', 'UNCONFIRMED')),
|
||||
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': v['adminUsername'],
|
||||
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': v['adminPassword'],
|
||||
'QUALITY_ARCHIVE_ROOT': str(self.root / 'runtimedata/archive'),
|
||||
'QUALITY_ALLOWED_ROOTS': str(self.root / '质量数据/系统原始数据示例'),
|
||||
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
|
||||
'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED': str(v.get('analysisProtocolV1Enabled', False)).lower(),
|
||||
'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED': str(v.get('analysisAutonomousEnabled', False)).lower(),
|
||||
'QUALITY_PROJECT_ROOT': str(self.root), 'QUALITY_DEFAULT_SOURCE_YEAR': '2026',
|
||||
'SERVER_ADDRESS': '127.0.0.1', 'PORT': '5281', 'PYTHONDONTWRITEBYTECODE': '1',
|
||||
})
|
||||
log_config = r / 'operations/logback-quality.xml'
|
||||
if log_config.is_file() and not log_config.is_symlink():
|
||||
env['LOGGING_CONFIG'] = str(log_config)
|
||||
env['QUALITY_LOG_DIR'] = str(self.root / 'output/logs')
|
||||
return env
|
||||
|
||||
def port(self, role):
|
||||
return self.value['mysqlPort'] if role == 'mysql' else 5281
|
||||
|
||||
def occupied(self, role):
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', self.port(role)), timeout=.5):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def db_ready(self):
|
||||
result = subprocess.run([str(self.runtime / 'mysql/bin/mysql'),
|
||||
'--defaults-file=' + str(self.runtime / 'mysql-root.cnf'), '--connect-timeout=2',
|
||||
'--batch', '--skip-column-names', '-e', 'SELECT 1;'],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
|
||||
return result.returncode == 0 and result.stdout.strip() == b'1'
|
||||
|
||||
def backend_ready(self):
|
||||
try:
|
||||
with self.opener.open('http://127.0.0.1:5281/actuator/health', timeout=2) as response:
|
||||
return json.load(response).get('status') == 'UP'
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
def adopt(self, role):
|
||||
path = self.runtime / (role + '.process.json')
|
||||
if not path.exists():
|
||||
return False
|
||||
info = json.loads(path.read_text())
|
||||
identity = matches(info, self.command(role))
|
||||
if identity is None:
|
||||
raise RuntimeError(role + ': existing process identity cannot be confirmed')
|
||||
if identity:
|
||||
self.records[role] = record(info['pid'], self.command(role))
|
||||
logging.info('%s adopted pid=%s', role, info['pid'])
|
||||
return bool(identity)
|
||||
|
||||
def ensure(self, role):
|
||||
child = self.children.get(role)
|
||||
if child is not None:
|
||||
child.poll() # Reap exited children before checking PID identity.
|
||||
info = self.records.get(role)
|
||||
state = matches(info, self.command(role))
|
||||
if state is None:
|
||||
raise RuntimeError(role + ': process identity is unknown; no replacement started')
|
||||
if state:
|
||||
return True
|
||||
if info:
|
||||
self.records.pop(role)
|
||||
self.next_start[role] = time.monotonic() + 10
|
||||
logging.warning('%s exited; restart scheduled in 10 seconds', role)
|
||||
if time.monotonic() < self.next_start[role]:
|
||||
return False
|
||||
if self.adopt(role):
|
||||
return True
|
||||
if self.occupied(role):
|
||||
raise RuntimeError(role + ': port occupied by an unconfirmed process')
|
||||
command = self.command(role)
|
||||
logs = self.root / 'output/logs'
|
||||
logs.mkdir(parents=True, exist_ok=True)
|
||||
log_name = ('backend-console.log' if role == 'backend' and
|
||||
(self.runtime / 'operations/logback-quality.xml').is_file() else role + '.log')
|
||||
with (logs / log_name).open('ab') as log:
|
||||
child = subprocess.Popen(command, cwd=self.root, env=self.environment(),
|
||||
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
||||
self.children[role] = child
|
||||
self.records[role] = info = record(child.pid, command)
|
||||
(self.runtime / (role + '.pid')).write_text(str(child.pid))
|
||||
atomic(self.runtime / (role + '.process.json'), {'pid': child.pid, 'started': info['started']})
|
||||
self.restarts[role] += 1
|
||||
logging.info('%s started pid=%s', role, child.pid)
|
||||
return True
|
||||
|
||||
def tick(self):
|
||||
mysql_alive = self.ensure('mysql')
|
||||
ready = mysql_alive and self.db_ready()
|
||||
if ready:
|
||||
self.ensure('backend')
|
||||
atomic(self.home / 'state.json', {'updatedAt': time.time(), 'supervisorPid': os.getpid(),
|
||||
'processes': self.records, 'starts': self.restarts,
|
||||
'databaseReady': bool(ready), 'backendReady': self.backend_ready()})
|
||||
|
||||
def run(self):
|
||||
self.preflight()
|
||||
with (self.runtime / 'quality-service.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: setattr(self, 'stopping', True))
|
||||
logging.info('guardian started; accepted backend=%s', self.config['jarSha256'])
|
||||
while not self.stopping:
|
||||
try:
|
||||
self.tick()
|
||||
except Exception as error:
|
||||
# Never log exception arguments: database credentials may be embedded.
|
||||
detail = str(error) if type(error) is RuntimeError else type(error).__name__
|
||||
logging.error('check failed: %s; leaving existing processes intact', detail)
|
||||
atomic(self.home / 'error.json', {'time': time.time(), 'type': type(error).__name__})
|
||||
for _ in range(30):
|
||||
if self.stopping:
|
||||
break
|
||||
time.sleep(.1)
|
||||
logging.info('guardian stopped; service processes remain available for maintenance')
|
||||
|
||||
|
||||
def ctl(*args, check=True):
|
||||
return subprocess.run(['/bin/launchctl', *args], check=check, capture_output=True, text=True)
|
||||
|
||||
|
||||
def install(root, jar, clear_maintenance_pause=True):
|
||||
root, jar = Path(root).resolve(), Path(jar).resolve()
|
||||
cfg = {'root': str(root), 'jarSha256': hashlib.sha256(jar.read_bytes()).hexdigest(),
|
||||
'path': '/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.13/bin:/usr/bin:/bin:/usr/sbin:/sbin'}
|
||||
Guardian(cfg).preflight()
|
||||
HOME.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
HOME.chmod(0o700)
|
||||
script = Path(__file__).resolve()
|
||||
digest = hashlib.sha256(script.read_bytes()).hexdigest()
|
||||
target = HOME / 'versions' / digest / 'service.py'
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
shutil.copyfile(script, target)
|
||||
target.chmod(0o600)
|
||||
# Remove only the previous guardian; adopted MySQL/backend processes keep running.
|
||||
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
||||
atomic(CONFIG, cfg)
|
||||
logs = HOME / 'logs'
|
||||
logs.mkdir(mode=0o700, exist_ok=True)
|
||||
plist = {'Label': LABEL, 'ProgramArguments': [str(Path(sys.executable).resolve()), str(target), 'run'],
|
||||
'WorkingDirectory': str(HOME), 'RunAtLoad': True, 'KeepAlive': True,
|
||||
'ThrottleInterval': 10, 'ProcessType': 'Background', 'ExitTimeOut': 30,
|
||||
'AbandonProcessGroup': True,
|
||||
'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
||||
'StandardOutPath': str(logs / 'launchd.log'), 'StandardErrorPath': str(logs / 'launchd-error.log')}
|
||||
PLIST.parent.mkdir(exist_ok=True)
|
||||
with PLIST.open('wb') as stream:
|
||||
plistlib.dump(plist, stream)
|
||||
PLIST.chmod(0o600)
|
||||
ctl('enable', DOMAIN + '/' + LABEL)
|
||||
for attempt in range(50):
|
||||
loaded=ctl('bootstrap', DOMAIN, str(PLIST), check=False)
|
||||
if loaded.returncode==0:break
|
||||
time.sleep(.2)
|
||||
else:raise RuntimeError('Service registration did not complete; existing processes preserved')
|
||||
if clear_maintenance_pause:
|
||||
(root/'.runtime/retention/paused').unlink(missing_ok=True)
|
||||
print('Installed login startup and continuous guardian:', PLIST)
|
||||
|
||||
|
||||
def status():
|
||||
loaded = ctl('print', DOMAIN + '/' + LABEL, check=False)
|
||||
state_path = HOME / 'state.json'
|
||||
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
output = {'launchdLoaded': loaded.returncode == 0, 'stateFresh': time.time() - state.get('updatedAt', 0) < 20,
|
||||
'databaseReady': state.get('databaseReady'), 'backendReady': state.get('backendReady'),
|
||||
'supervisorPid': state.get('supervisorPid'), 'starts': state.get('starts')}
|
||||
output['processes'] = {k: {'pid': v['pid'], 'confirmed': matches(v)} for k, v in state.get('processes', {}).items()}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def stop_services():
|
||||
cfg = json.loads(CONFIG.read_text())
|
||||
guardian = Guardian(cfg)
|
||||
guardian.preflight()
|
||||
for role in ('backend', 'mysql'):
|
||||
if not guardian.adopt(role):
|
||||
continue
|
||||
info = guardian.records[role]
|
||||
if matches(info) is not True:
|
||||
raise RuntimeError('Cannot confirm process identity')
|
||||
psutil.Process(info['pid']).terminate()
|
||||
deadline = time.monotonic() + 300
|
||||
while matches(info) is True and time.monotonic() < deadline:
|
||||
time.sleep(.2)
|
||||
if matches(info) is not False:
|
||||
raise RuntimeError('Graceful shutdown incomplete; no forced termination performed')
|
||||
print(role + ' stopped gracefully')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['install', 'run', 'status', 'start', 'pause', 'stop'])
|
||||
parser.add_argument('--root')
|
||||
parser.add_argument('--jar')
|
||||
args = parser.parse_args()
|
||||
if args.action == 'install':
|
||||
if not args.root or not args.jar:
|
||||
parser.error('install requires --root and --jar')
|
||||
install(args.root, args.jar)
|
||||
elif args.action == 'status':
|
||||
status()
|
||||
elif args.action in ('pause', 'stop'):
|
||||
cfg = json.loads(CONFIG.read_text())
|
||||
atomic(Path(cfg['root'])/'.runtime/retention/paused', {'since':time.time()})
|
||||
ctl('disable', DOMAIN + '/' + LABEL)
|
||||
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
||||
if args.action == 'stop':
|
||||
stop_services()
|
||||
print('Automatic restart disabled until the start command is used')
|
||||
elif args.action == 'start':
|
||||
cfg = json.loads(CONFIG.read_text())
|
||||
(Path(cfg['root'])/'.runtime/retention/paused').unlink(missing_ok=True)
|
||||
ctl('enable', DOMAIN + '/' + LABEL)
|
||||
if ctl('print', DOMAIN + '/' + LABEL, check=False).returncode:
|
||||
ctl('bootstrap', DOMAIN, str(PLIST))
|
||||
else:
|
||||
ctl('kickstart', DOMAIN + '/' + LABEL)
|
||||
status()
|
||||
else:
|
||||
logs = HOME / 'logs'
|
||||
logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
handler = RotatingFileHandler(logs / 'guardian.log', maxBytes=5*1024*1024, backupCount=4)
|
||||
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||
logging.getLogger().addHandler(handler)
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
Guardian(json.loads(CONFIG.read_text())).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user