343 lines
20 KiB
Python
343 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""Assemble native offline packages from reviewed builds and verified downloads."""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
from datetime import datetime, timezone
|
||
import hashlib
|
||
import io
|
||
import json
|
||
from pathlib import Path
|
||
import re
|
||
import shutil
|
||
import stat
|
||
import subprocess
|
||
import tarfile
|
||
import tempfile
|
||
import zipfile
|
||
from repair_windows_cache import cache_source
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
TEMPLATES = Path(__file__).resolve().parent
|
||
|
||
|
||
def strip_runtime_bytecode(root):
|
||
# 目标机可能重建字节码,不能将它当作固定程序资源计算哈希。
|
||
for path in (root / 'runtimes/python').rglob('*.pyc'):
|
||
source = cache_source(path.relative_to(root).as_posix())
|
||
if source:
|
||
if not (root / source).is_file():
|
||
raise RuntimeError('Python 缓存缺少对应源码:' + str(path))
|
||
path.unlink()
|
||
|
||
|
||
def digest(path, algorithm='sha256'):
|
||
with path.open('rb') as stream:
|
||
return hashlib.file_digest(stream, algorithm).hexdigest()
|
||
|
||
|
||
def extract(archive, target):
|
||
target.mkdir(parents=True)
|
||
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
|
||
stage = Path(directory)
|
||
if archive.suffix == '.zip':
|
||
with zipfile.ZipFile(archive) as source:
|
||
source.extractall(stage)
|
||
else:
|
||
with tarfile.open(archive) as source:
|
||
# Terminfo contains case-distinct names. They alias on a Mac's
|
||
# default filesystem and are unused by this noninteractive service.
|
||
members = [member for member in source.getmembers()
|
||
if '/share/terminfo/' not in member.name and not member.name.endswith('/share/terminfo')]
|
||
source.extractall(stage, members=members, filter='data')
|
||
roots = list(stage.iterdir())
|
||
if len(roots) != 1 or not roots[0].is_dir():
|
||
raise RuntimeError('Expected one archive root: ' + archive.name)
|
||
shutil.copytree(roots[0], target, symlinks=True, dirs_exist_ok=True)
|
||
|
||
|
||
def extract_deb(archive, target):
|
||
raw = archive.read_bytes()
|
||
if raw[:8] != b'!<arch>\n':
|
||
raise RuntimeError('Invalid Debian archive')
|
||
position = 8
|
||
while position + 60 <= len(raw):
|
||
header = raw[position:position + 60]
|
||
name = header[:16].decode().strip().rstrip('/')
|
||
length = int(header[48:58].decode().strip())
|
||
position += 60
|
||
data = raw[position:position + length]
|
||
position += length + length % 2
|
||
if name.startswith('data.tar'):
|
||
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
|
||
if name.endswith('.zst'):
|
||
import zstandard
|
||
with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader:
|
||
with tarfile.open(fileobj=reader, mode='r|') as source:
|
||
source.extractall(directory, filter='data')
|
||
else:
|
||
with tarfile.open(fileobj=io.BytesIO(data)) as source:
|
||
source.extractall(directory, filter='data')
|
||
for path in Path(directory).rglob('*'):
|
||
if '.so' in path.name and (path.is_file() or path.is_symlink()):
|
||
dest = target / path.name
|
||
if path.is_symlink():
|
||
dest.symlink_to(Path(path.readlink()).name)
|
||
else:
|
||
shutil.copy2(path, dest)
|
||
docs = Path(directory) / 'usr/share/doc'
|
||
if docs.exists():
|
||
shutil.copytree(docs, target / 'licenses', dirs_exist_ok=True, symlinks=True)
|
||
return
|
||
raise RuntimeError('No data member in ' + archive.name)
|
||
|
||
|
||
def install_wheel(wheel, python_root, windows):
|
||
site = python_root / ('Lib/site-packages' if windows else 'lib/python3.11/site-packages')
|
||
site.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(wheel) as source:
|
||
for member in source.infolist():
|
||
if member.is_dir():
|
||
continue
|
||
relative = Path(member.filename)
|
||
if relative.is_absolute() or '..' in relative.parts:
|
||
raise RuntimeError('Wheel path escape')
|
||
target = site / relative
|
||
if relative.parts[0].endswith('.data'):
|
||
scheme = relative.parts[1]
|
||
prefix = {'purelib': site, 'platlib': site, 'data': python_root,
|
||
'scripts': python_root / ('Scripts' if windows else 'bin'),
|
||
'headers': python_root / 'include'}[scheme]
|
||
target = prefix.joinpath(*relative.parts[2:])
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
with source.open(member) as reader, target.open('wb') as writer:
|
||
shutil.copyfileobj(reader, writer)
|
||
mode = member.external_attr >> 16
|
||
target.chmod((mode & 0o777) or 0o644)
|
||
|
||
|
||
def adapt_runtime(source):
|
||
def replace(old, new, count=None):
|
||
nonlocal source
|
||
actual = source.count(old)
|
||
if not actual or (count is not None and actual != count):
|
||
raise RuntimeError('Runtime source changed; inspect the portability patch: ' + old)
|
||
source = source.replace(old, new)
|
||
|
||
replace('import fcntl\n', 'import runtime_platform as _host\nfrom runtime_platform import fcntl\n', 1)
|
||
replace('PYTHON = VENV / "bin" / "python"',
|
||
'PYTHON = Path(os.environ.get("QUALITY_ASSISTANT_PYTHON", str(VENV / "bin" / "python")))', 1)
|
||
old = 'VENV / "lib" / "python3.11" / "site-packages" / "nao_core"'
|
||
replace(old, 'Path(os.environ.get("QUALITY_ASSISTANT_PACKAGE", str(' + old + ')))', 2)
|
||
replace('/ "nao-chat-server"', '/ ("nao-chat-server.exe" if os.name == "nt" else "nao-chat-server")', 1)
|
||
source, count = re.subn(r'\b(info|directory_info)\.st_mode & 0o077',
|
||
lambda match: '_host.private_bits(' + match[1] + '.st_mode)', source)
|
||
if count != 3:
|
||
raise RuntimeError('Runtime private-permission checks changed; inspect before packaging')
|
||
replace('os.fchmod(', '_host.fchmod(')
|
||
replace('os.getpgid(pid) != pid', 'not _host.group_leader(pid)', 1)
|
||
replace('os.killpg(', '_host.killpg(')
|
||
replace('signal.SIGKILL', 'getattr(signal, "SIGKILL", 9)')
|
||
replace('def _ps_value(pid: int, field: str) -> str | None:\n',
|
||
'def _ps_value(pid: int, field: str) -> str | None:\n'
|
||
' if os.name == "nt":\n return _host.ps_value(pid, field)\n', 1)
|
||
compile(source, 'nao_quality_runtime.py', 'exec')
|
||
return source
|
||
|
||
|
||
def adapt_script(name, source):
|
||
if name == 'nao_quality_runtime.py':
|
||
return adapt_runtime(source)
|
||
if name == 'analysis_visual_evidence.py':
|
||
source = source.replace('import stat\n', 'import stat\nimport runtime_platform as _host\n', 1)
|
||
source = source.replace('os.O_NOFOLLOW', 'getattr(os, "O_NOFOLLOW", 0)')
|
||
source = source.replace('os.O_NONBLOCK', 'getattr(os, "O_NONBLOCK", 0)')
|
||
source = source.replace('before.st_mode & 0o077', '_host.private_bits(before.st_mode)')
|
||
source = source.replace('before.st_uid != os.getuid()', '_host.owner_mismatch(before)')
|
||
source = source.replace(' fd = os.open(path,',
|
||
' if path.is_symlink():\n raise ValueError("invalid evidence link")\n fd = os.open(path,', 1)
|
||
elif name == 'nao_evidence_finalizer.py':
|
||
source = source.replace('import copy\n', 'import copy\nimport runtime_platform as _host\n', 1)
|
||
source = source.replace('path.stat().st_mode & 0o077', '_host.private_bits(path.stat().st_mode)')
|
||
compile(source, name, 'exec')
|
||
return source
|
||
|
||
|
||
def windows_local_runtime(root):
|
||
"""Use the signed vendor JRE's redistributable runtime; never install globally."""
|
||
libraries = ('msvcp140.dll', 'vcruntime140.dll', 'vcruntime140_1.dll')
|
||
for name in libraries:
|
||
source = root / 'runtimes/java/bin' / name
|
||
if not source.is_file():
|
||
raise RuntimeError('The official JRE is missing an app-local Microsoft runtime: ' + name)
|
||
for folder in ('runtimes/mysql/bin', 'runtimes/mysql/lib'):
|
||
shutil.copy2(source, root / folder / name)
|
||
mysql = root / 'runtimes/mysql'
|
||
# Debug-only tools depend on non-redistributable Microsoft debug runtimes.
|
||
for path in sorted(mysql.rglob('*'), reverse=True):
|
||
if path.is_file() and (any(part.lower() == 'debug' for part in path.relative_to(mysql).parts)
|
||
or '-debug.' in path.name.lower()):
|
||
path.unlink()
|
||
elif path.is_dir() and path.name.lower() == 'debug':
|
||
shutil.rmtree(path)
|
||
return {'name': 'app-local-msvc', 'os': 'windows', 'source': 'bundled-official-temurin-jre',
|
||
'libraries': {name: digest(root / 'runtimes/java/bin' / name) for name in libraries}}
|
||
|
||
|
||
def payload(root, work, osname, assets):
|
||
windows = osname == 'windows'
|
||
app = root / 'app'
|
||
app.mkdir(parents=True)
|
||
jars = list((work / 'source/backend/target').glob('*.jar'))
|
||
if len(jars) != 1:
|
||
raise RuntimeError('Expected one freshly built backend JAR')
|
||
with zipfile.ZipFile(jars[0]) as archive:
|
||
if 'BOOT-INF/classes/verification/original-workbook-acceptance.json' in archive.namelist():
|
||
raise RuntimeError('Exclude the private workbook acceptance input before building a distributable JAR')
|
||
shutil.copy2(jars[0], app / 'backend.jar')
|
||
shutil.copytree(work / 'source/web/dist', app / 'web')
|
||
for name in ('java', 'mysql', 'python'):
|
||
kind = 'jre' if name == 'java' else name
|
||
item = next(a for a in assets if a['name'] == kind and a['os'] == osname)
|
||
archive = work / 'downloads' / item['file']
|
||
if digest(archive) != item['sha256']:
|
||
raise RuntimeError('Download checksum mismatch: ' + item['file'])
|
||
extract(archive, root / 'runtimes' / name)
|
||
wheels = sorted((work / ('wheels-' + osname)).glob('*.whl'))
|
||
if len(wheels) < 60:
|
||
raise RuntimeError('The offline Python dependency set is incomplete')
|
||
lock_text = (work / f'requirements-{osname}.lock').read_text()
|
||
for wheel in wheels:
|
||
if '--hash=sha256:' + digest(wheel) not in lock_text:
|
||
raise RuntimeError('Wheel absent from hash lock: ' + wheel.name)
|
||
install_wheel(wheel, root / 'runtimes/python', windows)
|
||
scripts = root / 'scripts'
|
||
scripts.mkdir()
|
||
for pattern in ('nao_*.py', 'analysis_visual*.py'):
|
||
for path in (ROOT / 'scripts').glob(pattern):
|
||
(scripts / path.name).write_text(adapt_script(path.name, path.read_text()), encoding='utf-8')
|
||
for name in ('dbgpt_trial_data.py', 'local_api.py', 'project.py'):
|
||
shutil.copy2(ROOT / 'scripts' / name, scripts / name)
|
||
shutil.copy2(TEMPLATES / 'runtime_platform.py', scripts / 'runtime_platform.py')
|
||
policy = root / 'backend/src/main/resources/analysis-runtime-policy.json'
|
||
policy.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(ROOT / 'backend/src/main/resources/analysis-runtime-policy.json', policy)
|
||
shutil.copytree(ROOT / 'ai/nao-quality', root / 'ai/nao-quality', ignore=shutil.ignore_patterns('.DS_Store'))
|
||
deploy = root / 'deploy/offline'
|
||
deploy.mkdir(parents=True)
|
||
for name in ('control.py', 'gateway.py', 'runtime_platform.py', 'migration_common.py',
|
||
'restore_migration.py', 'update_manager.py', 'update-public-key.pem',
|
||
'windows-start.ps1', 'windows-prepare.ps1', 'README.md'):
|
||
if not windows and name.endswith('.ps1'):
|
||
continue
|
||
source = TEMPLATES / name
|
||
if name.endswith('.ps1'):
|
||
(deploy / name).write_text(source.read_text(), encoding='utf-8-sig')
|
||
else:
|
||
shutil.copy2(source, deploy / name)
|
||
shutil.copy2(work / f'requirements-{osname}.lock', root / 'requirements.lock')
|
||
if windows:
|
||
local_runtime = windows_local_runtime(root)
|
||
shutil.copy2(root / 'runtimes/python/python.exe', root / 'runtimes/python/python3.exe')
|
||
bootstrap = root / 'deploy/bootstrap'
|
||
bootstrap.mkdir()
|
||
shutil.copy2(TEMPLATES / 'update_bootstrap.py', bootstrap / 'update-recovery.py')
|
||
for name in ('windows-start.ps1', 'windows-prepare.ps1'):
|
||
(bootstrap / name).write_text((TEMPLATES / name).read_text(), encoding='utf-8-sig')
|
||
(root / '一键部署启动.cmd').write_bytes(
|
||
b'@echo off\r\nchcp 65001 >nul\r\npowershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0deploy\\bootstrap\\windows-start.ps1"\r\n'
|
||
b'if errorlevel 1 (echo Startup failed. See the message above.)\r\npause\r\n')
|
||
for name, action in (('停止服务.cmd', 'stop'), ('查看状态.cmd', 'status'), ('检查环境.cmd', 'verify')):
|
||
(root / name).write_bytes(('@echo off\r\nchcp 65001 >nul\r\nset PYTHONUTF8=1\r\nset PYTHONIOENCODING=utf-8\r\nset PYTHONDONTWRITEBYTECODE=1\r\n'
|
||
'"%~dp0runtimes\\python\\python.exe" "%~dp0deploy\\offline\\control.py" ' + action + '\r\npause\r\n').encode())
|
||
else:
|
||
libs = root / 'runtimes/linux-libs'
|
||
libs.mkdir()
|
||
for item in assets:
|
||
if item.get('kind') == 'deb':
|
||
archive = work / 'downloads' / item['file']
|
||
if digest(archive) != item['sha256']:
|
||
raise RuntimeError('Debian library checksum mismatch')
|
||
extract_deb(archive, libs)
|
||
for name, action in (('start.sh', 'start'), ('stop.sh', 'stop'), ('status.sh', 'status'), ('verify.sh', 'verify')):
|
||
(root / name).write_text('#!/bin/sh\nset -eu\nROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)\n'
|
||
'if [ "$(id -u)" -eq 0 ]; then echo "Run as a normal user, not root." >&2; exit 1; fi\n'
|
||
'export PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1\n'
|
||
'export LD_LIBRARY_PATH="$ROOT/runtimes/linux-libs:$ROOT/runtimes/mysql/lib:$ROOT/runtimes/mysql/lib/private"\n'
|
||
'exec "$ROOT/runtimes/python/bin/python3" "$ROOT/deploy/offline/control.py" ' + action + ' "$@"\n')
|
||
(root / name).chmod(0o755)
|
||
system = ('Windows 10/11 或 Windows Server 2019/2022/2025,Intel/AMD x64。仅当前用户运行,无需管理员权限。'
|
||
if windows else 'Ubuntu 22.04/24.04、Debian 12/13 的 Intel/AMD x86_64。需要 glibc 2.35 及以上;不支持 Alpine/musl、ARM 或旧版 CentOS。')
|
||
usage = ('解压到当前用户的 %LOCALAPPDATA%\\QualityCenter 等较短的本地 NTFS 路径,双击“一键部署启动.cmd”。不要在压缩包中直接打开。\n'
|
||
'运行库随包提供;不安装系统服务,不修改系统环境变量、注册表或防火墙,不申请管理员权限。\n'
|
||
'启动后自动打开浏览器,命令窗口显示本机和内网地址。关闭此窗口不会停止服务。\n'
|
||
'其他电脑连接同一内网后,用浏览器打开显示的 http://内网IP:5280 即可。\n'
|
||
'停止服务请双击“停止服务.cmd”;重新启动仍双击原入口。\n' if windows else
|
||
'解压后以普通用户运行 ./start.sh;内网共享运行 ./start.sh --lan。\n'
|
||
'同事用浏览器打开显示的 http://内网IP:5280;如系统有防火墙,仅对可信内网放行该端口。\n'
|
||
'停止运行 ./stop.sh,查看状态运行 ./status.sh,检查完整性运行 ./verify.sh。\n')
|
||
(root / '使用说明.txt').write_text('质量中心离线部署包\n\n系统要求\n' + system + '\n建议内存 8GB 以上,至少预留 5GB 磁盘及后续业务数据空间。\n\n'
|
||
+ usage + '\n登录信息保存在 runtimedata/登录信息.txt,初始管理员密码在本机随机生成。登录后请修改密码。\n'
|
||
'首次数据库为空,不包含原电脑的用户、业务数据、来源文件和模型密钥。通过网页上传授权的数据文件。\n'
|
||
'普通功能和安装不需要外网;AI 在线模型调用需要网络,或自行配置内网模型。\n'
|
||
'不要在服务运行时移动目录,不要覆盖 .runtime 和 runtimedata。数据迁移使用逻辑备份,不能复制 macOS MySQL 原始数据目录。\n'
|
||
'内网访问需要主机保持开机、服务运行、网络互通;本机 MySQL 和内部 API 端口不对内网开放。\n'
|
||
'默认网页端口 5280。更改 .runtime/deployment.json 中的 webPort 后重新运行入口。内网连接受现有防火墙规则限制,程序不会自动更改这些规则。\n'
|
||
'详细技术说明见 deploy/offline/README.md。\n\n'
|
||
'验证范围:前后端隔离构建及本机自动检查已完成;Windows/Linux 原生首次启动和跨电脑内网访问需要在目标机实测。\n', encoding='utf-8-sig')
|
||
strip_runtime_bytecode(root)
|
||
metadata = {'platform': osname, 'architecture': 'x86_64', 'builtAtUtc': datetime.now(timezone.utc).isoformat(),
|
||
'updateVersion': '2026.11.04.1', 'updateSequence': 1, 'installationScope': 'current-user', 'requiresAdministrator': False,
|
||
'databaseIncluded': False, 'credentialsIncluded': False, 'requiresDownloadAtInstall': False,
|
||
'nativeTargetSmokeTest': 'not-run-on-macos-host',
|
||
'runtimes': [{k: v for k, v in a.items() if k not in ('kind',)} for a in assets if a['os'] == osname and a['name'] != 'vcredist'] + ([local_runtime] if windows else []),
|
||
'pythonWheels': [{'file': p.name, 'sha256': digest(p)} for p in wheels], 'files': []}
|
||
for path in sorted(root.rglob('*')):
|
||
if path.is_symlink():
|
||
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'symlink': str(path.readlink())})
|
||
elif path.is_file():
|
||
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': digest(path)})
|
||
(root / 'manifest.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||
return metadata
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--work', type=Path, required=True)
|
||
parser.add_argument('--platform', choices=('windows', 'linux', 'both'), default='both')
|
||
parser.add_argument('--assemble-only', action='store_true')
|
||
args = parser.parse_args()
|
||
work = args.work.resolve()
|
||
assets = json.loads((work / 'downloads/assets.json').read_text())
|
||
platforms = ('windows', 'linux') if args.platform == 'both' else (args.platform,)
|
||
summaries = []
|
||
for osname in platforms:
|
||
name = 'QualityCenter-' + ('Windows' if osname == 'windows' else 'Linux') + '-x64'
|
||
directory = work / name
|
||
if directory.exists():
|
||
raise RuntimeError('Preserving an existing bundle; choose an empty work directory: ' + str(directory))
|
||
print('Assembling ' + name, flush=True)
|
||
metadata = payload(directory, work, osname, assets)
|
||
print('Files verified: ' + str(len(metadata['files'])), flush=True)
|
||
if args.assemble_only:
|
||
continue
|
||
if osname == 'windows':
|
||
archive = work / (name + '.zip')
|
||
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as target:
|
||
for path in sorted(directory.rglob('*')):
|
||
if path.is_file():
|
||
target.write(path, path.relative_to(work).as_posix())
|
||
else:
|
||
archive = work / (name + '.tar.gz')
|
||
with tarfile.open(archive, 'w:gz', compresslevel=6) as target:
|
||
target.add(directory, arcname=name)
|
||
summaries.append({'file': archive.name, 'bytes': archive.stat().st_size, 'sha256': digest(archive)})
|
||
print(json.dumps(summaries[-1]), flush=True)
|
||
if summaries:
|
||
(work / 'SHA256SUMS.txt').write_text(''.join(a['sha256'] + ' ' + a['file'] + '\n' for a in summaries))
|
||
(work / 'packages.json').write_text(json.dumps(summaries, indent=2) + '\n')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|