Files

343 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Assemble native offline packages from reviewed builds and verified downloads."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import hashlib
import io
import json
from pathlib import Path
import re
import shutil
import stat
import subprocess
import tarfile
import tempfile
import zipfile
from repair_windows_cache import cache_source
ROOT = Path(__file__).resolve().parents[2]
TEMPLATES = Path(__file__).resolve().parent
def strip_runtime_bytecode(root):
# 目标机可能重建字节码,不能将它当作固定程序资源计算哈希。
for path in (root / 'runtimes/python').rglob('*.pyc'):
source = cache_source(path.relative_to(root).as_posix())
if source:
if not (root / source).is_file():
raise RuntimeError('Python 缓存缺少对应源码:' + str(path))
path.unlink()
def digest(path, algorithm='sha256'):
with path.open('rb') as stream:
return hashlib.file_digest(stream, algorithm).hexdigest()
def extract(archive, target):
target.mkdir(parents=True)
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
stage = Path(directory)
if archive.suffix == '.zip':
with zipfile.ZipFile(archive) as source:
source.extractall(stage)
else:
with tarfile.open(archive) as source:
# Terminfo contains case-distinct names. They alias on a Mac's
# default filesystem and are unused by this noninteractive service.
members = [member for member in source.getmembers()
if '/share/terminfo/' not in member.name and not member.name.endswith('/share/terminfo')]
source.extractall(stage, members=members, filter='data')
roots = list(stage.iterdir())
if len(roots) != 1 or not roots[0].is_dir():
raise RuntimeError('Expected one archive root: ' + archive.name)
shutil.copytree(roots[0], target, symlinks=True, dirs_exist_ok=True)
def extract_deb(archive, target):
raw = archive.read_bytes()
if raw[:8] != b'!<arch>\n':
raise RuntimeError('Invalid Debian archive')
position = 8
while position + 60 <= len(raw):
header = raw[position:position + 60]
name = header[:16].decode().strip().rstrip('/')
length = int(header[48:58].decode().strip())
position += 60
data = raw[position:position + length]
position += length + length % 2
if name.startswith('data.tar'):
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
if name.endswith('.zst'):
import zstandard
with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader:
with tarfile.open(fileobj=reader, mode='r|') as source:
source.extractall(directory, filter='data')
else:
with tarfile.open(fileobj=io.BytesIO(data)) as source:
source.extractall(directory, filter='data')
for path in Path(directory).rglob('*'):
if '.so' in path.name and (path.is_file() or path.is_symlink()):
dest = target / path.name
if path.is_symlink():
dest.symlink_to(Path(path.readlink()).name)
else:
shutil.copy2(path, dest)
docs = Path(directory) / 'usr/share/doc'
if docs.exists():
shutil.copytree(docs, target / 'licenses', dirs_exist_ok=True, symlinks=True)
return
raise RuntimeError('No data member in ' + archive.name)
def install_wheel(wheel, python_root, windows):
site = python_root / ('Lib/site-packages' if windows else 'lib/python3.11/site-packages')
site.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(wheel) as source:
for member in source.infolist():
if member.is_dir():
continue
relative = Path(member.filename)
if relative.is_absolute() or '..' in relative.parts:
raise RuntimeError('Wheel path escape')
target = site / relative
if relative.parts[0].endswith('.data'):
scheme = relative.parts[1]
prefix = {'purelib': site, 'platlib': site, 'data': python_root,
'scripts': python_root / ('Scripts' if windows else 'bin'),
'headers': python_root / 'include'}[scheme]
target = prefix.joinpath(*relative.parts[2:])
target.parent.mkdir(parents=True, exist_ok=True)
with source.open(member) as reader, target.open('wb') as writer:
shutil.copyfileobj(reader, writer)
mode = member.external_attr >> 16
target.chmod((mode & 0o777) or 0o644)
def adapt_runtime(source):
def replace(old, new, count=None):
nonlocal source
actual = source.count(old)
if not actual or (count is not None and actual != count):
raise RuntimeError('Runtime source changed; inspect the portability patch: ' + old)
source = source.replace(old, new)
replace('import fcntl\n', 'import runtime_platform as _host\nfrom runtime_platform import fcntl\n', 1)
replace('PYTHON = VENV / "bin" / "python"',
'PYTHON = Path(os.environ.get("QUALITY_ASSISTANT_PYTHON", str(VENV / "bin" / "python")))', 1)
old = 'VENV / "lib" / "python3.11" / "site-packages" / "nao_core"'
replace(old, 'Path(os.environ.get("QUALITY_ASSISTANT_PACKAGE", str(' + old + ')))', 2)
replace('/ "nao-chat-server"', '/ ("nao-chat-server.exe" if os.name == "nt" else "nao-chat-server")', 1)
source, count = re.subn(r'\b(info|directory_info)\.st_mode & 0o077',
lambda match: '_host.private_bits(' + match[1] + '.st_mode)', source)
if count != 3:
raise RuntimeError('Runtime private-permission checks changed; inspect before packaging')
replace('os.fchmod(', '_host.fchmod(')
replace('os.getpgid(pid) != pid', 'not _host.group_leader(pid)', 1)
replace('os.killpg(', '_host.killpg(')
replace('signal.SIGKILL', 'getattr(signal, "SIGKILL", 9)')
replace('def _ps_value(pid: int, field: str) -> str | None:\n',
'def _ps_value(pid: int, field: str) -> str | None:\n'
' if os.name == "nt":\n return _host.ps_value(pid, field)\n', 1)
compile(source, 'nao_quality_runtime.py', 'exec')
return source
def adapt_script(name, source):
if name == 'nao_quality_runtime.py':
return adapt_runtime(source)
if name == 'analysis_visual_evidence.py':
source = source.replace('import stat\n', 'import stat\nimport runtime_platform as _host\n', 1)
source = source.replace('os.O_NOFOLLOW', 'getattr(os, "O_NOFOLLOW", 0)')
source = source.replace('os.O_NONBLOCK', 'getattr(os, "O_NONBLOCK", 0)')
source = source.replace('before.st_mode & 0o077', '_host.private_bits(before.st_mode)')
source = source.replace('before.st_uid != os.getuid()', '_host.owner_mismatch(before)')
source = source.replace(' fd = os.open(path,',
' if path.is_symlink():\n raise ValueError("invalid evidence link")\n fd = os.open(path,', 1)
elif name == 'nao_evidence_finalizer.py':
source = source.replace('import copy\n', 'import copy\nimport runtime_platform as _host\n', 1)
source = source.replace('path.stat().st_mode & 0o077', '_host.private_bits(path.stat().st_mode)')
compile(source, name, 'exec')
return source
def windows_local_runtime(root):
"""Use the signed vendor JRE's redistributable runtime; never install globally."""
libraries = ('msvcp140.dll', 'vcruntime140.dll', 'vcruntime140_1.dll')
for name in libraries:
source = root / 'runtimes/java/bin' / name
if not source.is_file():
raise RuntimeError('The official JRE is missing an app-local Microsoft runtime: ' + name)
for folder in ('runtimes/mysql/bin', 'runtimes/mysql/lib'):
shutil.copy2(source, root / folder / name)
mysql = root / 'runtimes/mysql'
# Debug-only tools depend on non-redistributable Microsoft debug runtimes.
for path in sorted(mysql.rglob('*'), reverse=True):
if path.is_file() and (any(part.lower() == 'debug' for part in path.relative_to(mysql).parts)
or '-debug.' in path.name.lower()):
path.unlink()
elif path.is_dir() and path.name.lower() == 'debug':
shutil.rmtree(path)
return {'name': 'app-local-msvc', 'os': 'windows', 'source': 'bundled-official-temurin-jre',
'libraries': {name: digest(root / 'runtimes/java/bin' / name) for name in libraries}}
def payload(root, work, osname, assets):
windows = osname == 'windows'
app = root / 'app'
app.mkdir(parents=True)
jars = list((work / 'source/backend/target').glob('*.jar'))
if len(jars) != 1:
raise RuntimeError('Expected one freshly built backend JAR')
with zipfile.ZipFile(jars[0]) as archive:
if 'BOOT-INF/classes/verification/original-workbook-acceptance.json' in archive.namelist():
raise RuntimeError('Exclude the private workbook acceptance input before building a distributable JAR')
shutil.copy2(jars[0], app / 'backend.jar')
shutil.copytree(work / 'source/web/dist', app / 'web')
for name in ('java', 'mysql', 'python'):
kind = 'jre' if name == 'java' else name
item = next(a for a in assets if a['name'] == kind and a['os'] == osname)
archive = work / 'downloads' / item['file']
if digest(archive) != item['sha256']:
raise RuntimeError('Download checksum mismatch: ' + item['file'])
extract(archive, root / 'runtimes' / name)
wheels = sorted((work / ('wheels-' + osname)).glob('*.whl'))
if len(wheels) < 60:
raise RuntimeError('The offline Python dependency set is incomplete')
lock_text = (work / f'requirements-{osname}.lock').read_text()
for wheel in wheels:
if '--hash=sha256:' + digest(wheel) not in lock_text:
raise RuntimeError('Wheel absent from hash lock: ' + wheel.name)
install_wheel(wheel, root / 'runtimes/python', windows)
scripts = root / 'scripts'
scripts.mkdir()
for pattern in ('nao_*.py', 'analysis_visual*.py'):
for path in (ROOT / 'scripts').glob(pattern):
(scripts / path.name).write_text(adapt_script(path.name, path.read_text()), encoding='utf-8')
for name in ('dbgpt_trial_data.py', 'local_api.py', 'project.py'):
shutil.copy2(ROOT / 'scripts' / name, scripts / name)
shutil.copy2(TEMPLATES / 'runtime_platform.py', scripts / 'runtime_platform.py')
policy = root / 'backend/src/main/resources/analysis-runtime-policy.json'
policy.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(ROOT / 'backend/src/main/resources/analysis-runtime-policy.json', policy)
shutil.copytree(ROOT / 'ai/nao-quality', root / 'ai/nao-quality', ignore=shutil.ignore_patterns('.DS_Store'))
deploy = root / 'deploy/offline'
deploy.mkdir(parents=True)
for name in ('control.py', 'gateway.py', 'runtime_platform.py', 'migration_common.py',
'restore_migration.py', 'update_manager.py', 'update-public-key.pem',
'windows-start.ps1', 'windows-prepare.ps1', 'README.md'):
if not windows and name.endswith('.ps1'):
continue
source = TEMPLATES / name
if name.endswith('.ps1'):
(deploy / name).write_text(source.read_text(), encoding='utf-8-sig')
else:
shutil.copy2(source, deploy / name)
shutil.copy2(work / f'requirements-{osname}.lock', root / 'requirements.lock')
if windows:
local_runtime = windows_local_runtime(root)
shutil.copy2(root / 'runtimes/python/python.exe', root / 'runtimes/python/python3.exe')
bootstrap = root / 'deploy/bootstrap'
bootstrap.mkdir()
shutil.copy2(TEMPLATES / 'update_bootstrap.py', bootstrap / 'update-recovery.py')
for name in ('windows-start.ps1', 'windows-prepare.ps1'):
(bootstrap / name).write_text((TEMPLATES / name).read_text(), encoding='utf-8-sig')
(root / '一键部署启动.cmd').write_bytes(
b'@echo off\r\nchcp 65001 >nul\r\npowershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0deploy\\bootstrap\\windows-start.ps1"\r\n'
b'if errorlevel 1 (echo Startup failed. See the message above.)\r\npause\r\n')
for name, action in (('停止服务.cmd', 'stop'), ('查看状态.cmd', 'status'), ('检查环境.cmd', 'verify')):
(root / name).write_bytes(('@echo off\r\nchcp 65001 >nul\r\nset PYTHONUTF8=1\r\nset PYTHONIOENCODING=utf-8\r\nset PYTHONDONTWRITEBYTECODE=1\r\n'
'"%~dp0runtimes\\python\\python.exe" "%~dp0deploy\\offline\\control.py" ' + action + '\r\npause\r\n').encode())
else:
libs = root / 'runtimes/linux-libs'
libs.mkdir()
for item in assets:
if item.get('kind') == 'deb':
archive = work / 'downloads' / item['file']
if digest(archive) != item['sha256']:
raise RuntimeError('Debian library checksum mismatch')
extract_deb(archive, libs)
for name, action in (('start.sh', 'start'), ('stop.sh', 'stop'), ('status.sh', 'status'), ('verify.sh', 'verify')):
(root / name).write_text('#!/bin/sh\nset -eu\nROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)\n'
'if [ "$(id -u)" -eq 0 ]; then echo "Run as a normal user, not root." >&2; exit 1; fi\n'
'export PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1\n'
'export LD_LIBRARY_PATH="$ROOT/runtimes/linux-libs:$ROOT/runtimes/mysql/lib:$ROOT/runtimes/mysql/lib/private"\n'
'exec "$ROOT/runtimes/python/bin/python3" "$ROOT/deploy/offline/control.py" ' + action + ' "$@"\n')
(root / name).chmod(0o755)
system = ('Windows 10/11 或 Windows Server 2019/2022/2025,Intel/AMD x64。仅当前用户运行,无需管理员权限。'
if windows else 'Ubuntu 22.04/24.04、Debian 12/13 的 Intel/AMD x86_64。需要 glibc 2.35 及以上;不支持 Alpine/musl、ARM 或旧版 CentOS。')
usage = ('解压到当前用户的 %LOCALAPPDATA%\\QualityCenter 等较短的本地 NTFS 路径,双击“一键部署启动.cmd”。不要在压缩包中直接打开。\n'
'运行库随包提供;不安装系统服务,不修改系统环境变量、注册表或防火墙,不申请管理员权限。\n'
'启动后自动打开浏览器,命令窗口显示本机和内网地址。关闭此窗口不会停止服务。\n'
'其他电脑连接同一内网后,用浏览器打开显示的 http://内网IP:5280 即可。\n'
'停止服务请双击“停止服务.cmd”;重新启动仍双击原入口。\n' if windows else
'解压后以普通用户运行 ./start.sh;内网共享运行 ./start.sh --lan。\n'
'同事用浏览器打开显示的 http://内网IP:5280;如系统有防火墙,仅对可信内网放行该端口。\n'
'停止运行 ./stop.sh,查看状态运行 ./status.sh,检查完整性运行 ./verify.sh。\n')
(root / '使用说明.txt').write_text('质量中心离线部署包\n\n系统要求\n' + system + '\n建议内存 8GB 以上,至少预留 5GB 磁盘及后续业务数据空间。\n\n'
+ usage + '\n登录信息保存在 runtimedata/登录信息.txt,初始管理员密码在本机随机生成。登录后请修改密码。\n'
'首次数据库为空,不包含原电脑的用户、业务数据、来源文件和模型密钥。通过网页上传授权的数据文件。\n'
'普通功能和安装不需要外网;AI 在线模型调用需要网络,或自行配置内网模型。\n'
'不要在服务运行时移动目录,不要覆盖 .runtime 和 runtimedata。数据迁移使用逻辑备份,不能复制 macOS MySQL 原始数据目录。\n'
'内网访问需要主机保持开机、服务运行、网络互通;本机 MySQL 和内部 API 端口不对内网开放。\n'
'默认网页端口 5280。更改 .runtime/deployment.json 中的 webPort 后重新运行入口。内网连接受现有防火墙规则限制,程序不会自动更改这些规则。\n'
'详细技术说明见 deploy/offline/README.md。\n\n'
'验证范围:前后端隔离构建及本机自动检查已完成;Windows/Linux 原生首次启动和跨电脑内网访问需要在目标机实测。\n', encoding='utf-8-sig')
strip_runtime_bytecode(root)
metadata = {'platform': osname, 'architecture': 'x86_64', 'builtAtUtc': datetime.now(timezone.utc).isoformat(),
'updateVersion': '2026.11.04.1', 'updateSequence': 1, 'installationScope': 'current-user', 'requiresAdministrator': False,
'databaseIncluded': False, 'credentialsIncluded': False, 'requiresDownloadAtInstall': False,
'nativeTargetSmokeTest': 'not-run-on-macos-host',
'runtimes': [{k: v for k, v in a.items() if k not in ('kind',)} for a in assets if a['os'] == osname and a['name'] != 'vcredist'] + ([local_runtime] if windows else []),
'pythonWheels': [{'file': p.name, 'sha256': digest(p)} for p in wheels], 'files': []}
for path in sorted(root.rglob('*')):
if path.is_symlink():
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'symlink': str(path.readlink())})
elif path.is_file():
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': digest(path)})
(root / 'manifest.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
return metadata
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--work', type=Path, required=True)
parser.add_argument('--platform', choices=('windows', 'linux', 'both'), default='both')
parser.add_argument('--assemble-only', action='store_true')
args = parser.parse_args()
work = args.work.resolve()
assets = json.loads((work / 'downloads/assets.json').read_text())
platforms = ('windows', 'linux') if args.platform == 'both' else (args.platform,)
summaries = []
for osname in platforms:
name = 'QualityCenter-' + ('Windows' if osname == 'windows' else 'Linux') + '-x64'
directory = work / name
if directory.exists():
raise RuntimeError('Preserving an existing bundle; choose an empty work directory: ' + str(directory))
print('Assembling ' + name, flush=True)
metadata = payload(directory, work, osname, assets)
print('Files verified: ' + str(len(metadata['files'])), flush=True)
if args.assemble_only:
continue
if osname == 'windows':
archive = work / (name + '.zip')
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as target:
for path in sorted(directory.rglob('*')):
if path.is_file():
target.write(path, path.relative_to(work).as_posix())
else:
archive = work / (name + '.tar.gz')
with tarfile.open(archive, 'w:gz', compresslevel=6) as target:
target.add(directory, arcname=name)
summaries.append({'file': archive.name, 'bytes': archive.stat().st_size, 'sha256': digest(archive)})
print(json.dumps(summaries[-1]), flush=True)
if summaries:
(work / 'SHA256SUMS.txt').write_text(''.join(a['sha256'] + ' ' + a['file'] + '\n' for a in summaries))
(work / 'packages.json').write_text(json.dumps(summaries, indent=2) + '\n')
if __name__ == '__main__':
main()