File size: 16,965 Bytes
47d5468 a499118 47d5468 e1515b2 47d5468 a499118 47d5468 a499118 47d5468 a499118 47d5468 a499118 47d5468 a499118 47d5468 a499118 47d5468 a499118 47d5468 e9451c1 47d5468 e1515b2 47d5468 e9451c1 47d5468 e9451c1 47d5468 e1515b2 47d5468 a499118 e1515b2 47d5468 a499118 47d5468 a499118 47d5468 e9451c1 47d5468 e9451c1 47d5468 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 | #!/usr/bin/env bash
# Standalone node-local restore/check entry point. Safe to copy into ctopd_all/.
set -euo pipefail
if [[ "$(uname -s)" != Linux || "$(uname -m)" != x86_64 ]]; then
echo 'CTOPD requires Linux x86_64; do not install this runtime on Windows.' >&2
exit 2
fi
SYSTEM_PY=""
for candidate in python3.12 python3; do
if command -v "$candidate" >/dev/null 2>&1; then SYSTEM_PY="$(command -v "$candidate")"; break; fi
done
[[ -n "$SYSTEM_PY" ]] || { echo 'Need a system Python 3 interpreter.' >&2; exit 2; }
exec "$SYSTEM_PY" - "$@" <<'CTOPD_ENSURE_PY'
import argparse
import copy
import datetime
import fcntl
import hashlib
import json
import os
import pathlib
import platform
import shutil
import subprocess
import sys
import tarfile
import tempfile
EXPECTED_SHA = '084c321c0b7af1cd3a36d12a595c623567d4a37525aa0eecbc3933aa7f9b15c3'
EXPECTED_BYTES = 16515065867
MIN_FREE_BYTES = 45 * 1024**3
RECEIPT = '.ctopd-runtime.json'
# Filled from the independently qualified compatibility artifact before publication.
COMPAT_SHA = '21cc38ecdfc2f61714922127b67f3a0be89bf70bb2178b8fda8d3659e14f76c7'
COMPAT_BYTES = 30628766
COMPAT_REVISION = '3b1d90bc044874dc20c65308ec72841fc3db22e9'
def digest(path):
result = hashlib.sha256()
with path.open('rb') as stream:
for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
result.update(block)
return result.hexdigest()
def default_prefix(host_glibc, base=pathlib.Path('/opt/ctopd-b200')):
"""Choose a node-local prefix without touching either existing install.
Keep in sync with tools/runtime_paths.py (AST parity tested). This helper
stays self-contained when copied into the shared ctopd_all directory.
"""
legacy = base.with_name(base.name + '-u22')
if base.exists() or base.is_symlink():
if base.is_symlink():
return base # Let the caller reject an unsafe explicit target.
try:
receipt = json.loads((base / '.ctopd-runtime.json').read_text())
except (OSError, ValueError):
return base # Preserve/reject an unmanaged or malformed install.
if (isinstance(receipt, dict) and receipt.get('archive_sha256') == EXPECTED_SHA
and host_glibc < (2, 39) and not receipt.get('compatibility_overlay')):
return legacy # Known original v4 on an older host: use a sibling.
return base
if legacy.exists() or legacy.is_symlink():
return legacy # Full identity/integrity checks still run before reuse.
return base
def check_host():
libc, version = platform.libc_ver()
parsed = tuple(map(int, version.split('.')[:2])) if libc == 'glibc' else (0, 0)
if parsed < (2, 35):
raise RuntimeError('Supported baselines: Ubuntu 22.04/glibc 2.35 and Ubuntu 24.04/glibc 2.39; do not replace system libc')
if not shutil.which('g++'):
raise RuntimeError('Host g++ is required for runtime JIT; use a prepared node image')
return parsed
def compatibility_overlay(root):
if not COMPAT_SHA or not COMPAT_BYTES or not COMPAT_REVISION:
raise RuntimeError('glibc 2.35 requires the qualified ABI overlay; this helper has no published overlay pin yet')
path = root / 'compat/glibc235-v2/overlay.tar.gz'
if path.is_file() and path.stat().st_size == COMPAT_BYTES and digest(path) == COMPAT_SHA:
return path
if path.exists():
raise RuntimeError('Existing ABI overlay differs from the pinned checksum; preserved: ' + str(path))
path.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(prefix='.overlay-download-', dir=path.parent)
os.close(fd)
temporary = pathlib.Path(name)
url = ('https://huggingface.co/datasets/codemaivanngu/cross-tokenizer-opd-b200-runtime/resolve/'
+ COMPAT_REVISION + '/compat/glibc235-v2/overlay.tar.gz')
proxy = (os.environ.get('CTOPD_PROXY') or os.environ.get('HTTPS_PROXY') or os.environ.get('https_proxy')
or os.environ.get('HTTP_PROXY') or 'http://10.30.154.118:80')
command = ['curl', '-fL', '--retry', '3', url, '-o', str(temporary)]
if proxy.lower() != 'direct':
command += ['--proxy', proxy, '--noproxy', '']
else:
command += ['--noproxy', '*']
try:
print('DOWNLOADING_ABI_OVERLAY_BYTES=' + str(COMPAT_BYTES), flush=True)
subprocess.run(command, check=True)
if temporary.stat().st_size != COMPAT_BYTES or digest(temporary) != COMPAT_SHA:
raise RuntimeError('Downloaded ABI overlay size/SHA-256 mismatch')
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
return path
def apply_overlay(stage, archive):
if archive.stat().st_size != COMPAT_BYTES or digest(archive) != COMPAT_SHA:
raise RuntimeError('ABI overlay checksum mismatch')
disabled = stage / 'host-libs-v4-disabled'
if disabled.exists():
raise FileExistsError(disabled)
(stage / 'host-libs').rename(disabled)
with tarfile.open(archive, 'r:gz') as bundle:
for original in bundle:
path = pathlib.PurePosixPath(original.name)
if path.is_absolute() or '..' in path.parts or path.parts[:2] != ('opt', 'ctopd-b200'):
raise ValueError('Unexpected ABI overlay member: ' + str(path))
if len(path.parts) == 2:
continue
relative = pathlib.PurePosixPath(*path.parts[2:])
allowed = (relative.parts[0] == 'host-libs' or str(relative) == 'glibc235-overlay-manifest.json'
or (relative.parts[:2] == ('site-packages', 'sglang_router')
and (str(relative).endswith('.py') or str(relative) == 'site-packages/sglang_router/sglang_router_rs.abi3.so'))
or relative.parts[:2] == ('site-packages', 'sglang_router.libs'))
if not allowed or not (original.isfile() or original.isdir()):
raise ValueError('Disallowed ABI overlay member: ' + str(path))
member = copy.copy(original)
member.name = str(relative)
bundle.extract(member, stage, filter='data')
manifest = json.loads((stage / 'glibc235-overlay-manifest.json').read_text())
if manifest['base_archive_sha256'] != EXPECTED_SHA or manifest['minimum_glibc'] != '2.35':
raise RuntimeError('ABI overlay targets a different base runtime')
for row in manifest['files']:
relative = pathlib.PurePosixPath(row['path'])
if relative.is_absolute() or '..' in relative.parts or digest(stage / relative) != row['sha256']:
raise RuntimeError('ABI overlay payload integrity mismatch')
return {'sha256': COMPAT_SHA, 'files': manifest['files'], 'minimum_glibc': '2.35'}
def fingerprints(prefix):
interpreters = list(prefix.glob('python/cpython-3.12.12-*/bin/python3.12'))
if len(interpreters) != 1:
raise RuntimeError('Portable Python is missing or ambiguous in ' + str(prefix))
paths = [prefix / 'python-b200.sh', interpreters[0], prefix / 'cuda/bin/nvcc']
for path in paths:
if not path.is_file():
raise RuntimeError('Missing runtime component: ' + str(path))
return {str(path.relative_to(prefix)): digest(path) for path in paths}
def restore(archive, prefix, overlay=None):
if not hasattr(tarfile, 'data_filter'):
raise RuntimeError('System Python needs safe tar extraction filters (Python 3.12 or patched 3.10/3.11)')
if archive.stat().st_size != EXPECTED_BYTES or digest(archive) != EXPECTED_SHA:
raise RuntimeError('Archive size/SHA-256 mismatch; reassemble numbered parts correctly before installing')
print('ARCHIVE_SHA256_OK=' + EXPECTED_SHA, flush=True)
if shutil.disk_usage(prefix.parent).free < MIN_FREE_BYTES:
raise RuntimeError('Need at least 45 GiB free on the target filesystem for extraction')
stage = pathlib.Path(tempfile.mkdtemp(prefix='.' + prefix.name + '.install-', dir=prefix.parent))
print('EXTRACTING_TO=' + str(stage), flush=True)
try:
with tarfile.open(archive, 'r:gz') as bundle:
for original in bundle:
path = pathlib.PurePosixPath(original.name)
if path.is_absolute() or '..' in path.parts or path.parts[:2] != ('opt', 'ctopd-b200'):
raise ValueError('Unexpected archive member: ' + str(path))
if len(path.parts) == 2:
continue
member = copy.copy(original)
member.name = str(pathlib.PurePosixPath(*path.parts[2:]))
if member.islnk():
link = pathlib.PurePosixPath(member.linkname)
if link.is_absolute() or '..' in link.parts or link.parts[:2] != ('opt', 'ctopd-b200'):
raise ValueError('Unsafe hardlink: ' + member.linkname)
member.linkname = str(pathlib.PurePosixPath(*link.parts[2:]))
bundle.extract(member, stage, filter='data')
compat = apply_overlay(stage, overlay) if overlay else None
receipt = {'archive_sha256': EXPECTED_SHA, 'archive_bytes': EXPECTED_BYTES,
'installed_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'critical_files_sha256': fingerprints(stage)}
if compat:
receipt['compatibility_overlay'] = compat
(stage / RECEIPT).write_text(json.dumps(receipt, indent=2) + '\n')
if prefix.exists() or prefix.is_symlink():
raise FileExistsError(prefix)
stage.rename(prefix)
except Exception:
print('INSTALL_FAILED: staging retained for inspection at ' + str(stage), file=sys.stderr)
raise
PROBE = '''import importlib, importlib.metadata as md, json, os, pathlib, sys
import torch
assert torch.cuda.device_count() == 1, 'Expose exactly the B200 allocated to this check'
assert 'B200' in torch.cuda.get_device_name(0), 'Expected a B200'
expected = {'torch': '2.11.0+cu129', 'transformers': '5.6.2', 'sglang': '0.5.12.post1',
'flash-attn': '2.8.3', 'flashinfer-python': '0.6.11.post1', 'sglang-router': '0.3.2'}
versions = {name: md.version(name) for name in expected}
assert versions == expected, ('Runtime package mismatch', versions)
assert sys.version_info[:3] == (3, 12, 12), sys.version
cuda = pathlib.Path(os.environ['CUDA_HOME'])
assert (cuda / 'include/nvrtc.h').is_file(), 'Missing NVRTC headers'
for name in ['transformers', 'sglang.srt.entrypoints.engine', 'sglang_router.launch_router', 'flash_attn', 'flashinfer']:
importlib.import_module(name)
assert torch.version.cuda == '12.9', torch.version.cuda
value = torch.tensor([1.0], device='cuda')
assert value.cpu().item() == 1.0
print(json.dumps({'versions': versions, 'python': sys.version.split()[0],
'cuda': torch.version.cuda, 'gpu': torch.cuda.get_device_name(0),
'cuda_visible_devices': os.environ.get('CUDA_VISIBLE_DEVICES'),
'check': 'imports + CUDA memory roundtrip; no training or full kernel suite'}))
'''
def main(argv=None):
parser = argparse.ArgumentParser(description='Restore/check the pinned CTOPD runtime on this node; no pip, build or training')
parser.add_argument('--root', type=pathlib.Path,
default=pathlib.Path(os.environ.get('CTOPD_ROOT', '/workspace/storage-shared/nlp/tungks/ctopd_all')))
parser.add_argument('--prefix', type=pathlib.Path,
help='Override automatic prefix selection (or set CTOPD_RUNTIME_PREFIX)')
parser.add_argument('--archive', type=pathlib.Path)
parser.add_argument('--gpu', help='One allocated physical GPU index or GPU UUID; sets CUDA_VISIBLE_DEVICES for the check and forwarded command')
mode = parser.add_mutually_exclusive_group()
mode.add_argument('--install-only', action='store_true', help='Restore without a GPU check; reports INSTALLED_UNCHECKED')
mode.add_argument('--check-only', action='store_true', help='Check an existing managed runtime; never extract')
parser.add_argument('python_args', nargs=argparse.REMAINDER, help='After --, execute these Python arguments through the runtime wrapper')
args = parser.parse_args(argv)
runtime_env = os.environ.copy()
if args.gpu is not None:
selected = args.gpu.strip()
if not selected or ',' in selected or not (selected.isdecimal() or (selected.startswith('GPU-') and len(selected) > 4)):
raise ValueError('--gpu must select exactly one allocated GPU index or UUID')
runtime_env['CUDA_VISIBLE_DEVICES'] = selected
forwarded = args.python_args[1:] if args.python_args[:1] == ['--'] else args.python_args
if args.install_only and forwarded:
raise ValueError('--install-only cannot also execute a runtime command')
host_glibc = check_host()
configured = args.prefix or runtime_env.get('CTOPD_RUNTIME_PREFIX')
raw = pathlib.Path(configured).expanduser() if configured else default_prefix(host_glibc)
if not raw.is_absolute() or raw.is_symlink():
raise ValueError('Prefix must be an absolute non-symlink directory')
prefix = raw.resolve()
if len(prefix.parts) < 3 or prefix == args.root.expanduser().resolve():
raise ValueError('Refusing an unsafe runtime prefix: ' + str(prefix))
needs_compat = host_glibc < (2, 39)
runtime_env['CTOPD_RUNTIME_PREFIX'] = str(prefix)
runtime_env['CTOPD_ROOT'] = str(args.root.expanduser().resolve())
if args.check_only and not prefix.exists():
raise FileNotFoundError('Runtime is absent; run without --check-only to restore it')
prefix.parent.mkdir(parents=True, exist_ok=True)
lock_path = prefix.parent / ('.' + prefix.name + '.ensure.lock')
if lock_path.is_symlink():
raise ValueError('Refusing symlink lock file')
with lock_path.open('a') as lock:
print('WAITING_FOR_RUNTIME_LOCK=' + str(lock_path), flush=True)
fcntl.flock(lock, fcntl.LOCK_EX)
if prefix.exists():
if not prefix.is_dir() or not (prefix / RECEIPT).is_file():
raise RuntimeError('Existing unmanaged prefix preserved; choose another --prefix or inspect it manually: ' + str(prefix))
receipt = json.loads((prefix / RECEIPT).read_text())
if receipt.get('archive_sha256') != EXPECTED_SHA or receipt.get('critical_files_sha256') != fingerprints(prefix):
raise RuntimeError('Existing runtime identity/integrity check failed; preserved without replacement')
compat = receipt.get('compatibility_overlay')
if needs_compat and (not compat or compat.get('sha256') != COMPAT_SHA):
raise RuntimeError('Existing v4 runtime is not glibc-2.35 compatible; preserved. Choose a fresh --prefix for the qualified ABI overlay')
if compat:
for row in compat['files']:
relative = pathlib.PurePosixPath(row['path'])
if relative.is_absolute() or '..' in relative.parts or digest(prefix / relative) != row['sha256']:
raise RuntimeError('Installed ABI overlay integrity mismatch; preserved without replacement')
print('REUSING_RUNTIME=' + str(prefix), flush=True)
else:
if args.check_only:
raise FileNotFoundError('Runtime disappeared while waiting for its lock; no extraction performed')
root = args.root.expanduser().resolve()
candidates = [root / 'ctopd-b200-cu129-v4/runtime.tar.gz', root / 'runtime.tar.gz']
archive = args.archive.expanduser().resolve() if args.archive else next((p for p in candidates if p.is_file()), None)
if archive is None or not archive.is_file():
raise FileNotFoundError('No runtime.tar.gz in ' + str(root) + '; download once to shared storage or pass --archive')
overlay = compatibility_overlay(root) if needs_compat else None
restore(archive, prefix, overlay)
if args.install_only:
print('RUNTIME_INSTALLED_UNCHECKED=' + str(prefix), flush=True)
else:
env = runtime_env.copy()
env.update(PYTHONPATH='', HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', PYTHONNOUSERSITE='1')
output = subprocess.check_output(['bash', str(prefix / 'python-b200.sh'), '-c', PROBE], env=env, text=True)
result = json.loads(output.strip().splitlines()[-1])
result['checked_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
(prefix / '.ctopd-last-check.json').write_text(json.dumps(result, indent=2) + '\n')
print(output, end='', flush=True)
print('RUNTIME_READY=' + str(prefix), flush=True)
print('RUNTIME_PYTHON_WRAPPER=' + str(prefix / 'python-b200.sh'), flush=True)
if forwarded:
os.execvpe('bash', ['bash', str(prefix / 'python-b200.sh'), *forwarded], runtime_env)
if __name__ == '__main__':
main()
CTOPD_ENSURE_PY
|