ctopd / tests /test_ensure_runtime.py
codemaivanngu's picture
fix(runtime): automatically select and ensure compatible node runtime
e1515b2 unverified
Raw History Blame Contribute Delete
14.9 kB
"""Tiny archive fixtures; never download/install the production runtime locally."""
import ast
import contextlib
import hashlib
import io
import json
import pathlib
import tempfile
import types
import unittest
ROOT = pathlib.Path(__file__).resolve().parents[1]
SCRIPT = (ROOT / 'ensure_runtime.sh').read_text()
INLINE = SCRIPT.split("<<'CTOPD_ENSURE_PY'\n", 1)[1].rsplit('\nCTOPD_ENSURE_PY', 1)[0]
class EnsureRuntimeTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = pathlib.Path(self.temp.name)
self.archive = self.root / 'runtime.tar.gz'
self.prefix = self.root / 'node-local/runtime'
self.code = {'__name__': 'ensure_fixture'}
exec(compile(INLINE, 'ensure_runtime.sh:inline', 'exec'), self.code)
# Fixture installation only: no real GPU/ML runtime or 45 GiB allocation.
self.code['check_host'] = lambda: (2, 39)
self.code['MIN_FREE_BYTES'] = 0
def make_overlay(self, forbidden=False):
import tarfile
self.archive_with({'opt/ctopd-b200/host-libs/old-incompatible.so': b'old-v4-host-library'})
self.overlay = self.root / 'compat/glibc235-v2/overlay.tar.gz'
self.overlay.parent.mkdir(parents=True)
files = {'host-libs/libstdc++.so.6': b'compatible-library',
'site-packages/sglang_router/sglang_router_rs.abi3.so': b'compatible-router'}
manifest = {'base_archive_sha256': self.code['EXPECTED_SHA'], 'minimum_glibc': '2.35',
'files': [{'path': path, 'sha256': hashlib.sha256(content).hexdigest()} for path, content in files.items()]}
files['glibc235-overlay-manifest.json'] = json.dumps(manifest).encode()
if forbidden:
files['site-packages/torch/unrequested.py'] = b'forbidden'
with tarfile.open(self.overlay, 'w:gz') as bundle:
for name, content in files.items():
member = tarfile.TarInfo('opt/ctopd-b200/' + name)
member.size = len(content)
bundle.addfile(member, io.BytesIO(content))
self.code.update(COMPAT_SHA=hashlib.sha256(self.overlay.read_bytes()).hexdigest(),
COMPAT_BYTES=self.overlay.stat().st_size, COMPAT_REVISION='fixture')
def tearDown(self):
self.temp.cleanup()
def archive_with(self, extra=None):
import tarfile
files = {'opt/ctopd-b200/python-b200.sh': b'#!/bin/bash\n',
'opt/ctopd-b200/python/cpython-3.12.12-fixture/bin/python3.12': b'fixture-python',
'opt/ctopd-b200/cuda/bin/nvcc': b'fixture-nvcc'}
if extra:
files.update(extra)
with tarfile.open(self.archive, 'w:gz') as bundle:
for name, content in files.items():
member = tarfile.TarInfo(name)
member.size = len(content)
bundle.addfile(member, io.BytesIO(content))
self.code['EXPECTED_SHA'] = hashlib.sha256(self.archive.read_bytes()).hexdigest()
self.code['EXPECTED_BYTES'] = self.archive.stat().st_size
def run_ensure(self, *options):
with contextlib.redirect_stdout(io.StringIO()) as output:
self.code['main'](['--root', str(self.root), '--prefix', str(self.prefix), *options])
return output.getvalue()
def test_inline_and_probe_syntax(self):
ast.parse(INLINE)
ast.parse(self.code['PROBE'])
def test_selected_gpu_reaches_check_and_forwarded_command(self):
self.archive_with()
self.run_ensure('--install-only')
probes, forwarded = [], []
def check_output(_command, *, env, text):
probes.append(env.copy())
return '{}\n'
def execvpe(_program, _argv, env):
forwarded.append(env.copy())
self.code['subprocess'] = types.SimpleNamespace(check_output=check_output)
self.code['os'] = types.SimpleNamespace(environ={'CUDA_VISIBLE_DEVICES': '3'}, execvpe=execvpe)
for selection, expected in [(None, '3'), ('2', '2'), ('GPU-fixture-uuid', 'GPU-fixture-uuid')]:
options = ['--check-only'] + (['--gpu', selection] if selection else [])
self.run_ensure(*options, '--', '-c', 'pass')
self.assertEqual(probes[-1]['CUDA_VISIBLE_DEVICES'], expected)
self.assertEqual(forwarded[-1]['CUDA_VISIBLE_DEVICES'], expected)
self.assertEqual(self.code['os'].environ['CUDA_VISIBLE_DEVICES'], '3')
def test_gpu_selection_rejects_empty_or_multiple_devices_before_install(self):
for selection in ['', '0,2', '-1', 'invalid']:
with self.assertRaisesRegex(ValueError, 'exactly one allocated GPU'):
self.run_ensure('--gpu=' + selection, '--install-only')
self.assertFalse(self.prefix.parent.exists())
def test_both_required_host_baselines_and_older_rejection(self):
isolated = {'__name__': 'host_test'}
exec(compile(INLINE, 'ensure_runtime.sh:inline', 'exec'), isolated)
isolated['shutil'] = types.SimpleNamespace(which=lambda _name: '/usr/bin/g++')
for version, expected in [('2.35', (2, 35)), ('2.39', (2, 39))]:
isolated['platform'] = types.SimpleNamespace(libc_ver=lambda: ('glibc', version))
self.assertEqual(isolated['check_host'](), expected)
isolated['platform'] = types.SimpleNamespace(libc_ver=lambda: ('glibc', '2.34'))
with self.assertRaises(RuntimeError):
isolated['check_host']()
def test_glibc235_does_not_silently_accept_unpatched_v4(self):
self.archive_with()
self.run_ensure('--install-only')
before = (self.prefix / self.code['RECEIPT']).read_bytes()
self.code['check_host'] = lambda: (2, 35)
with self.assertRaisesRegex(RuntimeError, 'not glibc-2.35 compatible'):
self.run_ensure('--install-only')
self.assertEqual((self.prefix / self.code['RECEIPT']).read_bytes(), before)
def test_automatic_prefix_on_both_hosts_preserves_original_and_reuses_u22(self):
import os
from unittest.mock import patch
base = self.prefix
original_default = self.code['default_prefix']
self.code['default_prefix'] = lambda host: original_default(host, base)
self.make_overlay()
with patch.dict(os.environ, {}, clear=True):
def automatic(*options):
with contextlib.redirect_stdout(io.StringIO()) as output:
self.code['main'](['--root', str(self.root), '--install-only', *options])
return output.getvalue()
automatic() # Ubuntu 24 original v4.
before = (base / self.code['RECEIPT']).read_bytes()
self.code['check_host'] = lambda: (2, 35)
sibling = base.with_name(base.name + '-u22')
self.assertIn(str(sibling), automatic())
self.assertEqual((base / self.code['RECEIPT']).read_bytes(), before)
self.assertTrue((sibling / 'glibc235-overlay-manifest.json').is_file())
self.archive.unlink(); self.overlay.unlink()
self.assertIn('REUSING_RUNTIME=' + str(sibling), automatic())
self.code['check_host'] = lambda: (2, 39)
self.assertIn('REUSING_RUNTIME=' + str(base), automatic())
def test_automatic_fresh_ubuntu22_uses_base_with_overlay_and_forwards_selected_env(self):
from unittest.mock import patch
import os
self.make_overlay()
self.code['check_host'] = lambda: (2, 35)
choose = self.code['default_prefix']
self.code['default_prefix'] = lambda host: choose(host, self.prefix)
forwarded = []
with patch.dict(os.environ, {}, clear=True), patch.object(os,'execvpe',lambda *a:forwarded.append(a[2])), \
patch.object(self.code['subprocess'],'check_output',return_value='{}\n'):
self.code['main'](['--root', str(self.root), '--gpu', '3', '--', '-c', 'pass'])
env = forwarded[0]
self.assertEqual(env['CTOPD_RUNTIME_PREFIX'], str(self.prefix))
self.assertEqual(env['CTOPD_ROOT'], str(self.root))
self.assertEqual(env['CUDA_VISIBLE_DEVICES'], '3')
self.assertTrue((self.prefix / 'glibc235-overlay-manifest.json').is_file())
def test_standalone_and_launcher_prefix_policy_have_identical_ast(self):
from tools import runtime_paths
policy = ast.parse((ROOT / 'tools/runtime_paths.py').read_text())
select = lambda tree: ast.dump(next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name=='default_prefix'), include_attributes=False)
self.assertEqual(select(ast.parse(INLINE)), select(policy))
self.assertEqual(self.code['EXPECTED_SHA'], runtime_paths.EXPECTED_SHA)
def test_glibc235_requires_real_pinned_overlay(self):
self.archive_with()
self.code['check_host'] = lambda: (2, 35)
self.code.update(COMPAT_SHA=None, COMPAT_BYTES=None, COMPAT_REVISION=None)
with self.assertRaisesRegex(RuntimeError, 'no published overlay pin'):
self.run_ensure('--install-only')
self.assertFalse(self.prefix.exists())
def test_overlay_download_verifies_bytes_and_uses_selected_proxy(self):
import os
self.make_overlay()
payload = self.overlay.read_bytes()
self.overlay.unlink()
for proxy in ['direct', 'http://fixture-proxy:80']:
calls = []
def download(command, **_kwargs):
calls.append(command)
pathlib.Path(command[command.index('-o') + 1]).write_bytes(payload)
self.code['os'] = types.SimpleNamespace(environ={'CTOPD_PROXY': proxy}, close=os.close)
self.code['subprocess'] = types.SimpleNamespace(run=download)
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(self.code['compatibility_overlay'](self.root), self.overlay)
self.assertEqual(self.overlay.read_bytes(), payload)
self.assertEqual(calls[0][-2:], ['--noproxy', '*' if proxy == 'direct' else ''])
self.assertEqual(list(self.overlay.parent.glob('.overlay-download-*')), [])
self.overlay.unlink()
def test_invalid_cached_overlay_is_preserved(self):
self.make_overlay()
self.overlay.write_bytes(b'invalid-existing-overlay')
with self.assertRaisesRegex(RuntimeError, 'preserved'):
self.code['compatibility_overlay'](self.root)
self.assertEqual(self.overlay.read_bytes(), b'invalid-existing-overlay')
def test_glibc235_installs_overlay_and_quarantines_old_host_libraries(self):
self.make_overlay()
self.code['check_host'] = lambda: (2, 35)
self.run_ensure('--install-only')
self.assertEqual((self.prefix / 'host-libs/libstdc++.so.6').read_bytes(), b'compatible-library')
self.assertEqual((self.prefix / 'host-libs-v4-disabled/old-incompatible.so').read_bytes(), b'old-v4-host-library')
receipt = json.loads((self.prefix / self.code['RECEIPT']).read_text())
self.assertEqual(receipt['compatibility_overlay']['sha256'], self.code['COMPAT_SHA'])
self.overlay.unlink()
self.archive.unlink()
self.assertIn('REUSING_RUNTIME=', self.run_ensure('--install-only'))
def test_overlay_cannot_modify_training_package(self):
self.make_overlay(forbidden=True)
self.code['check_host'] = lambda: (2, 35)
with contextlib.redirect_stderr(io.StringIO()):
with self.assertRaisesRegex(ValueError, 'Disallowed ABI overlay member'):
self.run_ensure('--install-only')
self.assertFalse(self.prefix.exists())
def test_install_and_reuse_without_archive(self):
self.archive_with()
output = self.run_ensure('--install-only')
self.assertIn('RUNTIME_INSTALLED_UNCHECKED=', output)
self.assertNotIn('RUNTIME_READY=', output)
receipt = self.prefix / self.code['RECEIPT']
before = receipt.read_bytes()
self.archive.unlink()
output = self.run_ensure('--install-only')
self.assertIn('REUSING_RUNTIME=', output)
self.assertEqual(receipt.read_bytes(), before)
def test_wrong_checksum_never_installs(self):
self.archive_with()
self.code['EXPECTED_SHA'] = '0' * 64
with self.assertRaisesRegex(RuntimeError, 'SHA-256 mismatch'):
self.run_ensure('--install-only')
self.assertFalse(self.prefix.exists())
self.assertFalse(list(self.prefix.parent.glob('.runtime.install-*')))
def test_unmanaged_existing_prefix_preserved(self):
self.prefix.mkdir(parents=True)
content = self.prefix / 'user-data'
content.write_text('keep')
with self.assertRaisesRegex(RuntimeError, 'unmanaged prefix preserved'):
self.run_ensure('--install-only')
self.assertEqual(content.read_text(), 'keep')
def test_changed_runtime_preserved_and_rejected(self):
self.archive_with()
self.run_ensure('--install-only')
wrapper = self.prefix / 'python-b200.sh'
wrapper.write_text('user change')
with self.assertRaisesRegex(RuntimeError, 'identity/integrity check failed'):
self.run_ensure('--install-only')
self.assertEqual(wrapper.read_text(), 'user change')
def test_traversal_rejected_and_evidence_retained(self):
self.archive_with({'opt/ctopd-b200/../escape': b'bad'})
with contextlib.redirect_stderr(io.StringIO()):
with self.assertRaisesRegex(ValueError, 'Unexpected archive member'):
self.run_ensure('--install-only')
self.assertFalse(self.prefix.exists())
self.assertFalse((self.prefix.parent / 'escape').exists())
self.assertEqual(len(list(self.prefix.parent.glob('.runtime.install-*'))), 1)
def test_check_only_does_not_install_or_create_parent(self):
with self.assertRaises(FileNotFoundError):
self.run_ensure('--check-only')
self.assertFalse(self.prefix.parent.exists())
def test_escaping_symlink_rejected(self):
import tarfile
self.archive_with()
with tarfile.open(self.archive, 'w:gz') as bundle:
member = tarfile.TarInfo('opt/ctopd-b200/escape')
member.type = tarfile.SYMTYPE
member.linkname = '/tmp/outside-ctopd'
bundle.addfile(member)
self.code['EXPECTED_SHA'] = hashlib.sha256(self.archive.read_bytes()).hexdigest()
self.code['EXPECTED_BYTES'] = self.archive.stat().st_size
with contextlib.redirect_stderr(io.StringIO()):
with self.assertRaises(tarfile.FilterError):
self.run_ensure('--install-only')
self.assertFalse(self.prefix.exists())
if __name__ == '__main__':
unittest.main()