feat(runtime): allow explicit GPU selection on shared nodes
Browse files- docs/RUNTIME_B200.md +17 -0
- ensure_runtime.SHA256SUMS +1 -1
- ensure_runtime.sh +10 -2
- tests/test_ensure_runtime.py +27 -0
docs/RUNTIME_B200.md
CHANGED
|
@@ -36,6 +36,23 @@ Published helper SHA-256: `a09e1c27bd7ee6f2919b49b48d03ff2a3ebb81444aa5f578dc828
|
|
| 36 |
|
| 37 |
Run the last line on each new node, inside a pod exposing the one B200 allocated for this check. With no GPU allocated yet, use `--install-only`; it reports `RUNTIME_INSTALLED_UNCHECKED`, not GPU readiness. With an existing runtime, `--check-only` checks without extracting. `--help` lists options.
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
The helper automatically finds either `$ROOT/ctopd-b200-cu129-v4/runtime.tar.gz` (the manually downloaded shard layout) or `$ROOT/runtime.tar.gz`. It verifies the pinned archive size/SHA-256 before extraction, checks Linux/glibc/g++, extracts safely to a new staging directory under the target filesystem, then renames to the final prefix. On glibc 2.35 through 2.38 it downloads the small verified overlay once into `$ROOT/compat/glibc235-v2/`; an existing valid overlay is reused. It uses `CTOPD_PROXY`, existing proxy variables, or the company proxy by default; set `CTOPD_PROXY=direct` for direct access. Allow at least 45 GiB free on that node-local filesystem. It never builds dependencies or launches training. Failed extraction keeps staging evidence. If `/opt` is not writable, choose an explicit node-local location:
|
| 40 |
|
| 41 |
```bash
|
|
|
|
| 36 |
|
| 37 |
Run the last line on each new node, inside a pod exposing the one B200 allocated for this check. With no GPU allocated yet, use `--install-only`; it reports `RUNTIME_INSTALLED_UNCHECKED`, not GPU readiness. With an existing runtime, `--check-only` checks without extracting. `--help` lists options.
|
| 38 |
|
| 39 |
+
### Choose the allocated GPU on a shared node
|
| 40 |
+
|
| 41 |
+
The helper does not select physical GPU 0 by default. It retains the caller's `CUDA_VISIBLE_DEVICES`; CUDA's device 0 in the probe is the first device in that filtered view. Every helper version supports:
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
CUDA_VISIBLE_DEVICES=2 bash ensure_runtime.sh --prefix /opt/ctopd-b200-u22 --check-only
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
The current source helper also supports an explicit physical GPU index or UUID:
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
bash ensure_runtime.sh --gpu 2 --prefix /opt/ctopd-b200-u22 --check-only
|
| 51 |
+
bash ensure_runtime.sh --gpu GPU-your-allocated-uuid --prefix /opt/ctopd-b200-u22 --check-only
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
`--gpu` explicitly replaces the inherited visibility mask for the check and any Python command passed after `--`. Without it, the inherited mask is preserved. It does not change the parent shell, allocate a GPU, or scan for an idle device. Choose the index/UUID assigned to you; leave the flag unset when the scheduler already exposes the correct single GPU. Keep the same visibility setting in a separately launched training command. The readiness receipt records the visibility mask.
|
| 55 |
+
|
| 56 |
The helper automatically finds either `$ROOT/ctopd-b200-cu129-v4/runtime.tar.gz` (the manually downloaded shard layout) or `$ROOT/runtime.tar.gz`. It verifies the pinned archive size/SHA-256 before extraction, checks Linux/glibc/g++, extracts safely to a new staging directory under the target filesystem, then renames to the final prefix. On glibc 2.35 through 2.38 it downloads the small verified overlay once into `$ROOT/compat/glibc235-v2/`; an existing valid overlay is reused. It uses `CTOPD_PROXY`, existing proxy variables, or the company proxy by default; set `CTOPD_PROXY=direct` for direct access. Allow at least 45 GiB free on that node-local filesystem. It never builds dependencies or launches training. Failed extraction keeps staging evidence. If `/opt` is not writable, choose an explicit node-local location:
|
| 57 |
|
| 58 |
```bash
|
ensure_runtime.SHA256SUMS
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
5274a6bae10ae5e83fa89550087dd4e901483375e779d2725832929f020529de ensure_runtime.sh
|
ensure_runtime.sh
CHANGED
|
@@ -190,6 +190,7 @@ value = torch.tensor([1.0], device='cuda')
|
|
| 190 |
assert value.cpu().item() == 1.0
|
| 191 |
print(json.dumps({'versions': versions, 'python': sys.version.split()[0],
|
| 192 |
'cuda': torch.version.cuda, 'gpu': torch.cuda.get_device_name(0),
|
|
|
|
| 193 |
'check': 'imports + CUDA memory roundtrip; no training or full kernel suite'}))
|
| 194 |
'''
|
| 195 |
|
|
@@ -201,11 +202,18 @@ def main(argv=None):
|
|
| 201 |
parser.add_argument('--prefix', type=pathlib.Path,
|
| 202 |
default=pathlib.Path(os.environ.get('CTOPD_RUNTIME_PREFIX', '/opt/ctopd-b200')))
|
| 203 |
parser.add_argument('--archive', type=pathlib.Path)
|
|
|
|
| 204 |
mode = parser.add_mutually_exclusive_group()
|
| 205 |
mode.add_argument('--install-only', action='store_true', help='Restore without a GPU check; reports INSTALLED_UNCHECKED')
|
| 206 |
mode.add_argument('--check-only', action='store_true', help='Check an existing managed runtime; never extract')
|
| 207 |
parser.add_argument('python_args', nargs=argparse.REMAINDER, help='After --, execute these Python arguments through the runtime wrapper')
|
| 208 |
args = parser.parse_args(argv)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
forwarded = args.python_args[1:] if args.python_args[:1] == ['--'] else args.python_args
|
| 210 |
if args.install_only and forwarded:
|
| 211 |
raise ValueError('--install-only cannot also execute a runtime command')
|
|
@@ -254,7 +262,7 @@ def main(argv=None):
|
|
| 254 |
if args.install_only:
|
| 255 |
print('RUNTIME_INSTALLED_UNCHECKED=' + str(prefix), flush=True)
|
| 256 |
else:
|
| 257 |
-
env =
|
| 258 |
env.update(PYTHONPATH='', HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', PYTHONNOUSERSITE='1')
|
| 259 |
output = subprocess.check_output(['bash', str(prefix / 'python-b200.sh'), '-c', PROBE], env=env, text=True)
|
| 260 |
result = json.loads(output.strip().splitlines()[-1])
|
|
@@ -264,7 +272,7 @@ def main(argv=None):
|
|
| 264 |
print('RUNTIME_READY=' + str(prefix), flush=True)
|
| 265 |
print('RUNTIME_PYTHON_WRAPPER=' + str(prefix / 'python-b200.sh'), flush=True)
|
| 266 |
if forwarded:
|
| 267 |
-
os.execvpe('bash', ['bash', str(prefix / 'python-b200.sh'), *forwarded],
|
| 268 |
|
| 269 |
|
| 270 |
if __name__ == '__main__':
|
|
|
|
| 190 |
assert value.cpu().item() == 1.0
|
| 191 |
print(json.dumps({'versions': versions, 'python': sys.version.split()[0],
|
| 192 |
'cuda': torch.version.cuda, 'gpu': torch.cuda.get_device_name(0),
|
| 193 |
+
'cuda_visible_devices': os.environ.get('CUDA_VISIBLE_DEVICES'),
|
| 194 |
'check': 'imports + CUDA memory roundtrip; no training or full kernel suite'}))
|
| 195 |
'''
|
| 196 |
|
|
|
|
| 202 |
parser.add_argument('--prefix', type=pathlib.Path,
|
| 203 |
default=pathlib.Path(os.environ.get('CTOPD_RUNTIME_PREFIX', '/opt/ctopd-b200')))
|
| 204 |
parser.add_argument('--archive', type=pathlib.Path)
|
| 205 |
+
parser.add_argument('--gpu', help='One allocated physical GPU index or GPU UUID; sets CUDA_VISIBLE_DEVICES for the check and forwarded command')
|
| 206 |
mode = parser.add_mutually_exclusive_group()
|
| 207 |
mode.add_argument('--install-only', action='store_true', help='Restore without a GPU check; reports INSTALLED_UNCHECKED')
|
| 208 |
mode.add_argument('--check-only', action='store_true', help='Check an existing managed runtime; never extract')
|
| 209 |
parser.add_argument('python_args', nargs=argparse.REMAINDER, help='After --, execute these Python arguments through the runtime wrapper')
|
| 210 |
args = parser.parse_args(argv)
|
| 211 |
+
runtime_env = os.environ.copy()
|
| 212 |
+
if args.gpu is not None:
|
| 213 |
+
selected = args.gpu.strip()
|
| 214 |
+
if not selected or ',' in selected or not (selected.isdecimal() or (selected.startswith('GPU-') and len(selected) > 4)):
|
| 215 |
+
raise ValueError('--gpu must select exactly one allocated GPU index or UUID')
|
| 216 |
+
runtime_env['CUDA_VISIBLE_DEVICES'] = selected
|
| 217 |
forwarded = args.python_args[1:] if args.python_args[:1] == ['--'] else args.python_args
|
| 218 |
if args.install_only and forwarded:
|
| 219 |
raise ValueError('--install-only cannot also execute a runtime command')
|
|
|
|
| 262 |
if args.install_only:
|
| 263 |
print('RUNTIME_INSTALLED_UNCHECKED=' + str(prefix), flush=True)
|
| 264 |
else:
|
| 265 |
+
env = runtime_env.copy()
|
| 266 |
env.update(PYTHONPATH='', HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', PYTHONNOUSERSITE='1')
|
| 267 |
output = subprocess.check_output(['bash', str(prefix / 'python-b200.sh'), '-c', PROBE], env=env, text=True)
|
| 268 |
result = json.loads(output.strip().splitlines()[-1])
|
|
|
|
| 272 |
print('RUNTIME_READY=' + str(prefix), flush=True)
|
| 273 |
print('RUNTIME_PYTHON_WRAPPER=' + str(prefix / 'python-b200.sh'), flush=True)
|
| 274 |
if forwarded:
|
| 275 |
+
os.execvpe('bash', ['bash', str(prefix / 'python-b200.sh'), *forwarded], runtime_env)
|
| 276 |
|
| 277 |
|
| 278 |
if __name__ == '__main__':
|
tests/test_ensure_runtime.py
CHANGED
|
@@ -73,6 +73,33 @@ class EnsureRuntimeTests(unittest.TestCase):
|
|
| 73 |
ast.parse(INLINE)
|
| 74 |
ast.parse(self.code['PROBE'])
|
| 75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
def test_both_required_host_baselines_and_older_rejection(self):
|
| 77 |
isolated = {'__name__': 'host_test'}
|
| 78 |
exec(compile(INLINE, 'ensure_runtime.sh:inline', 'exec'), isolated)
|
|
|
|
| 73 |
ast.parse(INLINE)
|
| 74 |
ast.parse(self.code['PROBE'])
|
| 75 |
|
| 76 |
+
def test_selected_gpu_reaches_check_and_forwarded_command(self):
|
| 77 |
+
self.archive_with()
|
| 78 |
+
self.run_ensure('--install-only')
|
| 79 |
+
probes, forwarded = [], []
|
| 80 |
+
|
| 81 |
+
def check_output(_command, *, env, text):
|
| 82 |
+
probes.append(env.copy())
|
| 83 |
+
return '{}\n'
|
| 84 |
+
|
| 85 |
+
def execvpe(_program, _argv, env):
|
| 86 |
+
forwarded.append(env.copy())
|
| 87 |
+
|
| 88 |
+
self.code['subprocess'] = types.SimpleNamespace(check_output=check_output)
|
| 89 |
+
self.code['os'] = types.SimpleNamespace(environ={'CUDA_VISIBLE_DEVICES': '3'}, execvpe=execvpe)
|
| 90 |
+
for selection, expected in [(None, '3'), ('2', '2'), ('GPU-fixture-uuid', 'GPU-fixture-uuid')]:
|
| 91 |
+
options = ['--check-only'] + (['--gpu', selection] if selection else [])
|
| 92 |
+
self.run_ensure(*options, '--', '-c', 'pass')
|
| 93 |
+
self.assertEqual(probes[-1]['CUDA_VISIBLE_DEVICES'], expected)
|
| 94 |
+
self.assertEqual(forwarded[-1]['CUDA_VISIBLE_DEVICES'], expected)
|
| 95 |
+
self.assertEqual(self.code['os'].environ['CUDA_VISIBLE_DEVICES'], '3')
|
| 96 |
+
|
| 97 |
+
def test_gpu_selection_rejects_empty_or_multiple_devices_before_install(self):
|
| 98 |
+
for selection in ['', '0,2', '-1', 'invalid']:
|
| 99 |
+
with self.assertRaisesRegex(ValueError, 'exactly one allocated GPU'):
|
| 100 |
+
self.run_ensure('--gpu=' + selection, '--install-only')
|
| 101 |
+
self.assertFalse(self.prefix.parent.exists())
|
| 102 |
+
|
| 103 |
def test_both_required_host_baselines_and_older_rejection(self):
|
| 104 |
isolated = {'__name__': 'host_test'}
|
| 105 |
exec(compile(INLINE, 'ensure_runtime.sh:inline', 'exec'), isolated)
|