File size: 3,660 Bytes
628e0f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | #!/usr/bin/env python3
"""Verify the GPU/accelerator environment on a Colab runtime.
Designed to run *on the remote Colab VM*, e.g.:
colab new -s noeon --gpu T4
colab exec -s noeon -f scripts/colab_gpu_check.py
or locally on any host that may or may not have a GPU:
python scripts/colab_gpu_check.py
It reports real, measured facts only: it never assumes a GPU exists and it
exits non-zero when no CUDA device is found, so it can gate a training run.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
def _run(cmd: list[str]) -> str:
try:
out = subprocess.run(
cmd, capture_output=True, text=True, timeout=30, check=False
)
return (out.stdout or out.stderr).strip()
except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
return f"<unavailable: {exc}>"
def collect() -> dict:
info: dict = {}
info["nvidia_smi_present"] = shutil.which("nvidia-smi") is not None
info["nvidia_smi_query"] = _run(
[
"nvidia-smi",
"--query-gpu=name,memory.total,memory.free,driver_version,compute_cap",
"--format=csv,noheader",
]
)
info["nvidia_smi_header"] = _run(["nvidia-smi"]).splitlines()[:1]
try:
import torch # noqa: PLC0415
info["torch_version"] = torch.__version__
info["cuda_available"] = bool(torch.cuda.is_available())
info["cuda_version"] = getattr(torch.version, "cuda", None)
info["cudnn_version"] = (
torch.backends.cudnn.version()
if torch.backends.cudnn.is_available()
else None
)
info["device_count"] = torch.cuda.device_count()
if torch.cuda.is_available():
devices = []
for i in range(torch.cuda.device_count()):
props = torch.cuda.get_device_properties(i)
devices.append(
{
"index": i,
"name": props.name,
"total_memory_gb": round(props.total_memory / 1e9, 2),
"major": props.major,
"minor": props.minor,
}
)
info["devices"] = devices
# A tiny allocation proves the driver and allocator really work.
tensor = torch.zeros(1024, 1024, device="cuda")
info["alloc_test"] = {
"ok": True,
"bytes": tensor.numel() * tensor.element_size(),
}
info["vram_allocated_gb"] = round(
torch.cuda.memory_allocated(0) / 1e9, 4
)
except ImportError:
info["torch_version"] = None
info["cuda_available"] = False
except Exception as exc: # pragma: no cover - remote-only path
info["torch_error"] = f"{type(exc).__name__}: {exc}"
return info
def main() -> int:
info = collect()
print(json.dumps(info, indent=2, sort_keys=True))
gpu_ok = bool(info.get("cuda_available")) and bool(
info.get("nvidia_smi_present")
)
if not gpu_ok:
print(
"\nNO USABLE GPU: this host has no CUDA device. "
"On Colab, pick a GPU runtime (Runtime > Change runtime type) or "
"create the session with `colab new --gpu T4`.",
file=sys.stderr,
)
return 1
print(
"\nGPU OK: {} device(s), torch {}, CUDA {}.".format(
info["device_count"], info["torch_version"], info.get("cuda_version")
)
)
return 0
if __name__ == "__main__":
sys.exit(main())
|