File size: 3,660 Bytes
628e0f3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/usr/bin/env python3
"""Verify the GPU/accelerator environment on a Colab runtime.

Designed to run *on the remote Colab VM*, e.g.:

    colab new -s noeon --gpu T4
    colab exec -s noeon -f scripts/colab_gpu_check.py

or locally on any host that may or may not have a GPU:

    python scripts/colab_gpu_check.py

It reports real, measured facts only: it never assumes a GPU exists and it
exits non-zero when no CUDA device is found, so it can gate a training run.
"""

from __future__ import annotations

import json
import os
import shutil
import subprocess
import sys


def _run(cmd: list[str]) -> str:
    try:
        out = subprocess.run(
            cmd, capture_output=True, text=True, timeout=30, check=False
        )
        return (out.stdout or out.stderr).strip()
    except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
        return f"<unavailable: {exc}>"


def collect() -> dict:
    info: dict = {}

    info["nvidia_smi_present"] = shutil.which("nvidia-smi") is not None
    info["nvidia_smi_query"] = _run(
        [
            "nvidia-smi",
            "--query-gpu=name,memory.total,memory.free,driver_version,compute_cap",
            "--format=csv,noheader",
        ]
    )
    info["nvidia_smi_header"] = _run(["nvidia-smi"]).splitlines()[:1]

    try:
        import torch  # noqa: PLC0415

        info["torch_version"] = torch.__version__
        info["cuda_available"] = bool(torch.cuda.is_available())
        info["cuda_version"] = getattr(torch.version, "cuda", None)
        info["cudnn_version"] = (
            torch.backends.cudnn.version()
            if torch.backends.cudnn.is_available()
            else None
        )
        info["device_count"] = torch.cuda.device_count()
        if torch.cuda.is_available():
            devices = []
            for i in range(torch.cuda.device_count()):
                props = torch.cuda.get_device_properties(i)
                devices.append(
                    {
                        "index": i,
                        "name": props.name,
                        "total_memory_gb": round(props.total_memory / 1e9, 2),
                        "major": props.major,
                        "minor": props.minor,
                    }
                )
            info["devices"] = devices
            # A tiny allocation proves the driver and allocator really work.
            tensor = torch.zeros(1024, 1024, device="cuda")
            info["alloc_test"] = {
                "ok": True,
                "bytes": tensor.numel() * tensor.element_size(),
            }
            info["vram_allocated_gb"] = round(
                torch.cuda.memory_allocated(0) / 1e9, 4
            )
    except ImportError:
        info["torch_version"] = None
        info["cuda_available"] = False
    except Exception as exc:  # pragma: no cover - remote-only path
        info["torch_error"] = f"{type(exc).__name__}: {exc}"

    return info


def main() -> int:
    info = collect()
    print(json.dumps(info, indent=2, sort_keys=True))

    gpu_ok = bool(info.get("cuda_available")) and bool(
        info.get("nvidia_smi_present")
    )
    if not gpu_ok:
        print(
            "\nNO USABLE GPU: this host has no CUDA device. "
            "On Colab, pick a GPU runtime (Runtime > Change runtime type) or "
            "create the session with `colab new --gpu T4`.",
            file=sys.stderr,
        )
        return 1

    print(
        "\nGPU OK: {} device(s), torch {}, CUDA {}.".format(
            info["device_count"], info["torch_version"], info.get("cuda_version")
        )
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())