Virtual-LPU / lpu /hardware /gpu_detector.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
5.33 kB
"""GPU discovery: torch CUDA properties + NVML live state.
Detection order (all real, no simulation):
1. torch.cuda.is_available() + get_device_properties(i) -> static facts
2. pynvml (NVML) -> live util/free VRAM
3. torch.cuda.can_device_access_peer(i, j) -> P2P matrix
The controller uses only NVML (torch-free); in-process paths add torch facts
when the target interpreter has CUDA torch.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, List, Optional
from ..compat import require_torch, pynvml
from .capabilities import GPUCapabilities
@dataclass
class GPUInfo:
"""Everything LPU knows about one GPU, static + live snapshot."""
capabilities: GPUCapabilities
free_vram_mb: Optional[float] = None # from mem_get_info / NVML
utilization_pct: Optional[float] = None # NVML only
temperature_c: Optional[float] = None
power_w: Optional[float] = None
driver_version: Optional[str] = None
def summary(self) -> dict:
d = self.capabilities.summary()
d.update({
"free_vram_mb": self.free_vram_mb,
"utilization_pct": self.utilization_pct,
"temperature_c": self.temperature_c,
"power_w": self.power_w,
"driver_version": self.driver_version,
})
return d
def detect_gpus(device_ids=None, include_live=True) -> List[GPUInfo]:
"""Detect CUDA GPUs through the current interpreter's torch build.
Requires torch WITH CUDA (i.e., must run inside the target project's
interpreter). The controller never calls this; it uses GPUMonitor/NVML.
"""
torch = require_torch()
out: List[GPUInfo] = []
if not torch.cuda.is_available():
return out
n = torch.cuda.device_count()
nv = pynvml() if include_live else None
driver = None
if nv is not None:
try:
v = nv.nvmlSystemGetDriverVersion()
driver = v.decode() if isinstance(v, bytes) else str(v)
except Exception: # noqa: BLE001
driver = None
indices = list(device_ids) if device_ids else range(n)
for i in indices:
if not isinstance(i, int) or i < 0 or i >= n:
continue
props = torch.cuda.get_device_properties(i)
caps = GPUCapabilities(
index=i,
name=props.name,
compute_capability=(props.major, props.minor),
total_vram_mb=props.total_memory / (1024 * 1024),
multiprocessor_count=getattr(props, "multi_processor_count", None),
uuid=str(getattr(props, "uuid", "")) or None,
)
free_mb = None
if include_live:
try:
free_b, _total = torch.cuda.mem_get_info(i)
free_mb = free_b / (1024 * 1024)
except Exception: # noqa: BLE001
pass
info = GPUInfo(capabilities=caps, free_vram_mb=free_mb, driver_version=driver)
if nv is not None:
try:
h = nv.nvmlDeviceGetHandleByIndex(i)
info.utilization_pct = float(nv.nvmlDeviceGetUtilizationRates(h).gpu)
mem = nv.nvmlDeviceGetMemoryInfo(h)
info.free_vram_mb = mem.free / (1024 * 1024)
info.temperature_c = float(
nv.nvmlDeviceGetTemperature(h, nv.NVML_TEMPERATURE_GPU)
)
p = nv.nvmlDeviceGetPowerUsage(h)
info.power_w = p / 1000.0
except Exception: # noqa: BLE001 - live stats are best-effort
pass
out.append(info)
if nv is not None:
try:
nv.nvmlShutdown()
except Exception: # noqa: BLE001
pass
return out
def p2p_matrix(gpu_count: int) -> Dict[str, bool]:
"""Measured peer-access matrix: {(i,j): can_access}.
Uses torch.cuda.can_device_access_peer which queries the driver. On a
dual-P100 PCIe system without P2P the answer is typically False and all
cross-GPU traffic goes through pinned host memory - the scheduler must
know this BEFORE choosing a strategy (it changes expected transfer cost).
"""
torch = require_torch()
matrix: Dict[str, bool] = {}
if not torch.cuda.is_available() or gpu_count < 2:
return matrix
for i in range(gpu_count):
for j in range(gpu_count):
if i == j:
continue
try:
matrix[f"{i}->{j}"] = bool(torch.cuda.can_device_access_peer(i, j))
except Exception: # noqa: BLE001
matrix[f"{i}->{j}"] = False
return matrix
def torch_cuda_summary() -> dict:
"""torch-side CUDA facts for diagnostics (versions + build)."""
torch = require_torch()
return {
"torch_version": torch.__version__,
"cuda_available": torch.cuda.is_available(),
"torch_built_with_cuda": torch.version.cuda is not None,
"torch_cuda_version": torch.version.cuda,
"cudnn_version": torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else None,
"cudnn_enabled": torch.backends.cudnn.is_available() and torch.backends.cudnn.enabled,
"device_count": torch.cuda.device_count() if torch.cuda.is_available() else 0,
}