"""GPU discovery: torch CUDA properties + NVML live state. Detection order (all real, no simulation): 1. torch.cuda.is_available() + get_device_properties(i) -> static facts 2. pynvml (NVML) -> live util/free VRAM 3. torch.cuda.can_device_access_peer(i, j) -> P2P matrix The controller uses only NVML (torch-free); in-process paths add torch facts when the target interpreter has CUDA torch. """ from __future__ import annotations from dataclasses import dataclass from typing import Dict, List, Optional from ..compat import require_torch, pynvml from .capabilities import GPUCapabilities @dataclass class GPUInfo: """Everything LPU knows about one GPU, static + live snapshot.""" capabilities: GPUCapabilities free_vram_mb: Optional[float] = None # from mem_get_info / NVML utilization_pct: Optional[float] = None # NVML only temperature_c: Optional[float] = None power_w: Optional[float] = None driver_version: Optional[str] = None def summary(self) -> dict: d = self.capabilities.summary() d.update({ "free_vram_mb": self.free_vram_mb, "utilization_pct": self.utilization_pct, "temperature_c": self.temperature_c, "power_w": self.power_w, "driver_version": self.driver_version, }) return d def detect_gpus(device_ids=None, include_live=True) -> List[GPUInfo]: """Detect CUDA GPUs through the current interpreter's torch build. Requires torch WITH CUDA (i.e., must run inside the target project's interpreter). The controller never calls this; it uses GPUMonitor/NVML. """ torch = require_torch() out: List[GPUInfo] = [] if not torch.cuda.is_available(): return out n = torch.cuda.device_count() nv = pynvml() if include_live else None driver = None if nv is not None: try: v = nv.nvmlSystemGetDriverVersion() driver = v.decode() if isinstance(v, bytes) else str(v) except Exception: # noqa: BLE001 driver = None indices = list(device_ids) if device_ids else range(n) for i in indices: if not isinstance(i, int) or i < 0 or i >= n: continue props = torch.cuda.get_device_properties(i) caps = GPUCapabilities( index=i, name=props.name, compute_capability=(props.major, props.minor), total_vram_mb=props.total_memory / (1024 * 1024), multiprocessor_count=getattr(props, "multi_processor_count", None), uuid=str(getattr(props, "uuid", "")) or None, ) free_mb = None if include_live: try: free_b, _total = torch.cuda.mem_get_info(i) free_mb = free_b / (1024 * 1024) except Exception: # noqa: BLE001 pass info = GPUInfo(capabilities=caps, free_vram_mb=free_mb, driver_version=driver) if nv is not None: try: h = nv.nvmlDeviceGetHandleByIndex(i) info.utilization_pct = float(nv.nvmlDeviceGetUtilizationRates(h).gpu) mem = nv.nvmlDeviceGetMemoryInfo(h) info.free_vram_mb = mem.free / (1024 * 1024) info.temperature_c = float( nv.nvmlDeviceGetTemperature(h, nv.NVML_TEMPERATURE_GPU) ) p = nv.nvmlDeviceGetPowerUsage(h) info.power_w = p / 1000.0 except Exception: # noqa: BLE001 - live stats are best-effort pass out.append(info) if nv is not None: try: nv.nvmlShutdown() except Exception: # noqa: BLE001 pass return out def p2p_matrix(gpu_count: int) -> Dict[str, bool]: """Measured peer-access matrix: {(i,j): can_access}. Uses torch.cuda.can_device_access_peer which queries the driver. On a dual-P100 PCIe system without P2P the answer is typically False and all cross-GPU traffic goes through pinned host memory - the scheduler must know this BEFORE choosing a strategy (it changes expected transfer cost). """ torch = require_torch() matrix: Dict[str, bool] = {} if not torch.cuda.is_available() or gpu_count < 2: return matrix for i in range(gpu_count): for j in range(gpu_count): if i == j: continue try: matrix[f"{i}->{j}"] = bool(torch.cuda.can_device_access_peer(i, j)) except Exception: # noqa: BLE001 matrix[f"{i}->{j}"] = False return matrix def torch_cuda_summary() -> dict: """torch-side CUDA facts for diagnostics (versions + build).""" torch = require_torch() return { "torch_version": torch.__version__, "cuda_available": torch.cuda.is_available(), "torch_built_with_cuda": torch.version.cuda is not None, "torch_cuda_version": torch.version.cuda, "cudnn_version": torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else None, "cudnn_enabled": torch.backends.cudnn.is_available() and torch.backends.cudnn.enabled, "device_count": torch.cuda.device_count() if torch.cuda.is_available() else 0, }