"""NVML-based GPU sampling with honest unavailability reporting. Rules: - When NVML is unavailable, metrics are reported as None and the report says "unavailable". Never 0, never fabricated. - Sampling is rate-limited by config; the controller adapts frequency between idle and active states so monitoring overhead stays negligible. - Per-process GPU attribution uses nvmlDeviceGetComputeRunningProcesses - this identifies VRAM use of external processes WITHOUT touching them. """ from __future__ import annotations import threading import time from dataclasses import dataclass, field from pathlib import Path from typing import Dict, List, Optional from ..compat import optional_import from .logger import get_logger @dataclass class GPUSample: """One sampling of a single GPU. None fields = not measurable.""" index: int name: str utilization_pct: Optional[float] mem_used_mb: Optional[float] mem_total_mb: Optional[float] temperature_c: Optional[float] power_w: Optional[float] ts: float = field(default_factory=time.time) @property def mem_used_pct(self) -> Optional[float]: if self.mem_used_mb is None or self.mem_total_mb in (None, 0): return None return 100.0 * self.mem_used_mb / self.mem_total_mb def as_dict(self) -> dict: return { "index": self.index, "name": self.name, "utilization_pct": self.utilization_pct, "mem_used_mb": self.mem_used_mb, "mem_total_mb": self.mem_total_mb, "mem_used_pct": self.mem_used_pct, "temperature_c": self.temperature_c, "power_w": self.power_w, "ts": self.ts, } class GPUMonitor: """Rate-limited NVML sampler. Thread-safe. Fails soft to 'unavailable'.""" def __init__(self, sample_seconds: float = 2.0, enabled: bool = True, log_dir=None): self.sample_seconds = max(0.5, float(sample_seconds)) self.enabled = enabled self._log = get_logger("gpu_monitor", log_dir=log_dir, console=False) self._lock = threading.Lock() self._last = 0.0 self._nvml = None self._handles: List = [] self._device_count = 0 self._names: List[str] = [] self._error_reported = False if enabled: self._init_nvml() # -- NVML lifecycle ----------------------------------------------------- def _init_nvml(self) -> None: pynvml = optional_import("pynvml") if pynvml is None: self._log.info("pynvml (nvidia-ml-py) not installed; GPU metrics unavailable") return try: pynvml.nvmlInit() self._nvml = pynvml self._device_count = pynvml.nvmlDeviceGetCount() self._handles = [ pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(self._device_count) ] self._names = [ pynvml.nvmlDeviceGetName(h) for h in self._handles ] # some driver/pynvml versions return bytes self._names = [ n.decode("utf-8", "replace") if isinstance(n, bytes) else str(n) for n in self._names ] self._log.info("NVML initialized: %d GPU(s): %s", self._device_count, self._names) except Exception as exc: # noqa: BLE001 self._nvml = None self._log.warning("NVML init failed (%s); GPU metrics unavailable", exc) def close(self) -> None: if self._nvml is not None: try: self._nvml.nvmlShutdown() except Exception: # noqa: BLE001 pass self._nvml = None @property def available(self) -> bool: return self._nvml is not None and self._device_count > 0 @property def device_count(self) -> int: return self._device_count # -- sampling ----------------------------------------------------------- def _safe(self, fn, *args): try: return fn(*args) except Exception: # noqa: BLE001 - per-metric absence is normal return None def sample_all(self, force: bool = False) -> List[GPUSample]: """Return one GPUSample per GPU, honoring the rate limit unless forced.""" if not self.enabled or not self.available: return [] now = time.time() with self._lock: if not force and (now - self._last) < self.sample_seconds: return [] self._last = now out: List[GPUSample] = [] for i, handle in enumerate(self._handles): util = self._safe( lambda h=handle: self._nvml.nvmlDeviceGetUtilizationRates(h).gpu ) mem = self._safe(self._nvml.nvmlDeviceGetMemoryInfo, handle) temp = self._safe(self._nvml.nvmlDeviceGetTemperature, handle, self._nvml.NVML_TEMPERATURE_GPU) power = self._safe(self._nvml.nvmlDeviceGetPowerUsage, handle) out.append( GPUSample( index=i, name=self._names[i] if i < len(self._names) else f"GPU{i}", utilization_pct=float(util) if util is not None else None, mem_used_mb=(mem.used / (1024 * 1024)) if mem else None, mem_total_mb=(mem.total / (1024 * 1024)) if mem else None, temperature_c=float(temp) if temp is not None else None, power_w=(power / 1000.0) if power is not None else None, ) ) return out def per_process_vram_mb(self) -> Dict[int, float]: """Map pid -> VRAM MB currently held by compute processes. Read-only NVML query. This is how LPU attributes external AI processes without injecting anything into them. """ result: Dict[int, float] = {} if not self.available: return result for handle in self._handles: procs = self._safe(self._nvml.nvmlDeviceGetComputeRunningProcesses, handle) if not procs: continue for p in procs: try: used = p.usedGpuMemory if isinstance(used, int): result[int(p.pid)] = result.get(int(p.pid), 0.0) + used / (1024 * 1024) except Exception: # noqa: BLE001 continue return result def driver_version(self) -> Optional[str]: if not self.available: return None v = self._safe(self._nvml.nvmlSystemGetDriverVersion) if isinstance(v, bytes): v = v.decode("utf-8", "replace") return v