Virtual-LPU / lpu /monitoring /gpu_monitor.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
6.79 kB
"""NVML-based GPU sampling with honest unavailability reporting.
Rules:
- When NVML is unavailable, metrics are reported as None and the report says
"unavailable". Never 0, never fabricated.
- Sampling is rate-limited by config; the controller adapts frequency between
idle and active states so monitoring overhead stays negligible.
- Per-process GPU attribution uses nvmlDeviceGetComputeRunningProcesses -
this identifies VRAM use of external processes WITHOUT touching them.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional
from ..compat import optional_import
from .logger import get_logger
@dataclass
class GPUSample:
"""One sampling of a single GPU. None fields = not measurable."""
index: int
name: str
utilization_pct: Optional[float]
mem_used_mb: Optional[float]
mem_total_mb: Optional[float]
temperature_c: Optional[float]
power_w: Optional[float]
ts: float = field(default_factory=time.time)
@property
def mem_used_pct(self) -> Optional[float]:
if self.mem_used_mb is None or self.mem_total_mb in (None, 0):
return None
return 100.0 * self.mem_used_mb / self.mem_total_mb
def as_dict(self) -> dict:
return {
"index": self.index,
"name": self.name,
"utilization_pct": self.utilization_pct,
"mem_used_mb": self.mem_used_mb,
"mem_total_mb": self.mem_total_mb,
"mem_used_pct": self.mem_used_pct,
"temperature_c": self.temperature_c,
"power_w": self.power_w,
"ts": self.ts,
}
class GPUMonitor:
"""Rate-limited NVML sampler. Thread-safe. Fails soft to 'unavailable'."""
def __init__(self, sample_seconds: float = 2.0, enabled: bool = True, log_dir=None):
self.sample_seconds = max(0.5, float(sample_seconds))
self.enabled = enabled
self._log = get_logger("gpu_monitor", log_dir=log_dir, console=False)
self._lock = threading.Lock()
self._last = 0.0
self._nvml = None
self._handles: List = []
self._device_count = 0
self._names: List[str] = []
self._error_reported = False
if enabled:
self._init_nvml()
# -- NVML lifecycle -----------------------------------------------------
def _init_nvml(self) -> None:
pynvml = optional_import("pynvml")
if pynvml is None:
self._log.info("pynvml (nvidia-ml-py) not installed; GPU metrics unavailable")
return
try:
pynvml.nvmlInit()
self._nvml = pynvml
self._device_count = pynvml.nvmlDeviceGetCount()
self._handles = [
pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(self._device_count)
]
self._names = [
pynvml.nvmlDeviceGetName(h) for h in self._handles
]
# some driver/pynvml versions return bytes
self._names = [
n.decode("utf-8", "replace") if isinstance(n, bytes) else str(n)
for n in self._names
]
self._log.info("NVML initialized: %d GPU(s): %s", self._device_count, self._names)
except Exception as exc: # noqa: BLE001
self._nvml = None
self._log.warning("NVML init failed (%s); GPU metrics unavailable", exc)
def close(self) -> None:
if self._nvml is not None:
try:
self._nvml.nvmlShutdown()
except Exception: # noqa: BLE001
pass
self._nvml = None
@property
def available(self) -> bool:
return self._nvml is not None and self._device_count > 0
@property
def device_count(self) -> int:
return self._device_count
# -- sampling -----------------------------------------------------------
def _safe(self, fn, *args):
try:
return fn(*args)
except Exception: # noqa: BLE001 - per-metric absence is normal
return None
def sample_all(self, force: bool = False) -> List[GPUSample]:
"""Return one GPUSample per GPU, honoring the rate limit unless forced."""
if not self.enabled or not self.available:
return []
now = time.time()
with self._lock:
if not force and (now - self._last) < self.sample_seconds:
return []
self._last = now
out: List[GPUSample] = []
for i, handle in enumerate(self._handles):
util = self._safe(
lambda h=handle: self._nvml.nvmlDeviceGetUtilizationRates(h).gpu
)
mem = self._safe(self._nvml.nvmlDeviceGetMemoryInfo, handle)
temp = self._safe(self._nvml.nvmlDeviceGetTemperature, handle,
self._nvml.NVML_TEMPERATURE_GPU)
power = self._safe(self._nvml.nvmlDeviceGetPowerUsage, handle)
out.append(
GPUSample(
index=i,
name=self._names[i] if i < len(self._names) else f"GPU{i}",
utilization_pct=float(util) if util is not None else None,
mem_used_mb=(mem.used / (1024 * 1024)) if mem else None,
mem_total_mb=(mem.total / (1024 * 1024)) if mem else None,
temperature_c=float(temp) if temp is not None else None,
power_w=(power / 1000.0) if power is not None else None,
)
)
return out
def per_process_vram_mb(self) -> Dict[int, float]:
"""Map pid -> VRAM MB currently held by compute processes.
Read-only NVML query. This is how LPU attributes external AI processes
without injecting anything into them.
"""
result: Dict[int, float] = {}
if not self.available:
return result
for handle in self._handles:
procs = self._safe(self._nvml.nvmlDeviceGetComputeRunningProcesses, handle)
if not procs:
continue
for p in procs:
try:
used = p.usedGpuMemory
if isinstance(used, int):
result[int(p.pid)] = result.get(int(p.pid), 0.0) + used / (1024 * 1024)
except Exception: # noqa: BLE001
continue
return result
def driver_version(self) -> Optional[str]:
if not self.available:
return None
v = self._safe(self._nvml.nvmlSystemGetDriverVersion)
if isinstance(v, bytes):
v = v.decode("utf-8", "replace")
return v