Spaces:
Running
Running
Download lpu/monitoring/gpu_monitor.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 6.79 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/monitoring/gpu_monitor.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/monitoring/gpu_monitor.py
-
curl -L -o gpu_monitor.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/monitoring/gpu_monitor.py
6.79 kB
| """NVML-based GPU sampling with honest unavailability reporting. | |
| Rules: | |
| - When NVML is unavailable, metrics are reported as None and the report says | |
| "unavailable". Never 0, never fabricated. | |
| - Sampling is rate-limited by config; the controller adapts frequency between | |
| idle and active states so monitoring overhead stays negligible. | |
| - Per-process GPU attribution uses nvmlDeviceGetComputeRunningProcesses - | |
| this identifies VRAM use of external processes WITHOUT touching them. | |
| """ | |
| from __future__ import annotations | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Dict, List, Optional | |
| from ..compat import optional_import | |
| from .logger import get_logger | |
| class GPUSample: | |
| """One sampling of a single GPU. None fields = not measurable.""" | |
| index: int | |
| name: str | |
| utilization_pct: Optional[float] | |
| mem_used_mb: Optional[float] | |
| mem_total_mb: Optional[float] | |
| temperature_c: Optional[float] | |
| power_w: Optional[float] | |
| ts: float = field(default_factory=time.time) | |
| def mem_used_pct(self) -> Optional[float]: | |
| if self.mem_used_mb is None or self.mem_total_mb in (None, 0): | |
| return None | |
| return 100.0 * self.mem_used_mb / self.mem_total_mb | |
| def as_dict(self) -> dict: | |
| return { | |
| "index": self.index, | |
| "name": self.name, | |
| "utilization_pct": self.utilization_pct, | |
| "mem_used_mb": self.mem_used_mb, | |
| "mem_total_mb": self.mem_total_mb, | |
| "mem_used_pct": self.mem_used_pct, | |
| "temperature_c": self.temperature_c, | |
| "power_w": self.power_w, | |
| "ts": self.ts, | |
| } | |
| class GPUMonitor: | |
| """Rate-limited NVML sampler. Thread-safe. Fails soft to 'unavailable'.""" | |
| def __init__(self, sample_seconds: float = 2.0, enabled: bool = True, log_dir=None): | |
| self.sample_seconds = max(0.5, float(sample_seconds)) | |
| self.enabled = enabled | |
| self._log = get_logger("gpu_monitor", log_dir=log_dir, console=False) | |
| self._lock = threading.Lock() | |
| self._last = 0.0 | |
| self._nvml = None | |
| self._handles: List = [] | |
| self._device_count = 0 | |
| self._names: List[str] = [] | |
| self._error_reported = False | |
| if enabled: | |
| self._init_nvml() | |
| # -- NVML lifecycle ----------------------------------------------------- | |
| def _init_nvml(self) -> None: | |
| pynvml = optional_import("pynvml") | |
| if pynvml is None: | |
| self._log.info("pynvml (nvidia-ml-py) not installed; GPU metrics unavailable") | |
| return | |
| try: | |
| pynvml.nvmlInit() | |
| self._nvml = pynvml | |
| self._device_count = pynvml.nvmlDeviceGetCount() | |
| self._handles = [ | |
| pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(self._device_count) | |
| ] | |
| self._names = [ | |
| pynvml.nvmlDeviceGetName(h) for h in self._handles | |
| ] | |
| # some driver/pynvml versions return bytes | |
| self._names = [ | |
| n.decode("utf-8", "replace") if isinstance(n, bytes) else str(n) | |
| for n in self._names | |
| ] | |
| self._log.info("NVML initialized: %d GPU(s): %s", self._device_count, self._names) | |
| except Exception as exc: # noqa: BLE001 | |
| self._nvml = None | |
| self._log.warning("NVML init failed (%s); GPU metrics unavailable", exc) | |
| def close(self) -> None: | |
| if self._nvml is not None: | |
| try: | |
| self._nvml.nvmlShutdown() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| self._nvml = None | |
| def available(self) -> bool: | |
| return self._nvml is not None and self._device_count > 0 | |
| def device_count(self) -> int: | |
| return self._device_count | |
| # -- sampling ----------------------------------------------------------- | |
| def _safe(self, fn, *args): | |
| try: | |
| return fn(*args) | |
| except Exception: # noqa: BLE001 - per-metric absence is normal | |
| return None | |
| def sample_all(self, force: bool = False) -> List[GPUSample]: | |
| """Return one GPUSample per GPU, honoring the rate limit unless forced.""" | |
| if not self.enabled or not self.available: | |
| return [] | |
| now = time.time() | |
| with self._lock: | |
| if not force and (now - self._last) < self.sample_seconds: | |
| return [] | |
| self._last = now | |
| out: List[GPUSample] = [] | |
| for i, handle in enumerate(self._handles): | |
| util = self._safe( | |
| lambda h=handle: self._nvml.nvmlDeviceGetUtilizationRates(h).gpu | |
| ) | |
| mem = self._safe(self._nvml.nvmlDeviceGetMemoryInfo, handle) | |
| temp = self._safe(self._nvml.nvmlDeviceGetTemperature, handle, | |
| self._nvml.NVML_TEMPERATURE_GPU) | |
| power = self._safe(self._nvml.nvmlDeviceGetPowerUsage, handle) | |
| out.append( | |
| GPUSample( | |
| index=i, | |
| name=self._names[i] if i < len(self._names) else f"GPU{i}", | |
| utilization_pct=float(util) if util is not None else None, | |
| mem_used_mb=(mem.used / (1024 * 1024)) if mem else None, | |
| mem_total_mb=(mem.total / (1024 * 1024)) if mem else None, | |
| temperature_c=float(temp) if temp is not None else None, | |
| power_w=(power / 1000.0) if power is not None else None, | |
| ) | |
| ) | |
| return out | |
| def per_process_vram_mb(self) -> Dict[int, float]: | |
| """Map pid -> VRAM MB currently held by compute processes. | |
| Read-only NVML query. This is how LPU attributes external AI processes | |
| without injecting anything into them. | |
| """ | |
| result: Dict[int, float] = {} | |
| if not self.available: | |
| return result | |
| for handle in self._handles: | |
| procs = self._safe(self._nvml.nvmlDeviceGetComputeRunningProcesses, handle) | |
| if not procs: | |
| continue | |
| for p in procs: | |
| try: | |
| used = p.usedGpuMemory | |
| if isinstance(used, int): | |
| result[int(p.pid)] = result.get(int(p.pid), 0.0) + used / (1024 * 1024) | |
| except Exception: # noqa: BLE001 | |
| continue | |
| return result | |
| def driver_version(self) -> Optional[str]: | |
| if not self.available: | |
| return None | |
| v = self._safe(self._nvml.nvmlSystemGetDriverVersion) | |
| if isinstance(v, bytes): | |
| v = v.decode("utf-8", "replace") | |
| return v | |