Spaces:
Running
Running
File size: 6,792 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | """NVML-based GPU sampling with honest unavailability reporting.
Rules:
- When NVML is unavailable, metrics are reported as None and the report says
"unavailable". Never 0, never fabricated.
- Sampling is rate-limited by config; the controller adapts frequency between
idle and active states so monitoring overhead stays negligible.
- Per-process GPU attribution uses nvmlDeviceGetComputeRunningProcesses -
this identifies VRAM use of external processes WITHOUT touching them.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional
from ..compat import optional_import
from .logger import get_logger
@dataclass
class GPUSample:
"""One sampling of a single GPU. None fields = not measurable."""
index: int
name: str
utilization_pct: Optional[float]
mem_used_mb: Optional[float]
mem_total_mb: Optional[float]
temperature_c: Optional[float]
power_w: Optional[float]
ts: float = field(default_factory=time.time)
@property
def mem_used_pct(self) -> Optional[float]:
if self.mem_used_mb is None or self.mem_total_mb in (None, 0):
return None
return 100.0 * self.mem_used_mb / self.mem_total_mb
def as_dict(self) -> dict:
return {
"index": self.index,
"name": self.name,
"utilization_pct": self.utilization_pct,
"mem_used_mb": self.mem_used_mb,
"mem_total_mb": self.mem_total_mb,
"mem_used_pct": self.mem_used_pct,
"temperature_c": self.temperature_c,
"power_w": self.power_w,
"ts": self.ts,
}
class GPUMonitor:
"""Rate-limited NVML sampler. Thread-safe. Fails soft to 'unavailable'."""
def __init__(self, sample_seconds: float = 2.0, enabled: bool = True, log_dir=None):
self.sample_seconds = max(0.5, float(sample_seconds))
self.enabled = enabled
self._log = get_logger("gpu_monitor", log_dir=log_dir, console=False)
self._lock = threading.Lock()
self._last = 0.0
self._nvml = None
self._handles: List = []
self._device_count = 0
self._names: List[str] = []
self._error_reported = False
if enabled:
self._init_nvml()
# -- NVML lifecycle -----------------------------------------------------
def _init_nvml(self) -> None:
pynvml = optional_import("pynvml")
if pynvml is None:
self._log.info("pynvml (nvidia-ml-py) not installed; GPU metrics unavailable")
return
try:
pynvml.nvmlInit()
self._nvml = pynvml
self._device_count = pynvml.nvmlDeviceGetCount()
self._handles = [
pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(self._device_count)
]
self._names = [
pynvml.nvmlDeviceGetName(h) for h in self._handles
]
# some driver/pynvml versions return bytes
self._names = [
n.decode("utf-8", "replace") if isinstance(n, bytes) else str(n)
for n in self._names
]
self._log.info("NVML initialized: %d GPU(s): %s", self._device_count, self._names)
except Exception as exc: # noqa: BLE001
self._nvml = None
self._log.warning("NVML init failed (%s); GPU metrics unavailable", exc)
def close(self) -> None:
if self._nvml is not None:
try:
self._nvml.nvmlShutdown()
except Exception: # noqa: BLE001
pass
self._nvml = None
@property
def available(self) -> bool:
return self._nvml is not None and self._device_count > 0
@property
def device_count(self) -> int:
return self._device_count
# -- sampling -----------------------------------------------------------
def _safe(self, fn, *args):
try:
return fn(*args)
except Exception: # noqa: BLE001 - per-metric absence is normal
return None
def sample_all(self, force: bool = False) -> List[GPUSample]:
"""Return one GPUSample per GPU, honoring the rate limit unless forced."""
if not self.enabled or not self.available:
return []
now = time.time()
with self._lock:
if not force and (now - self._last) < self.sample_seconds:
return []
self._last = now
out: List[GPUSample] = []
for i, handle in enumerate(self._handles):
util = self._safe(
lambda h=handle: self._nvml.nvmlDeviceGetUtilizationRates(h).gpu
)
mem = self._safe(self._nvml.nvmlDeviceGetMemoryInfo, handle)
temp = self._safe(self._nvml.nvmlDeviceGetTemperature, handle,
self._nvml.NVML_TEMPERATURE_GPU)
power = self._safe(self._nvml.nvmlDeviceGetPowerUsage, handle)
out.append(
GPUSample(
index=i,
name=self._names[i] if i < len(self._names) else f"GPU{i}",
utilization_pct=float(util) if util is not None else None,
mem_used_mb=(mem.used / (1024 * 1024)) if mem else None,
mem_total_mb=(mem.total / (1024 * 1024)) if mem else None,
temperature_c=float(temp) if temp is not None else None,
power_w=(power / 1000.0) if power is not None else None,
)
)
return out
def per_process_vram_mb(self) -> Dict[int, float]:
"""Map pid -> VRAM MB currently held by compute processes.
Read-only NVML query. This is how LPU attributes external AI processes
without injecting anything into them.
"""
result: Dict[int, float] = {}
if not self.available:
return result
for handle in self._handles:
procs = self._safe(self._nvml.nvmlDeviceGetComputeRunningProcesses, handle)
if not procs:
continue
for p in procs:
try:
used = p.usedGpuMemory
if isinstance(used, int):
result[int(p.pid)] = result.get(int(p.pid), 0.0) + used / (1024 * 1024)
except Exception: # noqa: BLE001
continue
return result
def driver_version(self) -> Optional[str]:
if not self.available:
return None
v = self._safe(self._nvml.nvmlSystemGetDriverVersion)
if isinstance(v, bytes):
v = v.decode("utf-8", "replace")
return v
|