File size: 6,792 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
"""NVML-based GPU sampling with honest unavailability reporting.

Rules:
- When NVML is unavailable, metrics are reported as None and the report says
  "unavailable". Never 0, never fabricated.
- Sampling is rate-limited by config; the controller adapts frequency between
  idle and active states so monitoring overhead stays negligible.
- Per-process GPU attribution uses nvmlDeviceGetComputeRunningProcesses -
  this identifies VRAM use of external processes WITHOUT touching them.
"""

from __future__ import annotations

import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional

from ..compat import optional_import
from .logger import get_logger


@dataclass
class GPUSample:
    """One sampling of a single GPU. None fields = not measurable."""
    index: int
    name: str
    utilization_pct: Optional[float]
    mem_used_mb: Optional[float]
    mem_total_mb: Optional[float]
    temperature_c: Optional[float]
    power_w: Optional[float]
    ts: float = field(default_factory=time.time)

    @property
    def mem_used_pct(self) -> Optional[float]:
        if self.mem_used_mb is None or self.mem_total_mb in (None, 0):
            return None
        return 100.0 * self.mem_used_mb / self.mem_total_mb

    def as_dict(self) -> dict:
        return {
            "index": self.index,
            "name": self.name,
            "utilization_pct": self.utilization_pct,
            "mem_used_mb": self.mem_used_mb,
            "mem_total_mb": self.mem_total_mb,
            "mem_used_pct": self.mem_used_pct,
            "temperature_c": self.temperature_c,
            "power_w": self.power_w,
            "ts": self.ts,
        }


class GPUMonitor:
    """Rate-limited NVML sampler. Thread-safe. Fails soft to 'unavailable'."""

    def __init__(self, sample_seconds: float = 2.0, enabled: bool = True, log_dir=None):
        self.sample_seconds = max(0.5, float(sample_seconds))
        self.enabled = enabled
        self._log = get_logger("gpu_monitor", log_dir=log_dir, console=False)
        self._lock = threading.Lock()
        self._last = 0.0
        self._nvml = None
        self._handles: List = []
        self._device_count = 0
        self._names: List[str] = []
        self._error_reported = False
        if enabled:
            self._init_nvml()

    # -- NVML lifecycle -----------------------------------------------------
    def _init_nvml(self) -> None:
        pynvml = optional_import("pynvml")
        if pynvml is None:
            self._log.info("pynvml (nvidia-ml-py) not installed; GPU metrics unavailable")
            return
        try:
            pynvml.nvmlInit()
            self._nvml = pynvml
            self._device_count = pynvml.nvmlDeviceGetCount()
            self._handles = [
                pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(self._device_count)
            ]
            self._names = [
                pynvml.nvmlDeviceGetName(h) for h in self._handles
            ]
            # some driver/pynvml versions return bytes
            self._names = [
                n.decode("utf-8", "replace") if isinstance(n, bytes) else str(n)
                for n in self._names
            ]
            self._log.info("NVML initialized: %d GPU(s): %s", self._device_count, self._names)
        except Exception as exc:  # noqa: BLE001
            self._nvml = None
            self._log.warning("NVML init failed (%s); GPU metrics unavailable", exc)

    def close(self) -> None:
        if self._nvml is not None:
            try:
                self._nvml.nvmlShutdown()
            except Exception:  # noqa: BLE001
                pass
            self._nvml = None

    @property
    def available(self) -> bool:
        return self._nvml is not None and self._device_count > 0

    @property
    def device_count(self) -> int:
        return self._device_count

    # -- sampling -----------------------------------------------------------
    def _safe(self, fn, *args):
        try:
            return fn(*args)
        except Exception:  # noqa: BLE001 - per-metric absence is normal
            return None

    def sample_all(self, force: bool = False) -> List[GPUSample]:
        """Return one GPUSample per GPU, honoring the rate limit unless forced."""
        if not self.enabled or not self.available:
            return []
        now = time.time()
        with self._lock:
            if not force and (now - self._last) < self.sample_seconds:
                return []
            self._last = now
        out: List[GPUSample] = []
        for i, handle in enumerate(self._handles):
            util = self._safe(
                lambda h=handle: self._nvml.nvmlDeviceGetUtilizationRates(h).gpu
            )
            mem = self._safe(self._nvml.nvmlDeviceGetMemoryInfo, handle)
            temp = self._safe(self._nvml.nvmlDeviceGetTemperature, handle,
                              self._nvml.NVML_TEMPERATURE_GPU)
            power = self._safe(self._nvml.nvmlDeviceGetPowerUsage, handle)
            out.append(
                GPUSample(
                    index=i,
                    name=self._names[i] if i < len(self._names) else f"GPU{i}",
                    utilization_pct=float(util) if util is not None else None,
                    mem_used_mb=(mem.used / (1024 * 1024)) if mem else None,
                    mem_total_mb=(mem.total / (1024 * 1024)) if mem else None,
                    temperature_c=float(temp) if temp is not None else None,
                    power_w=(power / 1000.0) if power is not None else None,
                )
            )
        return out

    def per_process_vram_mb(self) -> Dict[int, float]:
        """Map pid -> VRAM MB currently held by compute processes.

        Read-only NVML query. This is how LPU attributes external AI processes
        without injecting anything into them.
        """
        result: Dict[int, float] = {}
        if not self.available:
            return result
        for handle in self._handles:
            procs = self._safe(self._nvml.nvmlDeviceGetComputeRunningProcesses, handle)
            if not procs:
                continue
            for p in procs:
                try:
                    used = p.usedGpuMemory
                    if isinstance(used, int):
                        result[int(p.pid)] = result.get(int(p.pid), 0.0) + used / (1024 * 1024)
                except Exception:  # noqa: BLE001
                    continue
        return result

    def driver_version(self) -> Optional[str]:
        if not self.available:
            return None
        v = self._safe(self._nvml.nvmlSystemGetDriverVersion)
        if isinstance(v, bytes):
            v = v.decode("utf-8", "replace")
        return v