Buckets:

download
raw
23.9 kB
"""Real-time GPU monitoring: VRAM, utilization, temperature, power and alerts.
Tracks per-GPU metrics and active processes, and raises alerts on threshold
breaches.
Author: effGen Team
License: Apache-2.0
"""
from __future__ import annotations
import logging
import time
import warnings
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import Enum
from threading import Event, Lock, Thread
from types import TracebackType
from typing import Any
try:
import torch
TORCH_AVAILABLE = True
except ImportError:
TORCH_AVAILABLE = False
warnings.warn("PyTorch not available. GPU monitoring will use CPU fallback.")
try:
import pynvml
PYNVML_AVAILABLE = True
except ImportError:
PYNVML_AVAILABLE = False
warnings.warn(
"pynvml not available. Advanced GPU monitoring features disabled. "
"Install with: pip install nvidia-ml-py3"
)
logger = logging.getLogger(__name__)
class MetricType(Enum):
"""Types of GPU metrics."""
VRAM_USAGE = "vram_usage"
VRAM_UTILIZATION = "vram_utilization"
GPU_UTILIZATION = "gpu_utilization"
TEMPERATURE = "temperature"
POWER_USAGE = "power_usage"
FAN_SPEED = "fan_speed"
class AlertLevel(Enum):
"""Alert severity levels."""
INFO = "info"
WARNING = "warning"
CRITICAL = "critical"
@dataclass
class GPUMetrics:
"""Metrics for a single GPU at a point in time."""
device_id: int
timestamp: float
vram_total: int # bytes
vram_used: int # bytes
vram_free: int # bytes
gpu_utilization: float # 0.0 to 1.0
memory_utilization: float # 0.0 to 1.0
temperature: float | None = None # Celsius
power_usage: float | None = None # Watts
power_limit: float | None = None # Watts
fan_speed: float | None = None # 0.0 to 1.0
processes: list[dict[str, Any]] = field(default_factory=list)
@property
def vram_used_gb(self) -> float:
"""VRAM used in GB."""
return self.vram_used / (1024**3)
@property
def vram_total_gb(self) -> float:
"""Total VRAM in GB."""
return self.vram_total / (1024**3)
@property
def vram_free_gb(self) -> float:
"""Free VRAM in GB."""
return self.vram_free / (1024**3)
@dataclass
class Alert:
"""Alert for threshold breach."""
device_id: int
metric_type: MetricType
level: AlertLevel
message: str
value: float
threshold: float
timestamp: float = field(default_factory=time.time)
@dataclass
class MonitorConfig:
"""Configuration for GPU monitor."""
update_interval: float = 1.0 # seconds
enable_alerts: bool = True
enable_logging: bool = True
log_interval: float = 60.0 # seconds
# Alert thresholds
vram_warning_threshold: float = 0.8 # 80%
vram_critical_threshold: float = 0.95 # 95%
gpu_utilization_warning: float = 0.9 # 90%
temperature_warning: float = 80.0 # Celsius
temperature_critical: float = 90.0 # Celsius
power_warning_threshold: float = 0.9 # 90% of power limit
class GPUMonitor:
"""
Real-time GPU monitoring with alerts and metrics logging.
This class provides:
- Real-time VRAM usage tracking
- GPU utilization monitoring
- Temperature and power monitoring
- Active process tracking
- Alert system on threshold breach
- Comprehensive metrics logging
Example:
>>> config = MonitorConfig(update_interval=2.0)
>>> monitor = GPUMonitor(config)
>>> monitor.start()
>>>
>>> # Current metrics, one entry per visible device.
>>> usage = [
... f"GPU {device_id}: {metric.vram_used_gb:.2f}GB used"
... for device_id, metric in monitor.get_metrics().items()
... ]
>>>
>>> # A callback runs on the monitor's own thread the moment a
>>> # threshold is breached, so it records the alert rather than
>>> # writing to a console it does not own.
>>> alerts = []
>>> monitor.add_alert_callback(lambda alert: alerts.append(alert.message))
>>>
>>> monitor.stop()
"""
def __init__(self, config: MonitorConfig | None = None) -> None:
"""
Initialize GPU monitor.
Args:
config: Monitor configuration (uses defaults if None)
"""
self.config = config or MonitorConfig()
self._lock = Lock()
self._running = False
self._stop_event = Event()
self._monitor_thread: Thread | None = None
self._log_thread: Thread | None = None
# Metrics storage
self._current_metrics: dict[int, GPUMetrics] = {}
self._metrics_history: dict[int, list[GPUMetrics]] = {}
self._max_history_size = 1000
# Alerts
self._alerts: list[Alert] = []
self._max_alerts = 100
self._alert_callbacks: list[Callable[[Alert], None]] = []
# NVML initialization
self._nvml_initialized = False
if PYNVML_AVAILABLE:
try:
pynvml.nvmlInit()
self._nvml_initialized = True
logger.info("NVML initialized successfully")
except Exception as e:
logger.warning(f"Failed to initialize NVML: {e}")
# Discover devices
self._device_handles: dict[int, Any] = {}
self._discover_devices()
logger.info(f"GPUMonitor initialized for {len(self._device_handles)} device(s)")
def _discover_devices(self) -> None:
"""Discover available GPU devices"""
if self._nvml_initialized:
try:
device_count = pynvml.nvmlDeviceGetCount()
for i in range(device_count):
handle = pynvml.nvmlDeviceGetHandleByIndex(i)
self._device_handles[i] = handle
self._metrics_history[i] = []
logger.info(f"Discovered GPU {i}")
except Exception as e:
logger.error(f"Error discovering NVML devices: {e}")
elif TORCH_AVAILABLE and torch.cuda.is_available():
device_count = torch.cuda.device_count()
for i in range(device_count):
self._device_handles[i] = None # PyTorch doesn't use handles
self._metrics_history[i] = []
logger.info(f"Discovered GPU {i} (PyTorch)")
def start(self) -> None:
"""Start monitoring in background thread."""
if self._running:
logger.warning("Monitor already running")
return
self._running = True
self._stop_event.clear()
# Start monitoring thread
self._monitor_thread = Thread(target=self._monitor_loop, daemon=True)
self._monitor_thread.start()
# Start logging thread if enabled
if self.config.enable_logging:
self._log_thread = Thread(target=self._log_loop, daemon=True)
self._log_thread.start()
logger.info("GPU monitoring started")
def stop(self) -> None:
"""Stop monitoring."""
if not self._running:
return
self._running = False
self._stop_event.set()
# Wait for threads to finish
if self._monitor_thread:
self._monitor_thread.join(timeout=5.0)
if self._log_thread:
self._log_thread.join(timeout=5.0)
logger.info("GPU monitoring stopped")
def _monitor_loop(self) -> None:
"""Main monitoring loop"""
while self._running:
try:
self._update_metrics()
if self.config.enable_alerts:
self._check_thresholds()
except Exception as e:
logger.error(f"Error in monitor loop: {e}")
# Sleep with interruptible wait
self._stop_event.wait(self.config.update_interval)
def _log_loop(self) -> None:
"""Periodic logging loop"""
while self._running:
try:
self._log_metrics()
except Exception as e:
logger.error(f"Error in log loop: {e}")
# Sleep with interruptible wait
self._stop_event.wait(self.config.log_interval)
def _update_metrics(self) -> None:
"""Update metrics for all devices"""
timestamp = time.time()
for device_id in self._device_handles:
try:
metrics = self._collect_metrics(device_id, timestamp)
with self._lock:
self._current_metrics[device_id] = metrics
# Add to history
self._metrics_history[device_id].append(metrics)
# Limit history size
if len(self._metrics_history[device_id]) > self._max_history_size:
self._metrics_history[device_id].pop(0)
except Exception as e:
logger.error(f"Error collecting metrics for GPU {device_id}: {e}")
def _collect_metrics(self, device_id: int, timestamp: float) -> GPUMetrics:
"""
Collect metrics for a specific device.
Args:
device_id: GPU device ID
timestamp: Current timestamp
Returns:
GPUMetrics object
"""
if self._nvml_initialized:
return self._collect_nvml_metrics(device_id, timestamp)
elif TORCH_AVAILABLE and torch.cuda.is_available():
return self._collect_torch_metrics(device_id, timestamp)
else:
# CPU fallback
return GPUMetrics(
device_id=device_id,
timestamp=timestamp,
vram_total=0,
vram_used=0,
vram_free=0,
gpu_utilization=0.0,
memory_utilization=0.0
)
def _collect_nvml_metrics(self, device_id: int, timestamp: float) -> GPUMetrics:
"""Collect metrics using NVML (most detailed)"""
handle = self._device_handles[device_id]
# Memory info
mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
vram_total = mem_info.total
vram_used = mem_info.used
vram_free = mem_info.free
# Utilization
utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
gpu_util = utilization.gpu / 100.0
mem_util = utilization.memory / 100.0
# Temperature
try:
temperature = float(pynvml.nvmlDeviceGetTemperature(
handle, pynvml.NVML_TEMPERATURE_GPU
))
except Exception:
logger.debug(f"Could not read temperature for GPU {device_id}")
temperature = None
# Power
try:
power_usage = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0 # mW to W
power_limit = pynvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000.0
except Exception:
logger.debug(f"Could not read power info for GPU {device_id}")
power_usage = None
power_limit = None
# Fan speed
try:
fan_speed = pynvml.nvmlDeviceGetFanSpeed(handle) / 100.0
except Exception:
logger.debug(f"Could not read fan speed for GPU {device_id}")
fan_speed = None
# Processes
processes = []
try:
proc_infos = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
for proc in proc_infos:
processes.append({
'pid': proc.pid,
'used_memory': proc.usedGpuMemory,
})
except Exception:
logger.debug(f"Could not read process info for GPU {device_id}")
return GPUMetrics(
device_id=device_id,
timestamp=timestamp,
vram_total=vram_total,
vram_used=vram_used,
vram_free=vram_free,
gpu_utilization=gpu_util,
memory_utilization=mem_util,
temperature=temperature,
power_usage=power_usage,
power_limit=power_limit,
fan_speed=fan_speed,
processes=processes
)
def _collect_torch_metrics(self, device_id: int, timestamp: float) -> GPUMetrics:
"""Collect metrics using PyTorch (basic info)"""
# Memory info. Prefer the driver's free/total (device-scoped, reflects
# all processes) over this-process reservations; no global set_device.
vram_total = torch.cuda.get_device_properties(device_id).total_memory
try:
vram_free, vram_total = torch.cuda.mem_get_info(device_id)
except Exception:
vram_free = vram_total - torch.cuda.memory_reserved(device_id)
vram_used = vram_total - vram_free
return GPUMetrics(
device_id=device_id,
timestamp=timestamp,
vram_total=vram_total,
vram_used=vram_used,
vram_free=vram_free,
gpu_utilization=0.0, # Not available in PyTorch
memory_utilization=vram_used / vram_total if vram_total > 0 else 0.0
)
def _check_thresholds(self) -> None:
"""Check metrics against thresholds and generate alerts"""
with self._lock:
for device_id, metrics in self._current_metrics.items():
# VRAM utilization
if metrics.memory_utilization >= self.config.vram_critical_threshold:
self._create_alert(
device_id,
MetricType.VRAM_UTILIZATION,
AlertLevel.CRITICAL,
f"GPU {device_id} VRAM critically high: "
f"{metrics.memory_utilization*100:.1f}%",
metrics.memory_utilization,
self.config.vram_critical_threshold
)
elif metrics.memory_utilization >= self.config.vram_warning_threshold:
self._create_alert(
device_id,
MetricType.VRAM_UTILIZATION,
AlertLevel.WARNING,
f"GPU {device_id} VRAM usage high: "
f"{metrics.memory_utilization*100:.1f}%",
metrics.memory_utilization,
self.config.vram_warning_threshold
)
# GPU utilization
if (metrics.gpu_utilization >= self.config.gpu_utilization_warning
and metrics.gpu_utilization > 0):
self._create_alert(
device_id,
MetricType.GPU_UTILIZATION,
AlertLevel.WARNING,
f"GPU {device_id} utilization high: "
f"{metrics.gpu_utilization*100:.1f}%",
metrics.gpu_utilization,
self.config.gpu_utilization_warning
)
# Temperature
if metrics.temperature is not None:
if metrics.temperature >= self.config.temperature_critical:
self._create_alert(
device_id,
MetricType.TEMPERATURE,
AlertLevel.CRITICAL,
f"GPU {device_id} temperature critically high: "
f"{metrics.temperature:.1f}°C",
metrics.temperature,
self.config.temperature_critical
)
elif metrics.temperature >= self.config.temperature_warning:
self._create_alert(
device_id,
MetricType.TEMPERATURE,
AlertLevel.WARNING,
f"GPU {device_id} temperature high: "
f"{metrics.temperature:.1f}°C",
metrics.temperature,
self.config.temperature_warning
)
# Power usage
if (metrics.power_usage is not None
and metrics.power_limit is not None
and metrics.power_limit > 0):
power_ratio = metrics.power_usage / metrics.power_limit
if power_ratio >= self.config.power_warning_threshold:
self._create_alert(
device_id,
MetricType.POWER_USAGE,
AlertLevel.WARNING,
f"GPU {device_id} power usage high: "
f"{metrics.power_usage:.1f}W / {metrics.power_limit:.1f}W",
power_ratio,
self.config.power_warning_threshold
)
def _create_alert(
self,
device_id: int,
metric_type: MetricType,
level: AlertLevel,
message: str,
value: float,
threshold: float
) -> None:
"""Create and process an alert"""
alert = Alert(
device_id=device_id,
metric_type=metric_type,
level=level,
message=message,
value=value,
threshold=threshold
)
# Add to alerts list
self._alerts.append(alert)
if len(self._alerts) > self._max_alerts:
self._alerts.pop(0)
# Log alert
if level == AlertLevel.CRITICAL:
logger.critical(message)
elif level == AlertLevel.WARNING:
logger.warning(message)
else:
logger.info(message)
# Call callbacks
for callback in self._alert_callbacks:
try:
callback(alert)
except Exception as e:
logger.error(f"Error in alert callback: {e}")
def _log_metrics(self) -> None:
"""Log current metrics"""
with self._lock:
if not self._current_metrics:
return
metrics_summary = []
for device_id, metrics in sorted(self._current_metrics.items()):
summary = (
f"GPU {device_id}: "
f"VRAM {metrics.vram_used_gb:.2f}/{metrics.vram_total_gb:.2f}GB "
f"({metrics.memory_utilization*100:.1f}%), "
f"Util {metrics.gpu_utilization*100:.1f}%"
)
if metrics.temperature is not None:
summary += f", Temp {metrics.temperature:.1f}°C"
if metrics.power_usage is not None:
summary += f", Power {metrics.power_usage:.1f}W"
metrics_summary.append(summary)
logger.info("GPU Metrics: " + " | ".join(metrics_summary))
def get_metrics(
self,
device_id: int | None = None
) -> dict[int, GPUMetrics]:
"""
Get current metrics.
Args:
device_id: Specific device ID, or None for all devices
Returns:
Dictionary mapping device IDs to GPUMetrics
"""
with self._lock:
if device_id is not None:
if device_id in self._current_metrics:
return {device_id: self._current_metrics[device_id]}
else:
return {}
return self._current_metrics.copy()
def get_metrics_history(
self,
device_id: int,
limit: int | None = None
) -> list[GPUMetrics]:
"""
Get metrics history for a device.
Args:
device_id: GPU device ID
limit: Maximum number of metrics to return (most recent)
Returns:
List of GPUMetrics ordered by timestamp (oldest to newest)
"""
with self._lock:
if device_id not in self._metrics_history:
return []
history = self._metrics_history[device_id]
if limit:
return history[-limit:]
return history.copy()
def get_alerts(
self,
device_id: int | None = None,
level: AlertLevel | None = None,
limit: int | None = None
) -> list[Alert]:
"""
Get alerts.
Args:
device_id: Filter by device ID
level: Filter by alert level
limit: Maximum number of alerts to return (most recent)
Returns:
List of Alert objects
"""
with self._lock:
alerts = self._alerts.copy()
# Filter by device
if device_id is not None:
alerts = [a for a in alerts if a.device_id == device_id]
# Filter by level
if level is not None:
alerts = [a for a in alerts if a.level == level]
# Apply limit
if limit:
alerts = alerts[-limit:]
return alerts
def clear_alerts(self) -> None:
"""Clear all alerts."""
with self._lock:
self._alerts.clear()
logger.info("Alerts cleared")
def add_alert_callback(self, callback: Callable[[Alert], None]) -> None:
"""
Add callback for alerts.
Args:
callback: Function that takes Alert object as parameter
"""
self._alert_callbacks.append(callback)
logger.info(f"Added alert callback: {callback.__name__}")
def remove_alert_callback(self, callback: Callable[[Alert], None]) -> None:
"""
Remove alert callback.
Args:
callback: Callback function to remove
"""
if callback in self._alert_callbacks:
self._alert_callbacks.remove(callback)
logger.info(f"Removed alert callback: {callback.__name__}")
def get_summary(self) -> dict[str, Any]:
"""
Get summary of current GPU status.
Returns:
Dictionary with summary information
"""
with self._lock:
summary = {
'num_devices': len(self._device_handles),
'monitoring_active': self._running,
'total_alerts': len(self._alerts),
'devices': {}
}
for device_id, metrics in self._current_metrics.items():
summary['devices'][device_id] = {
'vram_used_gb': metrics.vram_used_gb,
'vram_total_gb': metrics.vram_total_gb,
'memory_utilization': metrics.memory_utilization,
'gpu_utilization': metrics.gpu_utilization,
'temperature': metrics.temperature,
'power_usage': metrics.power_usage,
'num_processes': len(metrics.processes)
}
return summary
def __enter__(self) -> GPUMonitor:
"""Context manager entry"""
self.start()
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> None:
"""Context manager exit"""
self.stop()
def __del__(self) -> None:
"""Cleanup on deletion"""
if self._running:
self.stop()
if self._nvml_initialized:
try:
pynvml.nvmlShutdown()
except Exception:
logger.debug("Failed to shutdown NVML during cleanup")
def __repr__(self) -> str:
"""String representation"""
return (
f"GPUMonitor(devices={len(self._device_handles)}, "
f"running={self._running}, "
f"alerts={len(self._alerts)})"
)

Xet Storage Details

Size:
23.9 kB
·
Xet hash:
c5c84982eba1885d20b1da5de2ff2d95cbe2ca4742f90226b29fe14bc5470645

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.