Virtual-LPU / lpu /memory /memory_manager.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame
6.43 kB
"""LPU Memory Manager: per-GPU VRAM accounting, OOM prevention, cache policy.
Non-negotiable rules encoded here (spec section 7):
- DEFAULT POLICY IS GPU_FIRST. Work is NOT offloaded to system RAM because
memory management was lazy; offloading only when explicitly configured.
- The two 16 GB P100s are NEVER treated as one 32 GB pool. Every accounting
structure is keyed by physical device index.
- empty_cache() is never called indiscriminately during training. The only
sanctioned call sites: (a) after LPU's own VRAM probing (config-gated),
(b) controlled OOM recovery targeting LPU-owned cache entries first.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional
from ..compat import require_torch
from .tensor_cache import TensorCache
from .gpu_allocator import estimate_batch_memory, find_max_batch_size
@dataclass
class DeviceMemoryState:
index: int
total_vram_mb: Optional[float] = None
free_vram_mb: Optional[float] = None
torch_allocated_mb: Optional[float] = None
torch_reserved_mb: Optional[float] = None
peak_allocated_mb: Optional[float] = None
last_refresh: float = 0.0
oom_events: int = 0
empty_cache_calls: int = 0
class MemoryManager:
"""Authoritative, per-device memory accounting + protection layer."""
def __init__(self, config=None, log_dir=None):
self.config = config
self.headroom_mb = (
config.get("memory.vram_headroom_mb", 1024.0) if config else 1024.0
)
self.policy = (config.get("memory.policy", "gpu_first") if config else "gpu_first") or "gpu_first"
self.devices: Dict[int, DeviceMemoryState] = {}
self.cache = TensorCache(owner="lpu")
self._lock = threading.RLock()
self._probe_empty_cache_allowed = (
config.get("memory.probe_empty_cache", True) if config else True
)
self._oom_recoveries: List[dict] = []
# -- discovery -----------------------------------------------------------
def refresh(self, device_index: Optional[int] = None) -> Dict[int, DeviceMemoryState]:
"""Query true per-device memory state (torch allocator + mem_get_info)."""
torch = require_torch()
with self._lock:
if not torch.cuda.is_available():
return {}
indices = (
[device_index] if device_index is not None else range(torch.cuda.device_count())
)
for i in indices:
st = self.devices.setdefault(i, DeviceMemoryState(index=i))
try:
free_b, total_b = torch.cuda.mem_get_info(i)
st.free_vram_mb = free_b / (1024 * 1024)
st.total_vram_mb = total_b / (1024 * 1024)
st.torch_allocated_mb = torch.cuda.memory_allocated(i) / (1024 * 1024)
st.torch_reserved_mb = torch.cuda.memory_reserved(i) / (1024 * 1024)
st.peak_allocated_mb = torch.cuda.max_memory_allocated(i) / (1024 * 1024)
st.last_refresh = time.time()
except Exception: # noqa: BLE001
continue
return dict(self.devices)
def usable_free_mb(self, device_index: int) -> Optional[float]:
"""Free VRAM minus the configured safety headroom (never negative)."""
st = self.refresh(device_index).get(device_index)
if st is None or st.free_vram_mb is None:
return None
return max(0.0, st.free_vram_mb - self.headroom_mb)
# -- pressure / OOM ------------------------------------------------------
def is_under_pressure(self, device_index: int, threshold_pct: float = 90.0) -> Optional[bool]:
"""True when this device's VRAM usage exceeds threshold_pct."""
st = self.refresh(device_index).get(device_index)
if st is None or st.free_vram_mb is None or st.total_vram_mb in (None, 0):
return None
used_pct = 100.0 * (1.0 - st.free_vram_mb / st.total_vram_mb)
return used_pct >= threshold_pct
def record_oom(self, device_index: int, detail: dict) -> None:
with self._lock:
st = self.devices.setdefault(
device_index, DeviceMemoryState(index=device_index)
)
st.oom_events += 1
self._oom_recoveries.append({"device": device_index, "ts": time.time(), **detail})
def controlled_cleanup(self, device_index: int, reason: str = "oom") -> None:
"""OOM recovery with explicit scope: LPU-owned cache entries first.
torch.cuda.empty_cache() is called at most once per recovery event and
only on the affected device - never blanket-called during training.
"""
torch = require_torch()
with self._lock:
evicted = self.cache.evict_device(f"cuda:{device_index}")
try:
torch.cuda.empty_cache()
st = self.devices.setdefault(
device_index, DeviceMemoryState(index=device_index)
)
st.empty_cache_calls += 1
except Exception: # noqa: BLE001
pass
return {"evicted_lpu_tensors": evicted, "reason": reason}
# -- reporting ---------------------------------------------------------------
def snapshot(self) -> dict:
self.refresh()
with self._lock:
return {
"policy": self.policy,
"headroom_mb": self.headroom_mb,
"devices": {
i: {
"total_vram_mb": st.total_vram_mb,
"free_vram_mb": st.free_vram_mb,
"torch_allocated_mb": st.torch_allocated_mb,
"torch_reserved_mb": st.torch_reserved_mb,
"peak_allocated_mb": st.peak_allocated_mb,
"oom_events": st.oom_events,
"empty_cache_calls": st.empty_cache_calls,
}
for i, st in self.devices.items()
},
"oom_recoveries": list(self._oom_recoveries[-10:]),
"cache": self.cache.stats(),
}
@property
def oom_recoveries(self) -> List[dict]:
with self._lock:
return list(self._oom_recoveries)