"""LPU Memory Manager: per-GPU VRAM accounting, OOM prevention, cache policy. Non-negotiable rules encoded here (spec section 7): - DEFAULT POLICY IS GPU_FIRST. Work is NOT offloaded to system RAM because memory management was lazy; offloading only when explicitly configured. - The two 16 GB P100s are NEVER treated as one 32 GB pool. Every accounting structure is keyed by physical device index. - empty_cache() is never called indiscriminately during training. The only sanctioned call sites: (a) after LPU's own VRAM probing (config-gated), (b) controlled OOM recovery targeting LPU-owned cache entries first. """ from __future__ import annotations import threading import time from dataclasses import dataclass, field from typing import Dict, List, Optional from ..compat import require_torch from .tensor_cache import TensorCache from .gpu_allocator import estimate_batch_memory, find_max_batch_size @dataclass class DeviceMemoryState: index: int total_vram_mb: Optional[float] = None free_vram_mb: Optional[float] = None torch_allocated_mb: Optional[float] = None torch_reserved_mb: Optional[float] = None peak_allocated_mb: Optional[float] = None last_refresh: float = 0.0 oom_events: int = 0 empty_cache_calls: int = 0 class MemoryManager: """Authoritative, per-device memory accounting + protection layer.""" def __init__(self, config=None, log_dir=None): self.config = config self.headroom_mb = ( config.get("memory.vram_headroom_mb", 1024.0) if config else 1024.0 ) self.policy = (config.get("memory.policy", "gpu_first") if config else "gpu_first") or "gpu_first" self.devices: Dict[int, DeviceMemoryState] = {} self.cache = TensorCache(owner="lpu") self._lock = threading.RLock() self._probe_empty_cache_allowed = ( config.get("memory.probe_empty_cache", True) if config else True ) self._oom_recoveries: List[dict] = [] # -- discovery ----------------------------------------------------------- def refresh(self, device_index: Optional[int] = None) -> Dict[int, DeviceMemoryState]: """Query true per-device memory state (torch allocator + mem_get_info).""" torch = require_torch() with self._lock: if not torch.cuda.is_available(): return {} indices = ( [device_index] if device_index is not None else range(torch.cuda.device_count()) ) for i in indices: st = self.devices.setdefault(i, DeviceMemoryState(index=i)) try: free_b, total_b = torch.cuda.mem_get_info(i) st.free_vram_mb = free_b / (1024 * 1024) st.total_vram_mb = total_b / (1024 * 1024) st.torch_allocated_mb = torch.cuda.memory_allocated(i) / (1024 * 1024) st.torch_reserved_mb = torch.cuda.memory_reserved(i) / (1024 * 1024) st.peak_allocated_mb = torch.cuda.max_memory_allocated(i) / (1024 * 1024) st.last_refresh = time.time() except Exception: # noqa: BLE001 continue return dict(self.devices) def usable_free_mb(self, device_index: int) -> Optional[float]: """Free VRAM minus the configured safety headroom (never negative).""" st = self.refresh(device_index).get(device_index) if st is None or st.free_vram_mb is None: return None return max(0.0, st.free_vram_mb - self.headroom_mb) # -- pressure / OOM ------------------------------------------------------ def is_under_pressure(self, device_index: int, threshold_pct: float = 90.0) -> Optional[bool]: """True when this device's VRAM usage exceeds threshold_pct.""" st = self.refresh(device_index).get(device_index) if st is None or st.free_vram_mb is None or st.total_vram_mb in (None, 0): return None used_pct = 100.0 * (1.0 - st.free_vram_mb / st.total_vram_mb) return used_pct >= threshold_pct def record_oom(self, device_index: int, detail: dict) -> None: with self._lock: st = self.devices.setdefault( device_index, DeviceMemoryState(index=device_index) ) st.oom_events += 1 self._oom_recoveries.append({"device": device_index, "ts": time.time(), **detail}) def controlled_cleanup(self, device_index: int, reason: str = "oom") -> None: """OOM recovery with explicit scope: LPU-owned cache entries first. torch.cuda.empty_cache() is called at most once per recovery event and only on the affected device - never blanket-called during training. """ torch = require_torch() with self._lock: evicted = self.cache.evict_device(f"cuda:{device_index}") try: torch.cuda.empty_cache() st = self.devices.setdefault( device_index, DeviceMemoryState(index=device_index) ) st.empty_cache_calls += 1 except Exception: # noqa: BLE001 pass return {"evicted_lpu_tensors": evicted, "reason": reason} # -- reporting --------------------------------------------------------------- def snapshot(self) -> dict: self.refresh() with self._lock: return { "policy": self.policy, "headroom_mb": self.headroom_mb, "devices": { i: { "total_vram_mb": st.total_vram_mb, "free_vram_mb": st.free_vram_mb, "torch_allocated_mb": st.torch_allocated_mb, "torch_reserved_mb": st.torch_reserved_mb, "peak_allocated_mb": st.peak_allocated_mb, "oom_events": st.oom_events, "empty_cache_calls": st.empty_cache_calls, } for i, st in self.devices.items() }, "oom_recoveries": list(self._oom_recoveries[-10:]), "cache": self.cache.stats(), } @property def oom_recoveries(self) -> List[dict]: with self._lock: return list(self._oom_recoveries)