Spaces:
Running
Running
Download lpu/memory/memory_manager.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 6.43 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/memory_manager.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/memory/memory_manager.py
-
curl -L -o memory_manager.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/memory_manager.py
6.43 kB
| """LPU Memory Manager: per-GPU VRAM accounting, OOM prevention, cache policy. | |
| Non-negotiable rules encoded here (spec section 7): | |
| - DEFAULT POLICY IS GPU_FIRST. Work is NOT offloaded to system RAM because | |
| memory management was lazy; offloading only when explicitly configured. | |
| - The two 16 GB P100s are NEVER treated as one 32 GB pool. Every accounting | |
| structure is keyed by physical device index. | |
| - empty_cache() is never called indiscriminately during training. The only | |
| sanctioned call sites: (a) after LPU's own VRAM probing (config-gated), | |
| (b) controlled OOM recovery targeting LPU-owned cache entries first. | |
| """ | |
| from __future__ import annotations | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import Dict, List, Optional | |
| from ..compat import require_torch | |
| from .tensor_cache import TensorCache | |
| from .gpu_allocator import estimate_batch_memory, find_max_batch_size | |
| class DeviceMemoryState: | |
| index: int | |
| total_vram_mb: Optional[float] = None | |
| free_vram_mb: Optional[float] = None | |
| torch_allocated_mb: Optional[float] = None | |
| torch_reserved_mb: Optional[float] = None | |
| peak_allocated_mb: Optional[float] = None | |
| last_refresh: float = 0.0 | |
| oom_events: int = 0 | |
| empty_cache_calls: int = 0 | |
| class MemoryManager: | |
| """Authoritative, per-device memory accounting + protection layer.""" | |
| def __init__(self, config=None, log_dir=None): | |
| self.config = config | |
| self.headroom_mb = ( | |
| config.get("memory.vram_headroom_mb", 1024.0) if config else 1024.0 | |
| ) | |
| self.policy = (config.get("memory.policy", "gpu_first") if config else "gpu_first") or "gpu_first" | |
| self.devices: Dict[int, DeviceMemoryState] = {} | |
| self.cache = TensorCache(owner="lpu") | |
| self._lock = threading.RLock() | |
| self._probe_empty_cache_allowed = ( | |
| config.get("memory.probe_empty_cache", True) if config else True | |
| ) | |
| self._oom_recoveries: List[dict] = [] | |
| # -- discovery ----------------------------------------------------------- | |
| def refresh(self, device_index: Optional[int] = None) -> Dict[int, DeviceMemoryState]: | |
| """Query true per-device memory state (torch allocator + mem_get_info).""" | |
| torch = require_torch() | |
| with self._lock: | |
| if not torch.cuda.is_available(): | |
| return {} | |
| indices = ( | |
| [device_index] if device_index is not None else range(torch.cuda.device_count()) | |
| ) | |
| for i in indices: | |
| st = self.devices.setdefault(i, DeviceMemoryState(index=i)) | |
| try: | |
| free_b, total_b = torch.cuda.mem_get_info(i) | |
| st.free_vram_mb = free_b / (1024 * 1024) | |
| st.total_vram_mb = total_b / (1024 * 1024) | |
| st.torch_allocated_mb = torch.cuda.memory_allocated(i) / (1024 * 1024) | |
| st.torch_reserved_mb = torch.cuda.memory_reserved(i) / (1024 * 1024) | |
| st.peak_allocated_mb = torch.cuda.max_memory_allocated(i) / (1024 * 1024) | |
| st.last_refresh = time.time() | |
| except Exception: # noqa: BLE001 | |
| continue | |
| return dict(self.devices) | |
| def usable_free_mb(self, device_index: int) -> Optional[float]: | |
| """Free VRAM minus the configured safety headroom (never negative).""" | |
| st = self.refresh(device_index).get(device_index) | |
| if st is None or st.free_vram_mb is None: | |
| return None | |
| return max(0.0, st.free_vram_mb - self.headroom_mb) | |
| # -- pressure / OOM ------------------------------------------------------ | |
| def is_under_pressure(self, device_index: int, threshold_pct: float = 90.0) -> Optional[bool]: | |
| """True when this device's VRAM usage exceeds threshold_pct.""" | |
| st = self.refresh(device_index).get(device_index) | |
| if st is None or st.free_vram_mb is None or st.total_vram_mb in (None, 0): | |
| return None | |
| used_pct = 100.0 * (1.0 - st.free_vram_mb / st.total_vram_mb) | |
| return used_pct >= threshold_pct | |
| def record_oom(self, device_index: int, detail: dict) -> None: | |
| with self._lock: | |
| st = self.devices.setdefault( | |
| device_index, DeviceMemoryState(index=device_index) | |
| ) | |
| st.oom_events += 1 | |
| self._oom_recoveries.append({"device": device_index, "ts": time.time(), **detail}) | |
| def controlled_cleanup(self, device_index: int, reason: str = "oom") -> None: | |
| """OOM recovery with explicit scope: LPU-owned cache entries first. | |
| torch.cuda.empty_cache() is called at most once per recovery event and | |
| only on the affected device - never blanket-called during training. | |
| """ | |
| torch = require_torch() | |
| with self._lock: | |
| evicted = self.cache.evict_device(f"cuda:{device_index}") | |
| try: | |
| torch.cuda.empty_cache() | |
| st = self.devices.setdefault( | |
| device_index, DeviceMemoryState(index=device_index) | |
| ) | |
| st.empty_cache_calls += 1 | |
| except Exception: # noqa: BLE001 | |
| pass | |
| return {"evicted_lpu_tensors": evicted, "reason": reason} | |
| # -- reporting --------------------------------------------------------------- | |
| def snapshot(self) -> dict: | |
| self.refresh() | |
| with self._lock: | |
| return { | |
| "policy": self.policy, | |
| "headroom_mb": self.headroom_mb, | |
| "devices": { | |
| i: { | |
| "total_vram_mb": st.total_vram_mb, | |
| "free_vram_mb": st.free_vram_mb, | |
| "torch_allocated_mb": st.torch_allocated_mb, | |
| "torch_reserved_mb": st.torch_reserved_mb, | |
| "peak_allocated_mb": st.peak_allocated_mb, | |
| "oom_events": st.oom_events, | |
| "empty_cache_calls": st.empty_cache_calls, | |
| } | |
| for i, st in self.devices.items() | |
| }, | |
| "oom_recoveries": list(self._oom_recoveries[-10:]), | |
| "cache": self.cache.stats(), | |
| } | |
| def oom_recoveries(self) -> List[dict]: | |
| with self._lock: | |
| return list(self._oom_recoveries) | |