Spaces:
Running
Running
File size: 6,430 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """LPU Memory Manager: per-GPU VRAM accounting, OOM prevention, cache policy.
Non-negotiable rules encoded here (spec section 7):
- DEFAULT POLICY IS GPU_FIRST. Work is NOT offloaded to system RAM because
memory management was lazy; offloading only when explicitly configured.
- The two 16 GB P100s are NEVER treated as one 32 GB pool. Every accounting
structure is keyed by physical device index.
- empty_cache() is never called indiscriminately during training. The only
sanctioned call sites: (a) after LPU's own VRAM probing (config-gated),
(b) controlled OOM recovery targeting LPU-owned cache entries first.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional
from ..compat import require_torch
from .tensor_cache import TensorCache
from .gpu_allocator import estimate_batch_memory, find_max_batch_size
@dataclass
class DeviceMemoryState:
index: int
total_vram_mb: Optional[float] = None
free_vram_mb: Optional[float] = None
torch_allocated_mb: Optional[float] = None
torch_reserved_mb: Optional[float] = None
peak_allocated_mb: Optional[float] = None
last_refresh: float = 0.0
oom_events: int = 0
empty_cache_calls: int = 0
class MemoryManager:
"""Authoritative, per-device memory accounting + protection layer."""
def __init__(self, config=None, log_dir=None):
self.config = config
self.headroom_mb = (
config.get("memory.vram_headroom_mb", 1024.0) if config else 1024.0
)
self.policy = (config.get("memory.policy", "gpu_first") if config else "gpu_first") or "gpu_first"
self.devices: Dict[int, DeviceMemoryState] = {}
self.cache = TensorCache(owner="lpu")
self._lock = threading.RLock()
self._probe_empty_cache_allowed = (
config.get("memory.probe_empty_cache", True) if config else True
)
self._oom_recoveries: List[dict] = []
# -- discovery -----------------------------------------------------------
def refresh(self, device_index: Optional[int] = None) -> Dict[int, DeviceMemoryState]:
"""Query true per-device memory state (torch allocator + mem_get_info)."""
torch = require_torch()
with self._lock:
if not torch.cuda.is_available():
return {}
indices = (
[device_index] if device_index is not None else range(torch.cuda.device_count())
)
for i in indices:
st = self.devices.setdefault(i, DeviceMemoryState(index=i))
try:
free_b, total_b = torch.cuda.mem_get_info(i)
st.free_vram_mb = free_b / (1024 * 1024)
st.total_vram_mb = total_b / (1024 * 1024)
st.torch_allocated_mb = torch.cuda.memory_allocated(i) / (1024 * 1024)
st.torch_reserved_mb = torch.cuda.memory_reserved(i) / (1024 * 1024)
st.peak_allocated_mb = torch.cuda.max_memory_allocated(i) / (1024 * 1024)
st.last_refresh = time.time()
except Exception: # noqa: BLE001
continue
return dict(self.devices)
def usable_free_mb(self, device_index: int) -> Optional[float]:
"""Free VRAM minus the configured safety headroom (never negative)."""
st = self.refresh(device_index).get(device_index)
if st is None or st.free_vram_mb is None:
return None
return max(0.0, st.free_vram_mb - self.headroom_mb)
# -- pressure / OOM ------------------------------------------------------
def is_under_pressure(self, device_index: int, threshold_pct: float = 90.0) -> Optional[bool]:
"""True when this device's VRAM usage exceeds threshold_pct."""
st = self.refresh(device_index).get(device_index)
if st is None or st.free_vram_mb is None or st.total_vram_mb in (None, 0):
return None
used_pct = 100.0 * (1.0 - st.free_vram_mb / st.total_vram_mb)
return used_pct >= threshold_pct
def record_oom(self, device_index: int, detail: dict) -> None:
with self._lock:
st = self.devices.setdefault(
device_index, DeviceMemoryState(index=device_index)
)
st.oom_events += 1
self._oom_recoveries.append({"device": device_index, "ts": time.time(), **detail})
def controlled_cleanup(self, device_index: int, reason: str = "oom") -> None:
"""OOM recovery with explicit scope: LPU-owned cache entries first.
torch.cuda.empty_cache() is called at most once per recovery event and
only on the affected device - never blanket-called during training.
"""
torch = require_torch()
with self._lock:
evicted = self.cache.evict_device(f"cuda:{device_index}")
try:
torch.cuda.empty_cache()
st = self.devices.setdefault(
device_index, DeviceMemoryState(index=device_index)
)
st.empty_cache_calls += 1
except Exception: # noqa: BLE001
pass
return {"evicted_lpu_tensors": evicted, "reason": reason}
# -- reporting ---------------------------------------------------------------
def snapshot(self) -> dict:
self.refresh()
with self._lock:
return {
"policy": self.policy,
"headroom_mb": self.headroom_mb,
"devices": {
i: {
"total_vram_mb": st.total_vram_mb,
"free_vram_mb": st.free_vram_mb,
"torch_allocated_mb": st.torch_allocated_mb,
"torch_reserved_mb": st.torch_reserved_mb,
"peak_allocated_mb": st.peak_allocated_mb,
"oom_events": st.oom_events,
"empty_cache_calls": st.empty_cache_calls,
}
for i, st in self.devices.items()
},
"oom_recoveries": list(self._oom_recoveries[-10:]),
"cache": self.cache.stats(),
}
@property
def oom_recoveries(self) -> List[dict]:
with self._lock:
return list(self._oom_recoveries)
|