File size: 6,430 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""LPU Memory Manager: per-GPU VRAM accounting, OOM prevention, cache policy.

Non-negotiable rules encoded here (spec section 7):
- DEFAULT POLICY IS GPU_FIRST. Work is NOT offloaded to system RAM because
  memory management was lazy; offloading only when explicitly configured.
- The two 16 GB P100s are NEVER treated as one 32 GB pool. Every accounting
  structure is keyed by physical device index.
- empty_cache() is never called indiscriminately during training. The only
  sanctioned call sites: (a) after LPU's own VRAM probing (config-gated),
  (b) controlled OOM recovery targeting LPU-owned cache entries first.
"""

from __future__ import annotations

import threading
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional

from ..compat import require_torch
from .tensor_cache import TensorCache
from .gpu_allocator import estimate_batch_memory, find_max_batch_size


@dataclass
class DeviceMemoryState:
    index: int
    total_vram_mb: Optional[float] = None
    free_vram_mb: Optional[float] = None
    torch_allocated_mb: Optional[float] = None
    torch_reserved_mb: Optional[float] = None
    peak_allocated_mb: Optional[float] = None
    last_refresh: float = 0.0
    oom_events: int = 0
    empty_cache_calls: int = 0


class MemoryManager:
    """Authoritative, per-device memory accounting + protection layer."""

    def __init__(self, config=None, log_dir=None):
        self.config = config
        self.headroom_mb = (
            config.get("memory.vram_headroom_mb", 1024.0) if config else 1024.0
        )
        self.policy = (config.get("memory.policy", "gpu_first") if config else "gpu_first") or "gpu_first"
        self.devices: Dict[int, DeviceMemoryState] = {}
        self.cache = TensorCache(owner="lpu")
        self._lock = threading.RLock()
        self._probe_empty_cache_allowed = (
            config.get("memory.probe_empty_cache", True) if config else True
        )
        self._oom_recoveries: List[dict] = []

    # -- discovery -----------------------------------------------------------
    def refresh(self, device_index: Optional[int] = None) -> Dict[int, DeviceMemoryState]:
        """Query true per-device memory state (torch allocator + mem_get_info)."""
        torch = require_torch()
        with self._lock:
            if not torch.cuda.is_available():
                return {}
            indices = (
                [device_index] if device_index is not None else range(torch.cuda.device_count())
            )
            for i in indices:
                st = self.devices.setdefault(i, DeviceMemoryState(index=i))
                try:
                    free_b, total_b = torch.cuda.mem_get_info(i)
                    st.free_vram_mb = free_b / (1024 * 1024)
                    st.total_vram_mb = total_b / (1024 * 1024)
                    st.torch_allocated_mb = torch.cuda.memory_allocated(i) / (1024 * 1024)
                    st.torch_reserved_mb = torch.cuda.memory_reserved(i) / (1024 * 1024)
                    st.peak_allocated_mb = torch.cuda.max_memory_allocated(i) / (1024 * 1024)
                    st.last_refresh = time.time()
                except Exception:  # noqa: BLE001
                    continue
            return dict(self.devices)

    def usable_free_mb(self, device_index: int) -> Optional[float]:
        """Free VRAM minus the configured safety headroom (never negative)."""
        st = self.refresh(device_index).get(device_index)
        if st is None or st.free_vram_mb is None:
            return None
        return max(0.0, st.free_vram_mb - self.headroom_mb)

    # -- pressure / OOM ------------------------------------------------------
    def is_under_pressure(self, device_index: int, threshold_pct: float = 90.0) -> Optional[bool]:
        """True when this device's VRAM usage exceeds threshold_pct."""
        st = self.refresh(device_index).get(device_index)
        if st is None or st.free_vram_mb is None or st.total_vram_mb in (None, 0):
            return None
        used_pct = 100.0 * (1.0 - st.free_vram_mb / st.total_vram_mb)
        return used_pct >= threshold_pct

    def record_oom(self, device_index: int, detail: dict) -> None:
        with self._lock:
            st = self.devices.setdefault(
                device_index, DeviceMemoryState(index=device_index)
            )
            st.oom_events += 1
            self._oom_recoveries.append({"device": device_index, "ts": time.time(), **detail})

    def controlled_cleanup(self, device_index: int, reason: str = "oom") -> None:
        """OOM recovery with explicit scope: LPU-owned cache entries first.

        torch.cuda.empty_cache() is called at most once per recovery event and
        only on the affected device - never blanket-called during training.
        """
        torch = require_torch()
        with self._lock:
            evicted = self.cache.evict_device(f"cuda:{device_index}")
            try:
                torch.cuda.empty_cache()
                st = self.devices.setdefault(
                    device_index, DeviceMemoryState(index=device_index)
                )
                st.empty_cache_calls += 1
            except Exception:  # noqa: BLE001
                pass
            return {"evicted_lpu_tensors": evicted, "reason": reason}

    # -- reporting ---------------------------------------------------------------
    def snapshot(self) -> dict:
        self.refresh()
        with self._lock:
            return {
                "policy": self.policy,
                "headroom_mb": self.headroom_mb,
                "devices": {
                    i: {
                        "total_vram_mb": st.total_vram_mb,
                        "free_vram_mb": st.free_vram_mb,
                        "torch_allocated_mb": st.torch_allocated_mb,
                        "torch_reserved_mb": st.torch_reserved_mb,
                        "peak_allocated_mb": st.peak_allocated_mb,
                        "oom_events": st.oom_events,
                        "empty_cache_calls": st.empty_cache_calls,
                    }
                    for i, st in self.devices.items()
                },
                "oom_recoveries": list(self._oom_recoveries[-10:]),
                "cache": self.cache.stats(),
            }

    @property
    def oom_recoveries(self) -> List[dict]:
        with self._lock:
            return list(self._oom_recoveries)