File size: 6,828 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
"""Empirical batch-size estimation and VRAM probing.

Technique (real, measured - not a formula fantasy):
  clone the model, run forward+backward on synthetic batches of increasing
  size, read torch.cuda.max_memory_allocated per device, catch real CUDA OOM,
  halve, retry. Estimates derived from measurements are labeled
  measured=False (dry-run heuristics) vs measured=True (actual probe runs).

The probe:
- deep-copies the model so real weights are untouched
- uses a copied throwaway optimizer? NO - optimizer states allocate lazily
  after step(); we add a +8 bytes/param allowance (Adam moments) instead,
  documented as an estimate.
- restores RNG state afterwards, and performs ONE gated empty_cache() after
  its own probe allocations (memory.probe_empty_cache=true). Rationale: the
  probe's cached blocks are LPU-owned garbage; without releasing them the
  real training start would inherit a fragmented allocator. This is the
  documented, config-gated exception to "no empty_cache during training".
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import List, Optional, Tuple

from ..compat import require_torch
from .tensor_cache import _safe_deepcopy


@dataclass
class BatchSizeEstimate:
    batch_size: int
    measured: bool
    per_image_vram_mb: Optional[float]
    headroom_mb: float
    method: str  # "probe" | "param_heuristic"
    note: str


def _synthetic_like(example_input, batch: int):
    torch = require_torch()
    if isinstance(example_input, (tuple, list)):
        return tuple(_synthetic_like(e, batch) for e in example_input)
    if isinstance(example_input, dict):
        return {k: _synthetic_like(v, batch) for k, v in example_input.items()}
    if hasattr(example_input, "new_full"):
        shape = (batch,) + tuple(example_input.shape[1:])
        return example_input.new_full(shape, 0.0).normal_()
    return example_input


def _to(t, device):
    if hasattr(t, "to"):
        return t.to(device, non_blocking=True)
    if isinstance(t, (tuple, list)):
        return type(t)(_to(x, device) for x in t)
    if isinstance(t, dict):
        return {k: _to(v, device) for k, v in t.items()}
    return t


def _param_bytes(module) -> int:
    return sum(p.numel() * p.element_size() for p in module.parameters())


def estimate_batch_memory(
    model,
    example_input,
    device_index: int = 0,
    max_batch: int = 256,
    headroom_mb: float = 1024.0,
    allow_probe: bool = True,
    allow_empty_cache: bool = True,
) -> BatchSizeEstimate:
    """Return the largest power-of-two batch that fits within VRAM limits."""
    torch = require_torch()
    if not torch.cuda.is_available():
        raise RuntimeError("estimate_batch_memory requires CUDA (real probing only)")
    device = f"cuda:{device_index}"
    free_b, _total = torch.cuda.mem_get_info(device_index)
    usable_mb = free_b / (1024 * 1024) - headroom_mb
    if usable_mb <= 0:
        return BatchSizeEstimate(
            1, False, None, headroom_mb, "param_heuristic",
            "free VRAM below headroom; returned minimum batch 1",
        )

    if not allow_probe:
        # dry heuristic: params*4 copies (weights+grads+2 Adam moments)
        # + crude activation estimate of 64x input pixels x 4B
        est = _param_bytes(model) * 4 / (1024 * 1024)
        act = 1.0  # minimal
        bs = max(1, int(usable_mb / max(1.0, est / 8 + act)))
        return BatchSizeEstimate(
            min(bs, max_batch), False, None, headroom_mb, "param_heuristic",
            "formula-only estimate (probe disabled); treat as an ESTIMATE",
        )

    # empirical probe
    was_training = model.training
    probe_model = _safe_deepcopy(model).to(device)
    probe_model.train()
    # freeze BN/dropout determinism is not needed for MEMORY sizing; skip.
    best, per_img = 1, None
    batch = 1
    gen_state = None
    cuda_gen_state = None
    try:
        gen_state = torch.get_rng_state()
        try:
            cuda_gen_state = torch.cuda.get_rng_state(device_index)  # probe consumes CUDA RNG too
        except Exception:  # noqa: BLE001
            pass
    except Exception:  # noqa: BLE001
        pass
    try:
        while batch <= max_batch:
            torch.cuda.reset_peak_memory_stats(device_index)
            x = None
            try:
                x = _to(_synthetic_like(example_input, batch), device)
                y = probe_model(x)
                if not isinstance(y, torch.Tensor):
                    y = y[0]
                loss = y.float().sum()
                loss.backward()
                peak = torch.cuda.max_memory_allocated(device_index) / (1024 * 1024)
                # include Adam moment allowance: +8B/param
                adam_mb = _param_bytes(probe_model) * 2 / (1024 * 1024)
                total = peak + adam_mb
                if total > usable_mb:
                    break
                best, per_img = batch, max(1e-6, total / batch)
                batch *= 2
            except torch.cuda.OutOfMemoryError:
                break
            except RuntimeError as exc:
                if "out of memory" in str(exc).lower():
                    break
                raise
            finally:
                # drop references so their cached blocks can be released below
                for p in probe_model.parameters():
                    if p.grad is not None:
                        p.grad = None
                y = None
                loss = None
                x = None
    finally:
        del probe_model
        try:
            torch.set_rng_state(gen_state)
            if cuda_gen_state is not None:
                torch.cuda.set_rng_state(cuda_gen_state, device_index)
        except Exception:  # noqa: BLE001
            pass
        if was_training:
            model.train()
        else:
            model.eval()
        # THE sanctioned empty_cache after LPU's own probing (config-gated):
        # memory.probe_empty_cache=false disables it everywhere.
        if allow_empty_cache:
            torch.cuda.empty_cache()
    return BatchSizeEstimate(
        batch_size=best,
        measured=True,
        per_image_vram_mb=per_img,
        headroom_mb=headroom_mb,
        method="probe",
        note="measured by empirical forward+backward probing on a model clone",
    )


def find_max_batch_size(
    model,
    example_input,
    requested_batch: int,
    device_index: int = 0,
    headroom_mb: float = 1024.0,
) -> Tuple[int, int]:
    """(safe_batch, accum_steps) such that safe_batch*accum >= requested_batch."""
    est = estimate_batch_memory(
        model, example_input, device_index=device_index, headroom_mb=headroom_mb
    )
    safe = min(requested_batch, est.batch_size)
    accum = max(1, (requested_batch + safe - 1) // safe)
    return safe, accum