Spaces:
Running
Running
File size: 6,828 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 | """Empirical batch-size estimation and VRAM probing.
Technique (real, measured - not a formula fantasy):
clone the model, run forward+backward on synthetic batches of increasing
size, read torch.cuda.max_memory_allocated per device, catch real CUDA OOM,
halve, retry. Estimates derived from measurements are labeled
measured=False (dry-run heuristics) vs measured=True (actual probe runs).
The probe:
- deep-copies the model so real weights are untouched
- uses a copied throwaway optimizer? NO - optimizer states allocate lazily
after step(); we add a +8 bytes/param allowance (Adam moments) instead,
documented as an estimate.
- restores RNG state afterwards, and performs ONE gated empty_cache() after
its own probe allocations (memory.probe_empty_cache=true). Rationale: the
probe's cached blocks are LPU-owned garbage; without releasing them the
real training start would inherit a fragmented allocator. This is the
documented, config-gated exception to "no empty_cache during training".
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import List, Optional, Tuple
from ..compat import require_torch
from .tensor_cache import _safe_deepcopy
@dataclass
class BatchSizeEstimate:
batch_size: int
measured: bool
per_image_vram_mb: Optional[float]
headroom_mb: float
method: str # "probe" | "param_heuristic"
note: str
def _synthetic_like(example_input, batch: int):
torch = require_torch()
if isinstance(example_input, (tuple, list)):
return tuple(_synthetic_like(e, batch) for e in example_input)
if isinstance(example_input, dict):
return {k: _synthetic_like(v, batch) for k, v in example_input.items()}
if hasattr(example_input, "new_full"):
shape = (batch,) + tuple(example_input.shape[1:])
return example_input.new_full(shape, 0.0).normal_()
return example_input
def _to(t, device):
if hasattr(t, "to"):
return t.to(device, non_blocking=True)
if isinstance(t, (tuple, list)):
return type(t)(_to(x, device) for x in t)
if isinstance(t, dict):
return {k: _to(v, device) for k, v in t.items()}
return t
def _param_bytes(module) -> int:
return sum(p.numel() * p.element_size() for p in module.parameters())
def estimate_batch_memory(
model,
example_input,
device_index: int = 0,
max_batch: int = 256,
headroom_mb: float = 1024.0,
allow_probe: bool = True,
allow_empty_cache: bool = True,
) -> BatchSizeEstimate:
"""Return the largest power-of-two batch that fits within VRAM limits."""
torch = require_torch()
if not torch.cuda.is_available():
raise RuntimeError("estimate_batch_memory requires CUDA (real probing only)")
device = f"cuda:{device_index}"
free_b, _total = torch.cuda.mem_get_info(device_index)
usable_mb = free_b / (1024 * 1024) - headroom_mb
if usable_mb <= 0:
return BatchSizeEstimate(
1, False, None, headroom_mb, "param_heuristic",
"free VRAM below headroom; returned minimum batch 1",
)
if not allow_probe:
# dry heuristic: params*4 copies (weights+grads+2 Adam moments)
# + crude activation estimate of 64x input pixels x 4B
est = _param_bytes(model) * 4 / (1024 * 1024)
act = 1.0 # minimal
bs = max(1, int(usable_mb / max(1.0, est / 8 + act)))
return BatchSizeEstimate(
min(bs, max_batch), False, None, headroom_mb, "param_heuristic",
"formula-only estimate (probe disabled); treat as an ESTIMATE",
)
# empirical probe
was_training = model.training
probe_model = _safe_deepcopy(model).to(device)
probe_model.train()
# freeze BN/dropout determinism is not needed for MEMORY sizing; skip.
best, per_img = 1, None
batch = 1
gen_state = None
cuda_gen_state = None
try:
gen_state = torch.get_rng_state()
try:
cuda_gen_state = torch.cuda.get_rng_state(device_index) # probe consumes CUDA RNG too
except Exception: # noqa: BLE001
pass
except Exception: # noqa: BLE001
pass
try:
while batch <= max_batch:
torch.cuda.reset_peak_memory_stats(device_index)
x = None
try:
x = _to(_synthetic_like(example_input, batch), device)
y = probe_model(x)
if not isinstance(y, torch.Tensor):
y = y[0]
loss = y.float().sum()
loss.backward()
peak = torch.cuda.max_memory_allocated(device_index) / (1024 * 1024)
# include Adam moment allowance: +8B/param
adam_mb = _param_bytes(probe_model) * 2 / (1024 * 1024)
total = peak + adam_mb
if total > usable_mb:
break
best, per_img = batch, max(1e-6, total / batch)
batch *= 2
except torch.cuda.OutOfMemoryError:
break
except RuntimeError as exc:
if "out of memory" in str(exc).lower():
break
raise
finally:
# drop references so their cached blocks can be released below
for p in probe_model.parameters():
if p.grad is not None:
p.grad = None
y = None
loss = None
x = None
finally:
del probe_model
try:
torch.set_rng_state(gen_state)
if cuda_gen_state is not None:
torch.cuda.set_rng_state(cuda_gen_state, device_index)
except Exception: # noqa: BLE001
pass
if was_training:
model.train()
else:
model.eval()
# THE sanctioned empty_cache after LPU's own probing (config-gated):
# memory.probe_empty_cache=false disables it everywhere.
if allow_empty_cache:
torch.cuda.empty_cache()
return BatchSizeEstimate(
batch_size=best,
measured=True,
per_image_vram_mb=per_img,
headroom_mb=headroom_mb,
method="probe",
note="measured by empirical forward+backward probing on a model clone",
)
def find_max_batch_size(
model,
example_input,
requested_batch: int,
device_index: int = 0,
headroom_mb: float = 1024.0,
) -> Tuple[int, int]:
"""(safe_batch, accum_steps) such that safe_batch*accum >= requested_batch."""
est = estimate_batch_memory(
model, example_input, device_index=device_index, headroom_mb=headroom_mb
)
safe = min(requested_batch, est.batch_size)
accum = max(1, (requested_batch + safe - 1) // safe)
return safe, accum
|