Virtual-LPU / lpu /memory /gpu_allocator.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame
6.83 kB
"""Empirical batch-size estimation and VRAM probing.
Technique (real, measured - not a formula fantasy):
clone the model, run forward+backward on synthetic batches of increasing
size, read torch.cuda.max_memory_allocated per device, catch real CUDA OOM,
halve, retry. Estimates derived from measurements are labeled
measured=False (dry-run heuristics) vs measured=True (actual probe runs).
The probe:
- deep-copies the model so real weights are untouched
- uses a copied throwaway optimizer? NO - optimizer states allocate lazily
after step(); we add a +8 bytes/param allowance (Adam moments) instead,
documented as an estimate.
- restores RNG state afterwards, and performs ONE gated empty_cache() after
its own probe allocations (memory.probe_empty_cache=true). Rationale: the
probe's cached blocks are LPU-owned garbage; without releasing them the
real training start would inherit a fragmented allocator. This is the
documented, config-gated exception to "no empty_cache during training".
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import List, Optional, Tuple
from ..compat import require_torch
from .tensor_cache import _safe_deepcopy
@dataclass
class BatchSizeEstimate:
batch_size: int
measured: bool
per_image_vram_mb: Optional[float]
headroom_mb: float
method: str # "probe" | "param_heuristic"
note: str
def _synthetic_like(example_input, batch: int):
torch = require_torch()
if isinstance(example_input, (tuple, list)):
return tuple(_synthetic_like(e, batch) for e in example_input)
if isinstance(example_input, dict):
return {k: _synthetic_like(v, batch) for k, v in example_input.items()}
if hasattr(example_input, "new_full"):
shape = (batch,) + tuple(example_input.shape[1:])
return example_input.new_full(shape, 0.0).normal_()
return example_input
def _to(t, device):
if hasattr(t, "to"):
return t.to(device, non_blocking=True)
if isinstance(t, (tuple, list)):
return type(t)(_to(x, device) for x in t)
if isinstance(t, dict):
return {k: _to(v, device) for k, v in t.items()}
return t
def _param_bytes(module) -> int:
return sum(p.numel() * p.element_size() for p in module.parameters())
def estimate_batch_memory(
model,
example_input,
device_index: int = 0,
max_batch: int = 256,
headroom_mb: float = 1024.0,
allow_probe: bool = True,
allow_empty_cache: bool = True,
) -> BatchSizeEstimate:
"""Return the largest power-of-two batch that fits within VRAM limits."""
torch = require_torch()
if not torch.cuda.is_available():
raise RuntimeError("estimate_batch_memory requires CUDA (real probing only)")
device = f"cuda:{device_index}"
free_b, _total = torch.cuda.mem_get_info(device_index)
usable_mb = free_b / (1024 * 1024) - headroom_mb
if usable_mb <= 0:
return BatchSizeEstimate(
1, False, None, headroom_mb, "param_heuristic",
"free VRAM below headroom; returned minimum batch 1",
)
if not allow_probe:
# dry heuristic: params*4 copies (weights+grads+2 Adam moments)
# + crude activation estimate of 64x input pixels x 4B
est = _param_bytes(model) * 4 / (1024 * 1024)
act = 1.0 # minimal
bs = max(1, int(usable_mb / max(1.0, est / 8 + act)))
return BatchSizeEstimate(
min(bs, max_batch), False, None, headroom_mb, "param_heuristic",
"formula-only estimate (probe disabled); treat as an ESTIMATE",
)
# empirical probe
was_training = model.training
probe_model = _safe_deepcopy(model).to(device)
probe_model.train()
# freeze BN/dropout determinism is not needed for MEMORY sizing; skip.
best, per_img = 1, None
batch = 1
gen_state = None
cuda_gen_state = None
try:
gen_state = torch.get_rng_state()
try:
cuda_gen_state = torch.cuda.get_rng_state(device_index) # probe consumes CUDA RNG too
except Exception: # noqa: BLE001
pass
except Exception: # noqa: BLE001
pass
try:
while batch <= max_batch:
torch.cuda.reset_peak_memory_stats(device_index)
x = None
try:
x = _to(_synthetic_like(example_input, batch), device)
y = probe_model(x)
if not isinstance(y, torch.Tensor):
y = y[0]
loss = y.float().sum()
loss.backward()
peak = torch.cuda.max_memory_allocated(device_index) / (1024 * 1024)
# include Adam moment allowance: +8B/param
adam_mb = _param_bytes(probe_model) * 2 / (1024 * 1024)
total = peak + adam_mb
if total > usable_mb:
break
best, per_img = batch, max(1e-6, total / batch)
batch *= 2
except torch.cuda.OutOfMemoryError:
break
except RuntimeError as exc:
if "out of memory" in str(exc).lower():
break
raise
finally:
# drop references so their cached blocks can be released below
for p in probe_model.parameters():
if p.grad is not None:
p.grad = None
y = None
loss = None
x = None
finally:
del probe_model
try:
torch.set_rng_state(gen_state)
if cuda_gen_state is not None:
torch.cuda.set_rng_state(cuda_gen_state, device_index)
except Exception: # noqa: BLE001
pass
if was_training:
model.train()
else:
model.eval()
# THE sanctioned empty_cache after LPU's own probing (config-gated):
# memory.probe_empty_cache=false disables it everywhere.
if allow_empty_cache:
torch.cuda.empty_cache()
return BatchSizeEstimate(
batch_size=best,
measured=True,
per_image_vram_mb=per_img,
headroom_mb=headroom_mb,
method="probe",
note="measured by empirical forward+backward probing on a model clone",
)
def find_max_batch_size(
model,
example_input,
requested_batch: int,
device_index: int = 0,
headroom_mb: float = 1024.0,
) -> Tuple[int, int]:
"""(safe_batch, accum_steps) such that safe_batch*accum >= requested_batch."""
est = estimate_batch_memory(
model, example_input, device_index=device_index, headroom_mb=headroom_mb
)
safe = min(requested_batch, est.batch_size)
accum = max(1, (requested_batch + safe - 1) // safe)
return safe, accum