"""Empirical batch-size estimation and VRAM probing. Technique (real, measured - not a formula fantasy): clone the model, run forward+backward on synthetic batches of increasing size, read torch.cuda.max_memory_allocated per device, catch real CUDA OOM, halve, retry. Estimates derived from measurements are labeled measured=False (dry-run heuristics) vs measured=True (actual probe runs). The probe: - deep-copies the model so real weights are untouched - uses a copied throwaway optimizer? NO - optimizer states allocate lazily after step(); we add a +8 bytes/param allowance (Adam moments) instead, documented as an estimate. - restores RNG state afterwards, and performs ONE gated empty_cache() after its own probe allocations (memory.probe_empty_cache=true). Rationale: the probe's cached blocks are LPU-owned garbage; without releasing them the real training start would inherit a fragmented allocator. This is the documented, config-gated exception to "no empty_cache during training". """ from __future__ import annotations from dataclasses import dataclass from typing import List, Optional, Tuple from ..compat import require_torch from .tensor_cache import _safe_deepcopy @dataclass class BatchSizeEstimate: batch_size: int measured: bool per_image_vram_mb: Optional[float] headroom_mb: float method: str # "probe" | "param_heuristic" note: str def _synthetic_like(example_input, batch: int): torch = require_torch() if isinstance(example_input, (tuple, list)): return tuple(_synthetic_like(e, batch) for e in example_input) if isinstance(example_input, dict): return {k: _synthetic_like(v, batch) for k, v in example_input.items()} if hasattr(example_input, "new_full"): shape = (batch,) + tuple(example_input.shape[1:]) return example_input.new_full(shape, 0.0).normal_() return example_input def _to(t, device): if hasattr(t, "to"): return t.to(device, non_blocking=True) if isinstance(t, (tuple, list)): return type(t)(_to(x, device) for x in t) if isinstance(t, dict): return {k: _to(v, device) for k, v in t.items()} return t def _param_bytes(module) -> int: return sum(p.numel() * p.element_size() for p in module.parameters()) def estimate_batch_memory( model, example_input, device_index: int = 0, max_batch: int = 256, headroom_mb: float = 1024.0, allow_probe: bool = True, allow_empty_cache: bool = True, ) -> BatchSizeEstimate: """Return the largest power-of-two batch that fits within VRAM limits.""" torch = require_torch() if not torch.cuda.is_available(): raise RuntimeError("estimate_batch_memory requires CUDA (real probing only)") device = f"cuda:{device_index}" free_b, _total = torch.cuda.mem_get_info(device_index) usable_mb = free_b / (1024 * 1024) - headroom_mb if usable_mb <= 0: return BatchSizeEstimate( 1, False, None, headroom_mb, "param_heuristic", "free VRAM below headroom; returned minimum batch 1", ) if not allow_probe: # dry heuristic: params*4 copies (weights+grads+2 Adam moments) # + crude activation estimate of 64x input pixels x 4B est = _param_bytes(model) * 4 / (1024 * 1024) act = 1.0 # minimal bs = max(1, int(usable_mb / max(1.0, est / 8 + act))) return BatchSizeEstimate( min(bs, max_batch), False, None, headroom_mb, "param_heuristic", "formula-only estimate (probe disabled); treat as an ESTIMATE", ) # empirical probe was_training = model.training probe_model = _safe_deepcopy(model).to(device) probe_model.train() # freeze BN/dropout determinism is not needed for MEMORY sizing; skip. best, per_img = 1, None batch = 1 gen_state = None cuda_gen_state = None try: gen_state = torch.get_rng_state() try: cuda_gen_state = torch.cuda.get_rng_state(device_index) # probe consumes CUDA RNG too except Exception: # noqa: BLE001 pass except Exception: # noqa: BLE001 pass try: while batch <= max_batch: torch.cuda.reset_peak_memory_stats(device_index) x = None try: x = _to(_synthetic_like(example_input, batch), device) y = probe_model(x) if not isinstance(y, torch.Tensor): y = y[0] loss = y.float().sum() loss.backward() peak = torch.cuda.max_memory_allocated(device_index) / (1024 * 1024) # include Adam moment allowance: +8B/param adam_mb = _param_bytes(probe_model) * 2 / (1024 * 1024) total = peak + adam_mb if total > usable_mb: break best, per_img = batch, max(1e-6, total / batch) batch *= 2 except torch.cuda.OutOfMemoryError: break except RuntimeError as exc: if "out of memory" in str(exc).lower(): break raise finally: # drop references so their cached blocks can be released below for p in probe_model.parameters(): if p.grad is not None: p.grad = None y = None loss = None x = None finally: del probe_model try: torch.set_rng_state(gen_state) if cuda_gen_state is not None: torch.cuda.set_rng_state(cuda_gen_state, device_index) except Exception: # noqa: BLE001 pass if was_training: model.train() else: model.eval() # THE sanctioned empty_cache after LPU's own probing (config-gated): # memory.probe_empty_cache=false disables it everywhere. if allow_empty_cache: torch.cuda.empty_cache() return BatchSizeEstimate( batch_size=best, measured=True, per_image_vram_mb=per_img, headroom_mb=headroom_mb, method="probe", note="measured by empirical forward+backward probing on a model clone", ) def find_max_batch_size( model, example_input, requested_batch: int, device_index: int = 0, headroom_mb: float = 1024.0, ) -> Tuple[int, int]: """(safe_batch, accum_steps) such that safe_batch*accum >= requested_batch.""" est = estimate_batch_memory( model, example_input, device_index=device_index, headroom_mb=headroom_mb ) safe = min(requested_batch, est.batch_size) accum = max(1, (requested_batch + safe - 1) // safe) return safe, accum