Spaces:
Running
Running
Download lpu/memory/gpu_allocator.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 6.83 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/gpu_allocator.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/memory/gpu_allocator.py
-
curl -L -o gpu_allocator.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/gpu_allocator.py
6.83 kB
| """Empirical batch-size estimation and VRAM probing. | |
| Technique (real, measured - not a formula fantasy): | |
| clone the model, run forward+backward on synthetic batches of increasing | |
| size, read torch.cuda.max_memory_allocated per device, catch real CUDA OOM, | |
| halve, retry. Estimates derived from measurements are labeled | |
| measured=False (dry-run heuristics) vs measured=True (actual probe runs). | |
| The probe: | |
| - deep-copies the model so real weights are untouched | |
| - uses a copied throwaway optimizer? NO - optimizer states allocate lazily | |
| after step(); we add a +8 bytes/param allowance (Adam moments) instead, | |
| documented as an estimate. | |
| - restores RNG state afterwards, and performs ONE gated empty_cache() after | |
| its own probe allocations (memory.probe_empty_cache=true). Rationale: the | |
| probe's cached blocks are LPU-owned garbage; without releasing them the | |
| real training start would inherit a fragmented allocator. This is the | |
| documented, config-gated exception to "no empty_cache during training". | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| from typing import List, Optional, Tuple | |
| from ..compat import require_torch | |
| from .tensor_cache import _safe_deepcopy | |
| class BatchSizeEstimate: | |
| batch_size: int | |
| measured: bool | |
| per_image_vram_mb: Optional[float] | |
| headroom_mb: float | |
| method: str # "probe" | "param_heuristic" | |
| note: str | |
| def _synthetic_like(example_input, batch: int): | |
| torch = require_torch() | |
| if isinstance(example_input, (tuple, list)): | |
| return tuple(_synthetic_like(e, batch) for e in example_input) | |
| if isinstance(example_input, dict): | |
| return {k: _synthetic_like(v, batch) for k, v in example_input.items()} | |
| if hasattr(example_input, "new_full"): | |
| shape = (batch,) + tuple(example_input.shape[1:]) | |
| return example_input.new_full(shape, 0.0).normal_() | |
| return example_input | |
| def _to(t, device): | |
| if hasattr(t, "to"): | |
| return t.to(device, non_blocking=True) | |
| if isinstance(t, (tuple, list)): | |
| return type(t)(_to(x, device) for x in t) | |
| if isinstance(t, dict): | |
| return {k: _to(v, device) for k, v in t.items()} | |
| return t | |
| def _param_bytes(module) -> int: | |
| return sum(p.numel() * p.element_size() for p in module.parameters()) | |
| def estimate_batch_memory( | |
| model, | |
| example_input, | |
| device_index: int = 0, | |
| max_batch: int = 256, | |
| headroom_mb: float = 1024.0, | |
| allow_probe: bool = True, | |
| allow_empty_cache: bool = True, | |
| ) -> BatchSizeEstimate: | |
| """Return the largest power-of-two batch that fits within VRAM limits.""" | |
| torch = require_torch() | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError("estimate_batch_memory requires CUDA (real probing only)") | |
| device = f"cuda:{device_index}" | |
| free_b, _total = torch.cuda.mem_get_info(device_index) | |
| usable_mb = free_b / (1024 * 1024) - headroom_mb | |
| if usable_mb <= 0: | |
| return BatchSizeEstimate( | |
| 1, False, None, headroom_mb, "param_heuristic", | |
| "free VRAM below headroom; returned minimum batch 1", | |
| ) | |
| if not allow_probe: | |
| # dry heuristic: params*4 copies (weights+grads+2 Adam moments) | |
| # + crude activation estimate of 64x input pixels x 4B | |
| est = _param_bytes(model) * 4 / (1024 * 1024) | |
| act = 1.0 # minimal | |
| bs = max(1, int(usable_mb / max(1.0, est / 8 + act))) | |
| return BatchSizeEstimate( | |
| min(bs, max_batch), False, None, headroom_mb, "param_heuristic", | |
| "formula-only estimate (probe disabled); treat as an ESTIMATE", | |
| ) | |
| # empirical probe | |
| was_training = model.training | |
| probe_model = _safe_deepcopy(model).to(device) | |
| probe_model.train() | |
| # freeze BN/dropout determinism is not needed for MEMORY sizing; skip. | |
| best, per_img = 1, None | |
| batch = 1 | |
| gen_state = None | |
| cuda_gen_state = None | |
| try: | |
| gen_state = torch.get_rng_state() | |
| try: | |
| cuda_gen_state = torch.cuda.get_rng_state(device_index) # probe consumes CUDA RNG too | |
| except Exception: # noqa: BLE001 | |
| pass | |
| except Exception: # noqa: BLE001 | |
| pass | |
| try: | |
| while batch <= max_batch: | |
| torch.cuda.reset_peak_memory_stats(device_index) | |
| x = None | |
| try: | |
| x = _to(_synthetic_like(example_input, batch), device) | |
| y = probe_model(x) | |
| if not isinstance(y, torch.Tensor): | |
| y = y[0] | |
| loss = y.float().sum() | |
| loss.backward() | |
| peak = torch.cuda.max_memory_allocated(device_index) / (1024 * 1024) | |
| # include Adam moment allowance: +8B/param | |
| adam_mb = _param_bytes(probe_model) * 2 / (1024 * 1024) | |
| total = peak + adam_mb | |
| if total > usable_mb: | |
| break | |
| best, per_img = batch, max(1e-6, total / batch) | |
| batch *= 2 | |
| except torch.cuda.OutOfMemoryError: | |
| break | |
| except RuntimeError as exc: | |
| if "out of memory" in str(exc).lower(): | |
| break | |
| raise | |
| finally: | |
| # drop references so their cached blocks can be released below | |
| for p in probe_model.parameters(): | |
| if p.grad is not None: | |
| p.grad = None | |
| y = None | |
| loss = None | |
| x = None | |
| finally: | |
| del probe_model | |
| try: | |
| torch.set_rng_state(gen_state) | |
| if cuda_gen_state is not None: | |
| torch.cuda.set_rng_state(cuda_gen_state, device_index) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| if was_training: | |
| model.train() | |
| else: | |
| model.eval() | |
| # THE sanctioned empty_cache after LPU's own probing (config-gated): | |
| # memory.probe_empty_cache=false disables it everywhere. | |
| if allow_empty_cache: | |
| torch.cuda.empty_cache() | |
| return BatchSizeEstimate( | |
| batch_size=best, | |
| measured=True, | |
| per_image_vram_mb=per_img, | |
| headroom_mb=headroom_mb, | |
| method="probe", | |
| note="measured by empirical forward+backward probing on a model clone", | |
| ) | |
| def find_max_batch_size( | |
| model, | |
| example_input, | |
| requested_batch: int, | |
| device_index: int = 0, | |
| headroom_mb: float = 1024.0, | |
| ) -> Tuple[int, int]: | |
| """(safe_batch, accum_steps) such that safe_batch*accum >= requested_batch.""" | |
| est = estimate_batch_memory( | |
| model, example_input, device_index=device_index, headroom_mb=headroom_mb | |
| ) | |
| safe = min(requested_batch, est.batch_size) | |
| accum = max(1, (requested_batch + safe - 1) // safe) | |
| return safe, accum | |