Spaces:
Running
Running
Download lpu/training/batch_optimizer.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/training/batch_optimizer.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/lpu/training/batch_optimizer.py
-
curl -L -o batch_optimizer.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/training/batch_optimizer.py
10.8 kB
| """Automatic batch-size management with OOM recovery (V4). | |
| Contract (spec 9.2 + V4 audit 3.2): | |
| - effective batch size is PRESERVED via gradient accumulation when the | |
| micro-batch must shrink. The optimization math (as far as LR/step semantics | |
| go) is unchanged in total sample count per optimizer step. | |
| - every adjustment is recorded (orig -> new, reason, timestamp) and pushed | |
| to the session report; nothing silent. | |
| - user can disable via memory.automatic_batch_size=false, in which case an | |
| OOM is re-raised as-is (user training semantics untouched). | |
| V4 changes vs V3 (audit fixes): | |
| - BatchState.should_step() is now a PURE QUERY (no hidden counter bump). | |
| The counter advances only through advance(), which the session calls once | |
| per SUCCESSFULLY back-propagated micro-batch. The V3 side-effectful | |
| should_step() made "asking" and "doing" the same operation - that semantic | |
| confusion is gone. | |
| - sample accounting: samples_in_window / samples_done make the zero-loss | |
| guarantee auditable (every sample processed exactly once per window). | |
| - per-GPU probing (memory.per_gpu_probe): the initial safe batch is derived | |
| by probing EACH card independently and using the most constrained one, | |
| scaled by replica count for multi-GPU strategies - V3 probed one device | |
| and treated dual-P100 VRAM as if pooled. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass, field | |
| from typing import List, Optional, Sequence | |
| from ..compat import require_torch | |
| from ..memory.gpu_allocator import estimate_batch_memory, BatchSizeEstimate | |
| class BatchAdjustment: | |
| ts: float | |
| from_batch: int | |
| to_batch: int | |
| accum_steps: int | |
| reason: str | |
| measured: bool | |
| def summary(self) -> dict: | |
| return { | |
| "ts": self.ts, | |
| "from_batch": self.from_batch, | |
| "to_batch": self.to_batch, | |
| "accum_steps": self.accum_steps, | |
| "effective_batch": self.to_batch * self.accum_steps, | |
| "reason": self.reason, | |
| "measured": self.measured, | |
| } | |
| class BatchState: | |
| requested_batch: int | |
| micro_batch: int | |
| accum_steps: int = 1 | |
| step_in_accum: int = 0 | |
| samples_in_window: int = 0 | |
| samples_done_epoch: int = 0 | |
| # The window target IN SAMPLES. When None it derives as micro*accum | |
| # (equal-micro assumption). Set EXPLICITLY by initial_state()/on_oom() | |
| # to the intended effective batch so that uneven tails (13+3 instead of | |
| # 8+8) still close the window at exactly the same sample count. | |
| target_effective: Optional[int] = None | |
| adjustments: List[BatchAdjustment] = field(default_factory=list) | |
| def effective_batch(self) -> int: | |
| """Samples per optimizer step (the exact window target).""" | |
| if self.target_effective is not None: | |
| return self.target_effective | |
| return self.micro_batch * self.accum_steps | |
| def advance(self, samples: int) -> bool: | |
| """Record one SUCCESSFUL backward pass over `samples` images. | |
| V4: window completion is SAMPLE-driven, not micro-count-driven: | |
| the window closes when samples_in_window reaches the effective | |
| batch (micro * accum). This keeps the effective batch EXACT even | |
| when loader batches produce uneven tail micro-batches (audit: | |
| "precise preservation of samples and Effective Batch"). | |
| Returns True when this micro-batch completes the window. | |
| """ | |
| self.step_in_accum += 1 | |
| self.samples_in_window += samples | |
| self.samples_done_epoch += samples | |
| return self.samples_in_window >= self.effective_batch | |
| def should_step(self) -> bool: | |
| """PURE QUERY: would step() now execute an optimizer step? | |
| V4 contract: this method NEVER mutates state. It answers the | |
| question "is the accumulation window complete?" so callers can | |
| log/branch safely. The counters move only via advance(). | |
| """ | |
| return self.samples_in_window >= self.effective_batch | |
| def close_window(self) -> int: | |
| """Reset the window after an optimizer step. Returns samples closed.""" | |
| n = self.samples_in_window | |
| self.step_in_accum = 0 | |
| self.samples_in_window = 0 | |
| return n | |
| def restart_window(self) -> None: | |
| """Window restart (OOM recovery / precision fallback): forget partial | |
| progress WITHOUT touching samples_done_epoch net - the samples are | |
| subtracted here because the replay will re-add them exactly once.""" | |
| self.samples_done_epoch -= self.samples_in_window | |
| self.step_in_accum = 0 | |
| self.samples_in_window = 0 | |
| def is_cuda_oom(exc: BaseException) -> bool: | |
| torch = require_torch() | |
| if isinstance(exc, torch.cuda.OutOfMemoryError): | |
| return True | |
| return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower() | |
| class BatchOptimizer: | |
| def __init__(self, config, log_dir=None): | |
| from ..monitoring.logger import get_logger | |
| self.config = config | |
| self._log = get_logger("batch", log_dir=log_dir, console=False) | |
| self.enabled = config.get("memory.automatic_batch_size", True) | |
| self.halving = config.get("memory.oom_retry_halving", True) | |
| self.grad_accum = config.get("training.gradient_accumulation", True) | |
| self.headroom = config.get("memory.vram_headroom_mb", 1024.0) | |
| self.per_gpu_probe = config.get("memory.per_gpu_probe", True) | |
| # threads the documented empty_cache gate into the probe (the schema | |
| # consumer for memory.probe_empty_cache is satisfied here) | |
| self.probe_empty_cache = config.get("memory.probe_empty_cache", True) | |
| # -- initial sizing ------------------------------------------------------- | |
| def initial_state(self, model, example_input, requested_batch: int, | |
| device_indices: Optional[Sequence[int]] = None, | |
| replicas: int = 1) -> BatchState: | |
| """Choose a safe starting micro-batch (measured probe when possible). | |
| V4: `device_indices` are probed INDEPENDENTLY (per-card VRAM is never | |
| pooled). For a multi-GPU data-parallel strategy the per-card batch is | |
| micro/replicas, so the total safe micro-batch is the most constrained | |
| card's capacity times the number of replicas. | |
| """ | |
| if not self.enabled: | |
| return BatchState(requested_batch, requested_batch, | |
| target_effective=requested_batch) | |
| indices = list(device_indices) if device_indices else [0] | |
| if not self.per_gpu_probe: | |
| indices = indices[:1] | |
| per_card: dict = {} | |
| method = "probe" | |
| measured_any = False | |
| for idx in indices: | |
| try: | |
| est: BatchSizeEstimate = estimate_batch_memory( | |
| model, example_input, | |
| device_index=idx, | |
| headroom_mb=self.headroom, | |
| allow_empty_cache=self.probe_empty_cache, | |
| ) | |
| per_card[idx] = est | |
| measured_any = measured_any or est.measured | |
| method = est.method | |
| except Exception as exc: # noqa: BLE001 | |
| self._log.warning("batch probe on cuda:%d unavailable (%s)", idx, exc) | |
| if not per_card: | |
| self._log.warning("batch probing unavailable; using requested %d", | |
| requested_batch) | |
| return BatchState(requested_batch, requested_batch) | |
| # the most constrained card decides the per-replica batch | |
| per_replica_cap = min(est.batch_size for est in per_card.values()) | |
| total_cap = per_replica_cap * max(1, replicas) | |
| safe = min(requested_batch, total_cap) | |
| safe = max(1, safe) | |
| accum = max(1, (requested_batch + safe - 1) // safe) if self.grad_accum else 1 | |
| target = requested_batch if self.grad_accum else safe | |
| state = BatchState(requested_batch, safe, accum, | |
| target_effective=target) | |
| if safe != requested_batch: | |
| card_note = ", ".join( | |
| f"cuda:{i}: {e.batch_size}" for i, e in sorted(per_card.items()) | |
| ) | |
| state.adjustments.append(BatchAdjustment( | |
| __import__("time").time(), requested_batch, safe, accum, | |
| f"initial safe size from {method} " | |
| f"({'measured' if measured_any else 'estimate'}; per-card safe " | |
| f"batches [{card_note}] x {max(1, replicas)} replica(s))", | |
| measured_any, | |
| )) | |
| self._log.info( | |
| "batch state: micro=%d accum=%d effective=%d " | |
| "(per-card safe=[%s], replicas=%d, method=%s)", | |
| safe, accum, safe * accum, | |
| ", ".join(f"cuda:{i}={e.batch_size}" for i, e in sorted(per_card.items())), | |
| max(1, replicas), method, | |
| ) | |
| return state | |
| # -- OOM reaction ---------------------------------------------------------- | |
| def on_oom(self, state: BatchState, detail: str = "") -> Optional[BatchState]: | |
| """OOM recovery: halve micro-batch, double accumulation. | |
| Returns a NEW state (original preserved for reporting), or None when | |
| recovery is impossible/disabled (caller must re-raise). | |
| V4 NOTE: this method only computes the new sizing. It does NOT touch | |
| gradients or re-run anything - the session's recovery procedure | |
| (window replay) is what guarantees no sample is lost or duplicated. | |
| """ | |
| import time | |
| if not self.enabled or not self.halving: | |
| return None | |
| if state.micro_batch <= 1: | |
| return None | |
| new_micro = max(1, state.micro_batch // 2) | |
| # keep the effective batch EXACTLY the same (the window target is | |
| # carried over; only the nominal micro/accum split changes) | |
| if self.grad_accum: | |
| accum = max(1, (state.effective_batch + new_micro - 1) // new_micro) | |
| else: | |
| accum = 1 | |
| new_state = BatchState( | |
| state.requested_batch, new_micro, accum, | |
| target_effective=state.effective_batch, | |
| samples_done_epoch=state.samples_done_epoch, | |
| adjustments=list(state.adjustments), | |
| ) | |
| new_state.adjustments.append(BatchAdjustment( | |
| time.time(), state.micro_batch, new_micro, accum, | |
| f"CUDA OOM recovery{' - ' + detail if detail else ''}: halved micro-batch, " | |
| f"effective batch preserved at {new_micro * accum}", | |
| True, | |
| )) | |
| self._log.warning("OOM recovery: micro %d -> %d, accum %d -> %d (effective %d)", | |
| state.micro_batch, new_micro, state.accum_steps, accum, | |
| new_micro * accum) | |
| return new_state | |