"""Automatic batch-size management with OOM recovery (V4). Contract (spec 9.2 + V4 audit 3.2): - effective batch size is PRESERVED via gradient accumulation when the micro-batch must shrink. The optimization math (as far as LR/step semantics go) is unchanged in total sample count per optimizer step. - every adjustment is recorded (orig -> new, reason, timestamp) and pushed to the session report; nothing silent. - user can disable via memory.automatic_batch_size=false, in which case an OOM is re-raised as-is (user training semantics untouched). V4 changes vs V3 (audit fixes): - BatchState.should_step() is now a PURE QUERY (no hidden counter bump). The counter advances only through advance(), which the session calls once per SUCCESSFULLY back-propagated micro-batch. The V3 side-effectful should_step() made "asking" and "doing" the same operation - that semantic confusion is gone. - sample accounting: samples_in_window / samples_done make the zero-loss guarantee auditable (every sample processed exactly once per window). - per-GPU probing (memory.per_gpu_probe): the initial safe batch is derived by probing EACH card independently and using the most constrained one, scaled by replica count for multi-GPU strategies - V3 probed one device and treated dual-P100 VRAM as if pooled. """ from __future__ import annotations from dataclasses import dataclass, field from typing import List, Optional, Sequence from ..compat import require_torch from ..memory.gpu_allocator import estimate_batch_memory, BatchSizeEstimate @dataclass class BatchAdjustment: ts: float from_batch: int to_batch: int accum_steps: int reason: str measured: bool def summary(self) -> dict: return { "ts": self.ts, "from_batch": self.from_batch, "to_batch": self.to_batch, "accum_steps": self.accum_steps, "effective_batch": self.to_batch * self.accum_steps, "reason": self.reason, "measured": self.measured, } @dataclass class BatchState: requested_batch: int micro_batch: int accum_steps: int = 1 step_in_accum: int = 0 samples_in_window: int = 0 samples_done_epoch: int = 0 # The window target IN SAMPLES. When None it derives as micro*accum # (equal-micro assumption). Set EXPLICITLY by initial_state()/on_oom() # to the intended effective batch so that uneven tails (13+3 instead of # 8+8) still close the window at exactly the same sample count. target_effective: Optional[int] = None adjustments: List[BatchAdjustment] = field(default_factory=list) @property def effective_batch(self) -> int: """Samples per optimizer step (the exact window target).""" if self.target_effective is not None: return self.target_effective return self.micro_batch * self.accum_steps def advance(self, samples: int) -> bool: """Record one SUCCESSFUL backward pass over `samples` images. V4: window completion is SAMPLE-driven, not micro-count-driven: the window closes when samples_in_window reaches the effective batch (micro * accum). This keeps the effective batch EXACT even when loader batches produce uneven tail micro-batches (audit: "precise preservation of samples and Effective Batch"). Returns True when this micro-batch completes the window. """ self.step_in_accum += 1 self.samples_in_window += samples self.samples_done_epoch += samples return self.samples_in_window >= self.effective_batch def should_step(self) -> bool: """PURE QUERY: would step() now execute an optimizer step? V4 contract: this method NEVER mutates state. It answers the question "is the accumulation window complete?" so callers can log/branch safely. The counters move only via advance(). """ return self.samples_in_window >= self.effective_batch def close_window(self) -> int: """Reset the window after an optimizer step. Returns samples closed.""" n = self.samples_in_window self.step_in_accum = 0 self.samples_in_window = 0 return n def restart_window(self) -> None: """Window restart (OOM recovery / precision fallback): forget partial progress WITHOUT touching samples_done_epoch net - the samples are subtracted here because the replay will re-add them exactly once.""" self.samples_done_epoch -= self.samples_in_window self.step_in_accum = 0 self.samples_in_window = 0 def is_cuda_oom(exc: BaseException) -> bool: torch = require_torch() if isinstance(exc, torch.cuda.OutOfMemoryError): return True return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower() class BatchOptimizer: def __init__(self, config, log_dir=None): from ..monitoring.logger import get_logger self.config = config self._log = get_logger("batch", log_dir=log_dir, console=False) self.enabled = config.get("memory.automatic_batch_size", True) self.halving = config.get("memory.oom_retry_halving", True) self.grad_accum = config.get("training.gradient_accumulation", True) self.headroom = config.get("memory.vram_headroom_mb", 1024.0) self.per_gpu_probe = config.get("memory.per_gpu_probe", True) # threads the documented empty_cache gate into the probe (the schema # consumer for memory.probe_empty_cache is satisfied here) self.probe_empty_cache = config.get("memory.probe_empty_cache", True) # -- initial sizing ------------------------------------------------------- def initial_state(self, model, example_input, requested_batch: int, device_indices: Optional[Sequence[int]] = None, replicas: int = 1) -> BatchState: """Choose a safe starting micro-batch (measured probe when possible). V4: `device_indices` are probed INDEPENDENTLY (per-card VRAM is never pooled). For a multi-GPU data-parallel strategy the per-card batch is micro/replicas, so the total safe micro-batch is the most constrained card's capacity times the number of replicas. """ if not self.enabled: return BatchState(requested_batch, requested_batch, target_effective=requested_batch) indices = list(device_indices) if device_indices else [0] if not self.per_gpu_probe: indices = indices[:1] per_card: dict = {} method = "probe" measured_any = False for idx in indices: try: est: BatchSizeEstimate = estimate_batch_memory( model, example_input, device_index=idx, headroom_mb=self.headroom, allow_empty_cache=self.probe_empty_cache, ) per_card[idx] = est measured_any = measured_any or est.measured method = est.method except Exception as exc: # noqa: BLE001 self._log.warning("batch probe on cuda:%d unavailable (%s)", idx, exc) if not per_card: self._log.warning("batch probing unavailable; using requested %d", requested_batch) return BatchState(requested_batch, requested_batch) # the most constrained card decides the per-replica batch per_replica_cap = min(est.batch_size for est in per_card.values()) total_cap = per_replica_cap * max(1, replicas) safe = min(requested_batch, total_cap) safe = max(1, safe) accum = max(1, (requested_batch + safe - 1) // safe) if self.grad_accum else 1 target = requested_batch if self.grad_accum else safe state = BatchState(requested_batch, safe, accum, target_effective=target) if safe != requested_batch: card_note = ", ".join( f"cuda:{i}: {e.batch_size}" for i, e in sorted(per_card.items()) ) state.adjustments.append(BatchAdjustment( __import__("time").time(), requested_batch, safe, accum, f"initial safe size from {method} " f"({'measured' if measured_any else 'estimate'}; per-card safe " f"batches [{card_note}] x {max(1, replicas)} replica(s))", measured_any, )) self._log.info( "batch state: micro=%d accum=%d effective=%d " "(per-card safe=[%s], replicas=%d, method=%s)", safe, accum, safe * accum, ", ".join(f"cuda:{i}={e.batch_size}" for i, e in sorted(per_card.items())), max(1, replicas), method, ) return state # -- OOM reaction ---------------------------------------------------------- def on_oom(self, state: BatchState, detail: str = "") -> Optional[BatchState]: """OOM recovery: halve micro-batch, double accumulation. Returns a NEW state (original preserved for reporting), or None when recovery is impossible/disabled (caller must re-raise). V4 NOTE: this method only computes the new sizing. It does NOT touch gradients or re-run anything - the session's recovery procedure (window replay) is what guarantees no sample is lost or duplicated. """ import time if not self.enabled or not self.halving: return None if state.micro_batch <= 1: return None new_micro = max(1, state.micro_batch // 2) # keep the effective batch EXACTLY the same (the window target is # carried over; only the nominal micro/accum split changes) if self.grad_accum: accum = max(1, (state.effective_batch + new_micro - 1) // new_micro) else: accum = 1 new_state = BatchState( state.requested_batch, new_micro, accum, target_effective=state.effective_batch, samples_done_epoch=state.samples_done_epoch, adjustments=list(state.adjustments), ) new_state.adjustments.append(BatchAdjustment( time.time(), state.micro_batch, new_micro, accum, f"CUDA OOM recovery{' - ' + detail if detail else ''}: halved micro-batch, " f"effective batch preserved at {new_micro * accum}", True, )) self._log.warning("OOM recovery: micro %d -> %d, accum %d -> %d (effective %d)", state.micro_batch, new_micro, state.accum_steps, accum, new_micro * accum) return new_state