Virtual-LPU / lpu /training /batch_optimizer.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
10.8 kB
"""Automatic batch-size management with OOM recovery (V4).
Contract (spec 9.2 + V4 audit 3.2):
- effective batch size is PRESERVED via gradient accumulation when the
micro-batch must shrink. The optimization math (as far as LR/step semantics
go) is unchanged in total sample count per optimizer step.
- every adjustment is recorded (orig -> new, reason, timestamp) and pushed
to the session report; nothing silent.
- user can disable via memory.automatic_batch_size=false, in which case an
OOM is re-raised as-is (user training semantics untouched).
V4 changes vs V3 (audit fixes):
- BatchState.should_step() is now a PURE QUERY (no hidden counter bump).
The counter advances only through advance(), which the session calls once
per SUCCESSFULLY back-propagated micro-batch. The V3 side-effectful
should_step() made "asking" and "doing" the same operation - that semantic
confusion is gone.
- sample accounting: samples_in_window / samples_done make the zero-loss
guarantee auditable (every sample processed exactly once per window).
- per-GPU probing (memory.per_gpu_probe): the initial safe batch is derived
by probing EACH card independently and using the most constrained one,
scaled by replica count for multi-GPU strategies - V3 probed one device
and treated dual-P100 VRAM as if pooled.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import List, Optional, Sequence
from ..compat import require_torch
from ..memory.gpu_allocator import estimate_batch_memory, BatchSizeEstimate
@dataclass
class BatchAdjustment:
ts: float
from_batch: int
to_batch: int
accum_steps: int
reason: str
measured: bool
def summary(self) -> dict:
return {
"ts": self.ts,
"from_batch": self.from_batch,
"to_batch": self.to_batch,
"accum_steps": self.accum_steps,
"effective_batch": self.to_batch * self.accum_steps,
"reason": self.reason,
"measured": self.measured,
}
@dataclass
class BatchState:
requested_batch: int
micro_batch: int
accum_steps: int = 1
step_in_accum: int = 0
samples_in_window: int = 0
samples_done_epoch: int = 0
# The window target IN SAMPLES. When None it derives as micro*accum
# (equal-micro assumption). Set EXPLICITLY by initial_state()/on_oom()
# to the intended effective batch so that uneven tails (13+3 instead of
# 8+8) still close the window at exactly the same sample count.
target_effective: Optional[int] = None
adjustments: List[BatchAdjustment] = field(default_factory=list)
@property
def effective_batch(self) -> int:
"""Samples per optimizer step (the exact window target)."""
if self.target_effective is not None:
return self.target_effective
return self.micro_batch * self.accum_steps
def advance(self, samples: int) -> bool:
"""Record one SUCCESSFUL backward pass over `samples` images.
V4: window completion is SAMPLE-driven, not micro-count-driven:
the window closes when samples_in_window reaches the effective
batch (micro * accum). This keeps the effective batch EXACT even
when loader batches produce uneven tail micro-batches (audit:
"precise preservation of samples and Effective Batch").
Returns True when this micro-batch completes the window.
"""
self.step_in_accum += 1
self.samples_in_window += samples
self.samples_done_epoch += samples
return self.samples_in_window >= self.effective_batch
def should_step(self) -> bool:
"""PURE QUERY: would step() now execute an optimizer step?
V4 contract: this method NEVER mutates state. It answers the
question "is the accumulation window complete?" so callers can
log/branch safely. The counters move only via advance().
"""
return self.samples_in_window >= self.effective_batch
def close_window(self) -> int:
"""Reset the window after an optimizer step. Returns samples closed."""
n = self.samples_in_window
self.step_in_accum = 0
self.samples_in_window = 0
return n
def restart_window(self) -> None:
"""Window restart (OOM recovery / precision fallback): forget partial
progress WITHOUT touching samples_done_epoch net - the samples are
subtracted here because the replay will re-add them exactly once."""
self.samples_done_epoch -= self.samples_in_window
self.step_in_accum = 0
self.samples_in_window = 0
def is_cuda_oom(exc: BaseException) -> bool:
torch = require_torch()
if isinstance(exc, torch.cuda.OutOfMemoryError):
return True
return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower()
class BatchOptimizer:
def __init__(self, config, log_dir=None):
from ..monitoring.logger import get_logger
self.config = config
self._log = get_logger("batch", log_dir=log_dir, console=False)
self.enabled = config.get("memory.automatic_batch_size", True)
self.halving = config.get("memory.oom_retry_halving", True)
self.grad_accum = config.get("training.gradient_accumulation", True)
self.headroom = config.get("memory.vram_headroom_mb", 1024.0)
self.per_gpu_probe = config.get("memory.per_gpu_probe", True)
# threads the documented empty_cache gate into the probe (the schema
# consumer for memory.probe_empty_cache is satisfied here)
self.probe_empty_cache = config.get("memory.probe_empty_cache", True)
# -- initial sizing -------------------------------------------------------
def initial_state(self, model, example_input, requested_batch: int,
device_indices: Optional[Sequence[int]] = None,
replicas: int = 1) -> BatchState:
"""Choose a safe starting micro-batch (measured probe when possible).
V4: `device_indices` are probed INDEPENDENTLY (per-card VRAM is never
pooled). For a multi-GPU data-parallel strategy the per-card batch is
micro/replicas, so the total safe micro-batch is the most constrained
card's capacity times the number of replicas.
"""
if not self.enabled:
return BatchState(requested_batch, requested_batch,
target_effective=requested_batch)
indices = list(device_indices) if device_indices else [0]
if not self.per_gpu_probe:
indices = indices[:1]
per_card: dict = {}
method = "probe"
measured_any = False
for idx in indices:
try:
est: BatchSizeEstimate = estimate_batch_memory(
model, example_input,
device_index=idx,
headroom_mb=self.headroom,
allow_empty_cache=self.probe_empty_cache,
)
per_card[idx] = est
measured_any = measured_any or est.measured
method = est.method
except Exception as exc: # noqa: BLE001
self._log.warning("batch probe on cuda:%d unavailable (%s)", idx, exc)
if not per_card:
self._log.warning("batch probing unavailable; using requested %d",
requested_batch)
return BatchState(requested_batch, requested_batch)
# the most constrained card decides the per-replica batch
per_replica_cap = min(est.batch_size for est in per_card.values())
total_cap = per_replica_cap * max(1, replicas)
safe = min(requested_batch, total_cap)
safe = max(1, safe)
accum = max(1, (requested_batch + safe - 1) // safe) if self.grad_accum else 1
target = requested_batch if self.grad_accum else safe
state = BatchState(requested_batch, safe, accum,
target_effective=target)
if safe != requested_batch:
card_note = ", ".join(
f"cuda:{i}: {e.batch_size}" for i, e in sorted(per_card.items())
)
state.adjustments.append(BatchAdjustment(
__import__("time").time(), requested_batch, safe, accum,
f"initial safe size from {method} "
f"({'measured' if measured_any else 'estimate'}; per-card safe "
f"batches [{card_note}] x {max(1, replicas)} replica(s))",
measured_any,
))
self._log.info(
"batch state: micro=%d accum=%d effective=%d "
"(per-card safe=[%s], replicas=%d, method=%s)",
safe, accum, safe * accum,
", ".join(f"cuda:{i}={e.batch_size}" for i, e in sorted(per_card.items())),
max(1, replicas), method,
)
return state
# -- OOM reaction ----------------------------------------------------------
def on_oom(self, state: BatchState, detail: str = "") -> Optional[BatchState]:
"""OOM recovery: halve micro-batch, double accumulation.
Returns a NEW state (original preserved for reporting), or None when
recovery is impossible/disabled (caller must re-raise).
V4 NOTE: this method only computes the new sizing. It does NOT touch
gradients or re-run anything - the session's recovery procedure
(window replay) is what guarantees no sample is lost or duplicated.
"""
import time
if not self.enabled or not self.halving:
return None
if state.micro_batch <= 1:
return None
new_micro = max(1, state.micro_batch // 2)
# keep the effective batch EXACTLY the same (the window target is
# carried over; only the nominal micro/accum split changes)
if self.grad_accum:
accum = max(1, (state.effective_batch + new_micro - 1) // new_micro)
else:
accum = 1
new_state = BatchState(
state.requested_batch, new_micro, accum,
target_effective=state.effective_batch,
samples_done_epoch=state.samples_done_epoch,
adjustments=list(state.adjustments),
)
new_state.adjustments.append(BatchAdjustment(
time.time(), state.micro_batch, new_micro, accum,
f"CUDA OOM recovery{' - ' + detail if detail else ''}: halved micro-batch, "
f"effective batch preserved at {new_micro * accum}",
True,
))
self._log.warning("OOM recovery: micro %d -> %d, accum %d -> %d (effective %d)",
state.micro_batch, new_micro, state.accum_steps, accum,
new_micro * accum)
return new_state