Spaces:
Running
Running
File size: 10,818 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 | """Automatic batch-size management with OOM recovery (V4).
Contract (spec 9.2 + V4 audit 3.2):
- effective batch size is PRESERVED via gradient accumulation when the
micro-batch must shrink. The optimization math (as far as LR/step semantics
go) is unchanged in total sample count per optimizer step.
- every adjustment is recorded (orig -> new, reason, timestamp) and pushed
to the session report; nothing silent.
- user can disable via memory.automatic_batch_size=false, in which case an
OOM is re-raised as-is (user training semantics untouched).
V4 changes vs V3 (audit fixes):
- BatchState.should_step() is now a PURE QUERY (no hidden counter bump).
The counter advances only through advance(), which the session calls once
per SUCCESSFULLY back-propagated micro-batch. The V3 side-effectful
should_step() made "asking" and "doing" the same operation - that semantic
confusion is gone.
- sample accounting: samples_in_window / samples_done make the zero-loss
guarantee auditable (every sample processed exactly once per window).
- per-GPU probing (memory.per_gpu_probe): the initial safe batch is derived
by probing EACH card independently and using the most constrained one,
scaled by replica count for multi-GPU strategies - V3 probed one device
and treated dual-P100 VRAM as if pooled.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import List, Optional, Sequence
from ..compat import require_torch
from ..memory.gpu_allocator import estimate_batch_memory, BatchSizeEstimate
@dataclass
class BatchAdjustment:
ts: float
from_batch: int
to_batch: int
accum_steps: int
reason: str
measured: bool
def summary(self) -> dict:
return {
"ts": self.ts,
"from_batch": self.from_batch,
"to_batch": self.to_batch,
"accum_steps": self.accum_steps,
"effective_batch": self.to_batch * self.accum_steps,
"reason": self.reason,
"measured": self.measured,
}
@dataclass
class BatchState:
requested_batch: int
micro_batch: int
accum_steps: int = 1
step_in_accum: int = 0
samples_in_window: int = 0
samples_done_epoch: int = 0
# The window target IN SAMPLES. When None it derives as micro*accum
# (equal-micro assumption). Set EXPLICITLY by initial_state()/on_oom()
# to the intended effective batch so that uneven tails (13+3 instead of
# 8+8) still close the window at exactly the same sample count.
target_effective: Optional[int] = None
adjustments: List[BatchAdjustment] = field(default_factory=list)
@property
def effective_batch(self) -> int:
"""Samples per optimizer step (the exact window target)."""
if self.target_effective is not None:
return self.target_effective
return self.micro_batch * self.accum_steps
def advance(self, samples: int) -> bool:
"""Record one SUCCESSFUL backward pass over `samples` images.
V4: window completion is SAMPLE-driven, not micro-count-driven:
the window closes when samples_in_window reaches the effective
batch (micro * accum). This keeps the effective batch EXACT even
when loader batches produce uneven tail micro-batches (audit:
"precise preservation of samples and Effective Batch").
Returns True when this micro-batch completes the window.
"""
self.step_in_accum += 1
self.samples_in_window += samples
self.samples_done_epoch += samples
return self.samples_in_window >= self.effective_batch
def should_step(self) -> bool:
"""PURE QUERY: would step() now execute an optimizer step?
V4 contract: this method NEVER mutates state. It answers the
question "is the accumulation window complete?" so callers can
log/branch safely. The counters move only via advance().
"""
return self.samples_in_window >= self.effective_batch
def close_window(self) -> int:
"""Reset the window after an optimizer step. Returns samples closed."""
n = self.samples_in_window
self.step_in_accum = 0
self.samples_in_window = 0
return n
def restart_window(self) -> None:
"""Window restart (OOM recovery / precision fallback): forget partial
progress WITHOUT touching samples_done_epoch net - the samples are
subtracted here because the replay will re-add them exactly once."""
self.samples_done_epoch -= self.samples_in_window
self.step_in_accum = 0
self.samples_in_window = 0
def is_cuda_oom(exc: BaseException) -> bool:
torch = require_torch()
if isinstance(exc, torch.cuda.OutOfMemoryError):
return True
return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower()
class BatchOptimizer:
def __init__(self, config, log_dir=None):
from ..monitoring.logger import get_logger
self.config = config
self._log = get_logger("batch", log_dir=log_dir, console=False)
self.enabled = config.get("memory.automatic_batch_size", True)
self.halving = config.get("memory.oom_retry_halving", True)
self.grad_accum = config.get("training.gradient_accumulation", True)
self.headroom = config.get("memory.vram_headroom_mb", 1024.0)
self.per_gpu_probe = config.get("memory.per_gpu_probe", True)
# threads the documented empty_cache gate into the probe (the schema
# consumer for memory.probe_empty_cache is satisfied here)
self.probe_empty_cache = config.get("memory.probe_empty_cache", True)
# -- initial sizing -------------------------------------------------------
def initial_state(self, model, example_input, requested_batch: int,
device_indices: Optional[Sequence[int]] = None,
replicas: int = 1) -> BatchState:
"""Choose a safe starting micro-batch (measured probe when possible).
V4: `device_indices` are probed INDEPENDENTLY (per-card VRAM is never
pooled). For a multi-GPU data-parallel strategy the per-card batch is
micro/replicas, so the total safe micro-batch is the most constrained
card's capacity times the number of replicas.
"""
if not self.enabled:
return BatchState(requested_batch, requested_batch,
target_effective=requested_batch)
indices = list(device_indices) if device_indices else [0]
if not self.per_gpu_probe:
indices = indices[:1]
per_card: dict = {}
method = "probe"
measured_any = False
for idx in indices:
try:
est: BatchSizeEstimate = estimate_batch_memory(
model, example_input,
device_index=idx,
headroom_mb=self.headroom,
allow_empty_cache=self.probe_empty_cache,
)
per_card[idx] = est
measured_any = measured_any or est.measured
method = est.method
except Exception as exc: # noqa: BLE001
self._log.warning("batch probe on cuda:%d unavailable (%s)", idx, exc)
if not per_card:
self._log.warning("batch probing unavailable; using requested %d",
requested_batch)
return BatchState(requested_batch, requested_batch)
# the most constrained card decides the per-replica batch
per_replica_cap = min(est.batch_size for est in per_card.values())
total_cap = per_replica_cap * max(1, replicas)
safe = min(requested_batch, total_cap)
safe = max(1, safe)
accum = max(1, (requested_batch + safe - 1) // safe) if self.grad_accum else 1
target = requested_batch if self.grad_accum else safe
state = BatchState(requested_batch, safe, accum,
target_effective=target)
if safe != requested_batch:
card_note = ", ".join(
f"cuda:{i}: {e.batch_size}" for i, e in sorted(per_card.items())
)
state.adjustments.append(BatchAdjustment(
__import__("time").time(), requested_batch, safe, accum,
f"initial safe size from {method} "
f"({'measured' if measured_any else 'estimate'}; per-card safe "
f"batches [{card_note}] x {max(1, replicas)} replica(s))",
measured_any,
))
self._log.info(
"batch state: micro=%d accum=%d effective=%d "
"(per-card safe=[%s], replicas=%d, method=%s)",
safe, accum, safe * accum,
", ".join(f"cuda:{i}={e.batch_size}" for i, e in sorted(per_card.items())),
max(1, replicas), method,
)
return state
# -- OOM reaction ----------------------------------------------------------
def on_oom(self, state: BatchState, detail: str = "") -> Optional[BatchState]:
"""OOM recovery: halve micro-batch, double accumulation.
Returns a NEW state (original preserved for reporting), or None when
recovery is impossible/disabled (caller must re-raise).
V4 NOTE: this method only computes the new sizing. It does NOT touch
gradients or re-run anything - the session's recovery procedure
(window replay) is what guarantees no sample is lost or duplicated.
"""
import time
if not self.enabled or not self.halving:
return None
if state.micro_batch <= 1:
return None
new_micro = max(1, state.micro_batch // 2)
# keep the effective batch EXACTLY the same (the window target is
# carried over; only the nominal micro/accum split changes)
if self.grad_accum:
accum = max(1, (state.effective_batch + new_micro - 1) // new_micro)
else:
accum = 1
new_state = BatchState(
state.requested_batch, new_micro, accum,
target_effective=state.effective_batch,
samples_done_epoch=state.samples_done_epoch,
adjustments=list(state.adjustments),
)
new_state.adjustments.append(BatchAdjustment(
time.time(), state.micro_batch, new_micro, accum,
f"CUDA OOM recovery{' - ' + detail if detail else ''}: halved micro-batch, "
f"effective batch preserved at {new_micro * accum}",
True,
))
self._log.warning("OOM recovery: micro %d -> %d, accum %d -> %d (effective %d)",
state.micro_batch, new_micro, state.accum_steps, accum,
new_micro * accum)
return new_state
|