File size: 10,818 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
"""Automatic batch-size management with OOM recovery (V4).

Contract (spec 9.2 + V4 audit 3.2):
- effective batch size is PRESERVED via gradient accumulation when the
  micro-batch must shrink. The optimization math (as far as LR/step semantics
  go) is unchanged in total sample count per optimizer step.
- every adjustment is recorded (orig -> new, reason, timestamp) and pushed
  to the session report; nothing silent.
- user can disable via memory.automatic_batch_size=false, in which case an
  OOM is re-raised as-is (user training semantics untouched).

V4 changes vs V3 (audit fixes):
- BatchState.should_step() is now a PURE QUERY (no hidden counter bump).
  The counter advances only through advance(), which the session calls once
  per SUCCESSFULLY back-propagated micro-batch. The V3 side-effectful
  should_step() made "asking" and "doing" the same operation - that semantic
  confusion is gone.
- sample accounting: samples_in_window / samples_done make the zero-loss
  guarantee auditable (every sample processed exactly once per window).
- per-GPU probing (memory.per_gpu_probe): the initial safe batch is derived
  by probing EACH card independently and using the most constrained one,
  scaled by replica count for multi-GPU strategies - V3 probed one device
  and treated dual-P100 VRAM as if pooled.
"""

from __future__ import annotations

from dataclasses import dataclass, field
from typing import List, Optional, Sequence

from ..compat import require_torch
from ..memory.gpu_allocator import estimate_batch_memory, BatchSizeEstimate


@dataclass
class BatchAdjustment:
    ts: float
    from_batch: int
    to_batch: int
    accum_steps: int
    reason: str
    measured: bool

    def summary(self) -> dict:
        return {
            "ts": self.ts,
            "from_batch": self.from_batch,
            "to_batch": self.to_batch,
            "accum_steps": self.accum_steps,
            "effective_batch": self.to_batch * self.accum_steps,
            "reason": self.reason,
            "measured": self.measured,
        }


@dataclass
class BatchState:
    requested_batch: int
    micro_batch: int
    accum_steps: int = 1
    step_in_accum: int = 0
    samples_in_window: int = 0
    samples_done_epoch: int = 0
    # The window target IN SAMPLES. When None it derives as micro*accum
    # (equal-micro assumption). Set EXPLICITLY by initial_state()/on_oom()
    # to the intended effective batch so that uneven tails (13+3 instead of
    # 8+8) still close the window at exactly the same sample count.
    target_effective: Optional[int] = None
    adjustments: List[BatchAdjustment] = field(default_factory=list)

    @property
    def effective_batch(self) -> int:
        """Samples per optimizer step (the exact window target)."""
        if self.target_effective is not None:
            return self.target_effective
        return self.micro_batch * self.accum_steps

    def advance(self, samples: int) -> bool:
        """Record one SUCCESSFUL backward pass over `samples` images.

        V4: window completion is SAMPLE-driven, not micro-count-driven:
        the window closes when samples_in_window reaches the effective
        batch (micro * accum). This keeps the effective batch EXACT even
        when loader batches produce uneven tail micro-batches (audit:
        "precise preservation of samples and Effective Batch").
        Returns True when this micro-batch completes the window.
        """
        self.step_in_accum += 1
        self.samples_in_window += samples
        self.samples_done_epoch += samples
        return self.samples_in_window >= self.effective_batch

    def should_step(self) -> bool:
        """PURE QUERY: would step() now execute an optimizer step?

        V4 contract: this method NEVER mutates state. It answers the
        question "is the accumulation window complete?" so callers can
        log/branch safely. The counters move only via advance().
        """
        return self.samples_in_window >= self.effective_batch

    def close_window(self) -> int:
        """Reset the window after an optimizer step. Returns samples closed."""
        n = self.samples_in_window
        self.step_in_accum = 0
        self.samples_in_window = 0
        return n

    def restart_window(self) -> None:
        """Window restart (OOM recovery / precision fallback): forget partial
        progress WITHOUT touching samples_done_epoch net - the samples are
        subtracted here because the replay will re-add them exactly once."""
        self.samples_done_epoch -= self.samples_in_window
        self.step_in_accum = 0
        self.samples_in_window = 0


def is_cuda_oom(exc: BaseException) -> bool:
    torch = require_torch()
    if isinstance(exc, torch.cuda.OutOfMemoryError):
        return True
    return isinstance(exc, RuntimeError) and "out of memory" in str(exc).lower()


class BatchOptimizer:
    def __init__(self, config, log_dir=None):
        from ..monitoring.logger import get_logger

        self.config = config
        self._log = get_logger("batch", log_dir=log_dir, console=False)
        self.enabled = config.get("memory.automatic_batch_size", True)
        self.halving = config.get("memory.oom_retry_halving", True)
        self.grad_accum = config.get("training.gradient_accumulation", True)
        self.headroom = config.get("memory.vram_headroom_mb", 1024.0)
        self.per_gpu_probe = config.get("memory.per_gpu_probe", True)
        # threads the documented empty_cache gate into the probe (the schema
        # consumer for memory.probe_empty_cache is satisfied here)
        self.probe_empty_cache = config.get("memory.probe_empty_cache", True)

    # -- initial sizing -------------------------------------------------------
    def initial_state(self, model, example_input, requested_batch: int,
                      device_indices: Optional[Sequence[int]] = None,
                      replicas: int = 1) -> BatchState:
        """Choose a safe starting micro-batch (measured probe when possible).

        V4: `device_indices` are probed INDEPENDENTLY (per-card VRAM is never
        pooled). For a multi-GPU data-parallel strategy the per-card batch is
        micro/replicas, so the total safe micro-batch is the most constrained
        card's capacity times the number of replicas.
        """
        if not self.enabled:
            return BatchState(requested_batch, requested_batch,
                              target_effective=requested_batch)

        indices = list(device_indices) if device_indices else [0]
        if not self.per_gpu_probe:
            indices = indices[:1]

        per_card: dict = {}
        method = "probe"
        measured_any = False
        for idx in indices:
            try:
                est: BatchSizeEstimate = estimate_batch_memory(
                    model, example_input,
                    device_index=idx,
                    headroom_mb=self.headroom,
                    allow_empty_cache=self.probe_empty_cache,
                )
                per_card[idx] = est
                measured_any = measured_any or est.measured
                method = est.method
            except Exception as exc:  # noqa: BLE001
                self._log.warning("batch probe on cuda:%d unavailable (%s)", idx, exc)

        if not per_card:
            self._log.warning("batch probing unavailable; using requested %d",
                              requested_batch)
            return BatchState(requested_batch, requested_batch)

        # the most constrained card decides the per-replica batch
        per_replica_cap = min(est.batch_size for est in per_card.values())
        total_cap = per_replica_cap * max(1, replicas)
        safe = min(requested_batch, total_cap)
        safe = max(1, safe)
        accum = max(1, (requested_batch + safe - 1) // safe) if self.grad_accum else 1
        target = requested_batch if self.grad_accum else safe
        state = BatchState(requested_batch, safe, accum,
                           target_effective=target)
        if safe != requested_batch:
            card_note = ", ".join(
                f"cuda:{i}: {e.batch_size}" for i, e in sorted(per_card.items())
            )
            state.adjustments.append(BatchAdjustment(
                __import__("time").time(), requested_batch, safe, accum,
                f"initial safe size from {method} "
                f"({'measured' if measured_any else 'estimate'}; per-card safe "
                f"batches [{card_note}] x {max(1, replicas)} replica(s))",
                measured_any,
            ))
        self._log.info(
            "batch state: micro=%d accum=%d effective=%d "
            "(per-card safe=[%s], replicas=%d, method=%s)",
            safe, accum, safe * accum,
            ", ".join(f"cuda:{i}={e.batch_size}" for i, e in sorted(per_card.items())),
            max(1, replicas), method,
        )
        return state

    # -- OOM reaction ----------------------------------------------------------
    def on_oom(self, state: BatchState, detail: str = "") -> Optional[BatchState]:
        """OOM recovery: halve micro-batch, double accumulation.

        Returns a NEW state (original preserved for reporting), or None when
        recovery is impossible/disabled (caller must re-raise).

        V4 NOTE: this method only computes the new sizing. It does NOT touch
        gradients or re-run anything - the session's recovery procedure
        (window replay) is what guarantees no sample is lost or duplicated.
        """
        import time

        if not self.enabled or not self.halving:
            return None
        if state.micro_batch <= 1:
            return None
        new_micro = max(1, state.micro_batch // 2)
        # keep the effective batch EXACTLY the same (the window target is
        # carried over; only the nominal micro/accum split changes)
        if self.grad_accum:
            accum = max(1, (state.effective_batch + new_micro - 1) // new_micro)
        else:
            accum = 1
        new_state = BatchState(
            state.requested_batch, new_micro, accum,
            target_effective=state.effective_batch,
            samples_done_epoch=state.samples_done_epoch,
            adjustments=list(state.adjustments),
        )
        new_state.adjustments.append(BatchAdjustment(
            time.time(), state.micro_batch, new_micro, accum,
            f"CUDA OOM recovery{' - ' + detail if detail else ''}: halved micro-batch, "
            f"effective batch preserved at {new_micro * accum}",
            True,
        ))
        self._log.warning("OOM recovery: micro %d -> %d, accum %d -> %d (effective %d)",
                          state.micro_batch, new_micro, state.accum_steps, accum,
                          new_micro * accum)
        return new_state