"""Training performance counters. The PRIMARY success metrics of this project (per the engineering brief) are: * images per second * average iteration time GPU utilization percentage is recorded as a secondary diagnostic only - high usage with low images/sec indicates a bottleneck, not success. All CUDA timing uses events / synchronize ONLY at measurement boundaries so the measured loop itself is not distorted. Phase timings (data, forward, backward, optimizer) are recorded per iteration and aggregated as mean / min / max / p95. """ from __future__ import annotations import math import statistics import time from collections import deque from dataclasses import dataclass, field from typing import Deque, List, Optional from .logger import get_logger @dataclass class IterationTimings: """Seconds for one training iteration, split by phase. None = not measured.""" data: Optional[float] = None forward: Optional[float] = None backward: Optional[float] = None optimizer: Optional[float] = None total: Optional[float] = None images: int = 0 loss: Optional[float] = None @dataclass class _Stats: mean: float min: float max: float p95: float std: float count: int def _summarize(values: List[float]) -> Optional[_Stats]: vals = [v for v in values if v is not None and math.isfinite(v)] if not vals: return None sv = sorted(vals) p95 = sv[min(len(sv) - 1, max(0, math.ceil(0.95 * len(sv)) - 1))] return _Stats( mean=statistics.fmean(vals), min=sv[0], max=sv[-1], p95=p95, std=statistics.pstdev(vals) if len(vals) > 1 else 0.0, count=len(vals), ) class PerformanceMonitor: """Accumulates iteration records and derives images/sec + iteration stats. The monitor is deliberately passive: it never touches the training loop. The session/interceptor push records in; reports are pulled out at the end. Warmup exclusion uses an absolute counter (not deque slicing) so long runs past the deque horizon cannot resurrect warmup records. """ def __init__(self, warmup: int = 10, history: int = 10000, log_dir=None): self._log = get_logger("perf", log_dir=log_dir, console=False) self.warmup = warmup self.records: Deque[IterationTimings] = deque(maxlen=history) self._seen_total = 0 # every record ever pushed self._seen_measured = 0 # records after warmup (never decremented) self._measured_images = 0 self.started_at = time.time() self.total_images = 0 self.oom_events = 0 self.precision_fallbacks = 0 self._first_loss: Optional[float] = None self._last_loss: Optional[float] = None # -- recording ---------------------------------------------------------- def record(self, t: IterationTimings) -> None: self.records.append(t) self.total_images += t.images self._seen_total += 1 if self._seen_total > self.warmup: self._seen_measured += 1 self._measured_images += t.images if t.loss is not None and math.isfinite(t.loss): if self._first_loss is None: self._first_loss = t.loss self._last_loss = t.loss def note_oom(self) -> None: self.oom_events += 1 def note_precision_fallback(self) -> None: self.precision_fallbacks += 1 # -- derived metrics ----------------------------------------------------- @staticmethod def _phase(name: str, recs: List[IterationTimings]) -> Optional[dict]: st = _summarize([getattr(r, name) for r in recs]) if st is None: return None return {"mean_s": st.mean, "min_s": st.min, "max_s": st.max, "p95_s": st.p95, "std_s": st.std, "count": st.count} def summary(self) -> dict: if self._seen_measured == 0: return { "status": "insufficient_data", "warmup_iterations": self.warmup, "recorded_iterations": self._seen_total, "note": "increase measure_iterations or wait for training to run", } # measured set = the most recent _seen_measured records, which after # deque rollover stays correct (absolute counter, not slicing) measured = [r for r in self.records][-self._seen_measured:] if self._seen_measured <= len(self.records) else list(self.records) total_times = [r.total for r in measured if r.total is not None] tot = _summarize(total_times) wall = time.time() - self.started_at # images/sec from measured loop time (excludes monitor overhead). # Numerator counts ONLY records that also contribute to loop_time # (records without a total would inflate the rate). loop_time = sum(t for t in total_times if t is not None) loop_images = sum(r.images for r in measured if r.total is not None) ips_loop = (loop_images / loop_time) if loop_time > 0 else None out = { "status": "ok", "iterations_measured": self._seen_measured, "images_total": self.total_images, "images_per_second": ips_loop, "images_per_second_wall": (self.total_images / wall) if wall > 0 else None, "iteration_time_s": ({"mean_s": tot.mean, "min_s": tot.min, "max_s": tot.max, "p95_s": tot.p95, "std_s": tot.std, "count": tot.count} if tot else None), "phases": { "data": self._phase("data", measured), "forward": self._phase("forward", measured), "backward": self._phase("backward", measured), "optimizer": self._phase("optimizer", measured), }, "oom_events": self.oom_events, "precision_fallbacks": self.precision_fallbacks, "loss_first": self._first_loss, "loss_last": self._last_loss, } # data starvation ratio - identifies CPU/dataloader bottleneck ph = out["phases"] if ph["data"] and ph["data"]["mean_s"] > 0 and out["iteration_time_s"]: out["data_wait_ratio"] = ph["data"]["mean_s"] / out["iteration_time_s"]["mean_s"] return out