Spaces:
Running
Running
File size: 6,354 Bytes
e59bd9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | """Training performance counters.
The PRIMARY success metrics of this project (per the engineering brief) are:
* images per second
* average iteration time
GPU utilization percentage is recorded as a secondary diagnostic only - high
usage with low images/sec indicates a bottleneck, not success.
All CUDA timing uses events / synchronize ONLY at measurement boundaries so
the measured loop itself is not distorted. Phase timings (data, forward,
backward, optimizer) are recorded per iteration and aggregated as
mean / min / max / p95.
"""
from __future__ import annotations
import math
import statistics
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Deque, List, Optional
from .logger import get_logger
@dataclass
class IterationTimings:
"""Seconds for one training iteration, split by phase. None = not measured."""
data: Optional[float] = None
forward: Optional[float] = None
backward: Optional[float] = None
optimizer: Optional[float] = None
total: Optional[float] = None
images: int = 0
loss: Optional[float] = None
@dataclass
class _Stats:
mean: float
min: float
max: float
p95: float
std: float
count: int
def _summarize(values: List[float]) -> Optional[_Stats]:
vals = [v for v in values if v is not None and math.isfinite(v)]
if not vals:
return None
sv = sorted(vals)
p95 = sv[min(len(sv) - 1, max(0, math.ceil(0.95 * len(sv)) - 1))]
return _Stats(
mean=statistics.fmean(vals),
min=sv[0],
max=sv[-1],
p95=p95,
std=statistics.pstdev(vals) if len(vals) > 1 else 0.0,
count=len(vals),
)
class PerformanceMonitor:
"""Accumulates iteration records and derives images/sec + iteration stats.
The monitor is deliberately passive: it never touches the training loop.
The session/interceptor push records in; reports are pulled out at the end.
Warmup exclusion uses an absolute counter (not deque slicing) so long runs
past the deque horizon cannot resurrect warmup records.
"""
def __init__(self, warmup: int = 10, history: int = 10000, log_dir=None):
self._log = get_logger("perf", log_dir=log_dir, console=False)
self.warmup = warmup
self.records: Deque[IterationTimings] = deque(maxlen=history)
self._seen_total = 0 # every record ever pushed
self._seen_measured = 0 # records after warmup (never decremented)
self._measured_images = 0
self.started_at = time.time()
self.total_images = 0
self.oom_events = 0
self.precision_fallbacks = 0
self._first_loss: Optional[float] = None
self._last_loss: Optional[float] = None
# -- recording ----------------------------------------------------------
def record(self, t: IterationTimings) -> None:
self.records.append(t)
self.total_images += t.images
self._seen_total += 1
if self._seen_total > self.warmup:
self._seen_measured += 1
self._measured_images += t.images
if t.loss is not None and math.isfinite(t.loss):
if self._first_loss is None:
self._first_loss = t.loss
self._last_loss = t.loss
def note_oom(self) -> None:
self.oom_events += 1
def note_precision_fallback(self) -> None:
self.precision_fallbacks += 1
# -- derived metrics -----------------------------------------------------
@staticmethod
def _phase(name: str, recs: List[IterationTimings]) -> Optional[dict]:
st = _summarize([getattr(r, name) for r in recs])
if st is None:
return None
return {"mean_s": st.mean, "min_s": st.min, "max_s": st.max,
"p95_s": st.p95, "std_s": st.std, "count": st.count}
def summary(self) -> dict:
if self._seen_measured == 0:
return {
"status": "insufficient_data",
"warmup_iterations": self.warmup,
"recorded_iterations": self._seen_total,
"note": "increase measure_iterations or wait for training to run",
}
# measured set = the most recent _seen_measured records, which after
# deque rollover stays correct (absolute counter, not slicing)
measured = [r for r in self.records][-self._seen_measured:] if self._seen_measured <= len(self.records) else list(self.records)
total_times = [r.total for r in measured if r.total is not None]
tot = _summarize(total_times)
wall = time.time() - self.started_at
# images/sec from measured loop time (excludes monitor overhead).
# Numerator counts ONLY records that also contribute to loop_time
# (records without a total would inflate the rate).
loop_time = sum(t for t in total_times if t is not None)
loop_images = sum(r.images for r in measured if r.total is not None)
ips_loop = (loop_images / loop_time) if loop_time > 0 else None
out = {
"status": "ok",
"iterations_measured": self._seen_measured,
"images_total": self.total_images,
"images_per_second": ips_loop,
"images_per_second_wall": (self.total_images / wall) if wall > 0 else None,
"iteration_time_s": ({"mean_s": tot.mean, "min_s": tot.min, "max_s": tot.max,
"p95_s": tot.p95, "std_s": tot.std, "count": tot.count}
if tot else None),
"phases": {
"data": self._phase("data", measured),
"forward": self._phase("forward", measured),
"backward": self._phase("backward", measured),
"optimizer": self._phase("optimizer", measured),
},
"oom_events": self.oom_events,
"precision_fallbacks": self.precision_fallbacks,
"loss_first": self._first_loss,
"loss_last": self._last_loss,
}
# data starvation ratio - identifies CPU/dataloader bottleneck
ph = out["phases"]
if ph["data"] and ph["data"]["mean_s"] > 0 and out["iteration_time_s"]:
out["data_wait_ratio"] = ph["data"]["mean_s"] / out["iteration_time_s"]["mean_s"]
return out
|