Virtual-LPU / lpu /monitoring /performance_monitor.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
6.35 kB
"""Training performance counters.
The PRIMARY success metrics of this project (per the engineering brief) are:
* images per second
* average iteration time
GPU utilization percentage is recorded as a secondary diagnostic only - high
usage with low images/sec indicates a bottleneck, not success.
All CUDA timing uses events / synchronize ONLY at measurement boundaries so
the measured loop itself is not distorted. Phase timings (data, forward,
backward, optimizer) are recorded per iteration and aggregated as
mean / min / max / p95.
"""
from __future__ import annotations
import math
import statistics
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Deque, List, Optional
from .logger import get_logger
@dataclass
class IterationTimings:
"""Seconds for one training iteration, split by phase. None = not measured."""
data: Optional[float] = None
forward: Optional[float] = None
backward: Optional[float] = None
optimizer: Optional[float] = None
total: Optional[float] = None
images: int = 0
loss: Optional[float] = None
@dataclass
class _Stats:
mean: float
min: float
max: float
p95: float
std: float
count: int
def _summarize(values: List[float]) -> Optional[_Stats]:
vals = [v for v in values if v is not None and math.isfinite(v)]
if not vals:
return None
sv = sorted(vals)
p95 = sv[min(len(sv) - 1, max(0, math.ceil(0.95 * len(sv)) - 1))]
return _Stats(
mean=statistics.fmean(vals),
min=sv[0],
max=sv[-1],
p95=p95,
std=statistics.pstdev(vals) if len(vals) > 1 else 0.0,
count=len(vals),
)
class PerformanceMonitor:
"""Accumulates iteration records and derives images/sec + iteration stats.
The monitor is deliberately passive: it never touches the training loop.
The session/interceptor push records in; reports are pulled out at the end.
Warmup exclusion uses an absolute counter (not deque slicing) so long runs
past the deque horizon cannot resurrect warmup records.
"""
def __init__(self, warmup: int = 10, history: int = 10000, log_dir=None):
self._log = get_logger("perf", log_dir=log_dir, console=False)
self.warmup = warmup
self.records: Deque[IterationTimings] = deque(maxlen=history)
self._seen_total = 0 # every record ever pushed
self._seen_measured = 0 # records after warmup (never decremented)
self._measured_images = 0
self.started_at = time.time()
self.total_images = 0
self.oom_events = 0
self.precision_fallbacks = 0
self._first_loss: Optional[float] = None
self._last_loss: Optional[float] = None
# -- recording ----------------------------------------------------------
def record(self, t: IterationTimings) -> None:
self.records.append(t)
self.total_images += t.images
self._seen_total += 1
if self._seen_total > self.warmup:
self._seen_measured += 1
self._measured_images += t.images
if t.loss is not None and math.isfinite(t.loss):
if self._first_loss is None:
self._first_loss = t.loss
self._last_loss = t.loss
def note_oom(self) -> None:
self.oom_events += 1
def note_precision_fallback(self) -> None:
self.precision_fallbacks += 1
# -- derived metrics -----------------------------------------------------
@staticmethod
def _phase(name: str, recs: List[IterationTimings]) -> Optional[dict]:
st = _summarize([getattr(r, name) for r in recs])
if st is None:
return None
return {"mean_s": st.mean, "min_s": st.min, "max_s": st.max,
"p95_s": st.p95, "std_s": st.std, "count": st.count}
def summary(self) -> dict:
if self._seen_measured == 0:
return {
"status": "insufficient_data",
"warmup_iterations": self.warmup,
"recorded_iterations": self._seen_total,
"note": "increase measure_iterations or wait for training to run",
}
# measured set = the most recent _seen_measured records, which after
# deque rollover stays correct (absolute counter, not slicing)
measured = [r for r in self.records][-self._seen_measured:] if self._seen_measured <= len(self.records) else list(self.records)
total_times = [r.total for r in measured if r.total is not None]
tot = _summarize(total_times)
wall = time.time() - self.started_at
# images/sec from measured loop time (excludes monitor overhead).
# Numerator counts ONLY records that also contribute to loop_time
# (records without a total would inflate the rate).
loop_time = sum(t for t in total_times if t is not None)
loop_images = sum(r.images for r in measured if r.total is not None)
ips_loop = (loop_images / loop_time) if loop_time > 0 else None
out = {
"status": "ok",
"iterations_measured": self._seen_measured,
"images_total": self.total_images,
"images_per_second": ips_loop,
"images_per_second_wall": (self.total_images / wall) if wall > 0 else None,
"iteration_time_s": ({"mean_s": tot.mean, "min_s": tot.min, "max_s": tot.max,
"p95_s": tot.p95, "std_s": tot.std, "count": tot.count}
if tot else None),
"phases": {
"data": self._phase("data", measured),
"forward": self._phase("forward", measured),
"backward": self._phase("backward", measured),
"optimizer": self._phase("optimizer", measured),
},
"oom_events": self.oom_events,
"precision_fallbacks": self.precision_fallbacks,
"loss_first": self._first_loss,
"loss_last": self._last_loss,
}
# data starvation ratio - identifies CPU/dataloader bottleneck
ph = out["phases"]
if ph["data"] and ph["data"]["mean_s"] > 0 and out["iteration_time_s"]:
out["data_wait_ratio"] = ph["data"]["mean_s"] / out["iteration_time_s"]["mean_s"]
return out