Spaces:
Running
Running
Download lpu/monitoring/performance_monitor.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 6.35 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/monitoring/performance_monitor.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/monitoring/performance_monitor.py
-
curl -L -o performance_monitor.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/monitoring/performance_monitor.py
6.35 kB
| """Training performance counters. | |
| The PRIMARY success metrics of this project (per the engineering brief) are: | |
| * images per second | |
| * average iteration time | |
| GPU utilization percentage is recorded as a secondary diagnostic only - high | |
| usage with low images/sec indicates a bottleneck, not success. | |
| All CUDA timing uses events / synchronize ONLY at measurement boundaries so | |
| the measured loop itself is not distorted. Phase timings (data, forward, | |
| backward, optimizer) are recorded per iteration and aggregated as | |
| mean / min / max / p95. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import statistics | |
| import time | |
| from collections import deque | |
| from dataclasses import dataclass, field | |
| from typing import Deque, List, Optional | |
| from .logger import get_logger | |
| class IterationTimings: | |
| """Seconds for one training iteration, split by phase. None = not measured.""" | |
| data: Optional[float] = None | |
| forward: Optional[float] = None | |
| backward: Optional[float] = None | |
| optimizer: Optional[float] = None | |
| total: Optional[float] = None | |
| images: int = 0 | |
| loss: Optional[float] = None | |
| class _Stats: | |
| mean: float | |
| min: float | |
| max: float | |
| p95: float | |
| std: float | |
| count: int | |
| def _summarize(values: List[float]) -> Optional[_Stats]: | |
| vals = [v for v in values if v is not None and math.isfinite(v)] | |
| if not vals: | |
| return None | |
| sv = sorted(vals) | |
| p95 = sv[min(len(sv) - 1, max(0, math.ceil(0.95 * len(sv)) - 1))] | |
| return _Stats( | |
| mean=statistics.fmean(vals), | |
| min=sv[0], | |
| max=sv[-1], | |
| p95=p95, | |
| std=statistics.pstdev(vals) if len(vals) > 1 else 0.0, | |
| count=len(vals), | |
| ) | |
| class PerformanceMonitor: | |
| """Accumulates iteration records and derives images/sec + iteration stats. | |
| The monitor is deliberately passive: it never touches the training loop. | |
| The session/interceptor push records in; reports are pulled out at the end. | |
| Warmup exclusion uses an absolute counter (not deque slicing) so long runs | |
| past the deque horizon cannot resurrect warmup records. | |
| """ | |
| def __init__(self, warmup: int = 10, history: int = 10000, log_dir=None): | |
| self._log = get_logger("perf", log_dir=log_dir, console=False) | |
| self.warmup = warmup | |
| self.records: Deque[IterationTimings] = deque(maxlen=history) | |
| self._seen_total = 0 # every record ever pushed | |
| self._seen_measured = 0 # records after warmup (never decremented) | |
| self._measured_images = 0 | |
| self.started_at = time.time() | |
| self.total_images = 0 | |
| self.oom_events = 0 | |
| self.precision_fallbacks = 0 | |
| self._first_loss: Optional[float] = None | |
| self._last_loss: Optional[float] = None | |
| # -- recording ---------------------------------------------------------- | |
| def record(self, t: IterationTimings) -> None: | |
| self.records.append(t) | |
| self.total_images += t.images | |
| self._seen_total += 1 | |
| if self._seen_total > self.warmup: | |
| self._seen_measured += 1 | |
| self._measured_images += t.images | |
| if t.loss is not None and math.isfinite(t.loss): | |
| if self._first_loss is None: | |
| self._first_loss = t.loss | |
| self._last_loss = t.loss | |
| def note_oom(self) -> None: | |
| self.oom_events += 1 | |
| def note_precision_fallback(self) -> None: | |
| self.precision_fallbacks += 1 | |
| # -- derived metrics ----------------------------------------------------- | |
| def _phase(name: str, recs: List[IterationTimings]) -> Optional[dict]: | |
| st = _summarize([getattr(r, name) for r in recs]) | |
| if st is None: | |
| return None | |
| return {"mean_s": st.mean, "min_s": st.min, "max_s": st.max, | |
| "p95_s": st.p95, "std_s": st.std, "count": st.count} | |
| def summary(self) -> dict: | |
| if self._seen_measured == 0: | |
| return { | |
| "status": "insufficient_data", | |
| "warmup_iterations": self.warmup, | |
| "recorded_iterations": self._seen_total, | |
| "note": "increase measure_iterations or wait for training to run", | |
| } | |
| # measured set = the most recent _seen_measured records, which after | |
| # deque rollover stays correct (absolute counter, not slicing) | |
| measured = [r for r in self.records][-self._seen_measured:] if self._seen_measured <= len(self.records) else list(self.records) | |
| total_times = [r.total for r in measured if r.total is not None] | |
| tot = _summarize(total_times) | |
| wall = time.time() - self.started_at | |
| # images/sec from measured loop time (excludes monitor overhead). | |
| # Numerator counts ONLY records that also contribute to loop_time | |
| # (records without a total would inflate the rate). | |
| loop_time = sum(t for t in total_times if t is not None) | |
| loop_images = sum(r.images for r in measured if r.total is not None) | |
| ips_loop = (loop_images / loop_time) if loop_time > 0 else None | |
| out = { | |
| "status": "ok", | |
| "iterations_measured": self._seen_measured, | |
| "images_total": self.total_images, | |
| "images_per_second": ips_loop, | |
| "images_per_second_wall": (self.total_images / wall) if wall > 0 else None, | |
| "iteration_time_s": ({"mean_s": tot.mean, "min_s": tot.min, "max_s": tot.max, | |
| "p95_s": tot.p95, "std_s": tot.std, "count": tot.count} | |
| if tot else None), | |
| "phases": { | |
| "data": self._phase("data", measured), | |
| "forward": self._phase("forward", measured), | |
| "backward": self._phase("backward", measured), | |
| "optimizer": self._phase("optimizer", measured), | |
| }, | |
| "oom_events": self.oom_events, | |
| "precision_fallbacks": self.precision_fallbacks, | |
| "loss_first": self._first_loss, | |
| "loss_last": self._last_loss, | |
| } | |
| # data starvation ratio - identifies CPU/dataloader bottleneck | |
| ph = out["phases"] | |
| if ph["data"] and ph["data"]["mean_s"] > 0 and out["iteration_time_s"]: | |
| out["data_wait_ratio"] = ph["data"]["mean_s"] / out["iteration_time_s"]["mean_s"] | |
| return out | |