"""Full normal training steps, with separate capacity telemetry and no huge smoke checkpoint.""" import json from pathlib import Path import sys import time import torch ROOT = Path(__file__).resolve().parents[2] sys.path[:0] = [str(ROOT), str(ROOT / "src")] import train_vlm as normal original_step = normal.PosttrainV1VlmTrainer.training_step def measured_step(self, batch): torch.cuda.synchronize() torch.cuda.reset_peak_memory_stats() started = time.monotonic() result = original_step(self, batch) torch.cuda.synchronize() logs = result[1] info = {"event": "smoke_step", "step": self._data_batches, "seconds": time.monotonic() - started, "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30, "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, "loss": self.train_loss_tracker, "metrics": {k: float(v) for k, v in logs.items()}} if not all(torch.isfinite(torch.tensor(v)) for v in info["metrics"].values()): raise FloatingPointError("Nonfinite smoke training metric") print(json.dumps(info), flush=True) return result def skip_smoke_checkpoint(self, global_step): print(json.dumps({"event": "smoke_checkpoint_intentionally_skipped", "step": global_step}), flush=True) return None normal.PosttrainV1VlmTrainer.training_step = measured_step normal.PosttrainV1VlmTrainer.save_checkpoint = skip_smoke_checkpoint if __name__ == "__main__": normal.main()