code-Movens-Zero_train / scale /script /smoke_worker.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
1.5 kB
"""Full normal training steps, with separate capacity telemetry and no huge smoke checkpoint."""
import json
from pathlib import Path
import sys
import time
import torch
ROOT = Path(__file__).resolve().parents[2]
sys.path[:0] = [str(ROOT), str(ROOT / "src")]
import train_vlm as normal
original_step = normal.PosttrainV1VlmTrainer.training_step
def measured_step(self, batch):
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
started = time.monotonic()
result = original_step(self, batch)
torch.cuda.synchronize()
logs = result[1]
info = {"event": "smoke_step", "step": self._data_batches,
"seconds": time.monotonic() - started,
"peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30,
"peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30,
"loss": self.train_loss_tracker,
"metrics": {k: float(v) for k, v in logs.items()}}
if not all(torch.isfinite(torch.tensor(v)) for v in info["metrics"].values()):
raise FloatingPointError("Nonfinite smoke training metric")
print(json.dumps(info), flush=True)
return result
def skip_smoke_checkpoint(self, global_step):
print(json.dumps({"event": "smoke_checkpoint_intentionally_skipped", "step": global_step}), flush=True)
return None
normal.PosttrainV1VlmTrainer.training_step = measured_step
normal.PosttrainV1VlmTrainer.save_checkpoint = skip_smoke_checkpoint
if __name__ == "__main__":
normal.main()