Download scale/script/smoke_worker.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 1.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/smoke_worker.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/smoke_worker.py
-
curl -L -o smoke_worker.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/smoke_worker.py
1.5 kB
| """Full normal training steps, with separate capacity telemetry and no huge smoke checkpoint.""" | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import time | |
| import torch | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path[:0] = [str(ROOT), str(ROOT / "src")] | |
| import train_vlm as normal | |
| original_step = normal.PosttrainV1VlmTrainer.training_step | |
| def measured_step(self, batch): | |
| torch.cuda.synchronize() | |
| torch.cuda.reset_peak_memory_stats() | |
| started = time.monotonic() | |
| result = original_step(self, batch) | |
| torch.cuda.synchronize() | |
| logs = result[1] | |
| info = {"event": "smoke_step", "step": self._data_batches, | |
| "seconds": time.monotonic() - started, | |
| "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30, | |
| "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, | |
| "loss": self.train_loss_tracker, | |
| "metrics": {k: float(v) for k, v in logs.items()}} | |
| if not all(torch.isfinite(torch.tensor(v)) for v in info["metrics"].values()): | |
| raise FloatingPointError("Nonfinite smoke training metric") | |
| print(json.dumps(info), flush=True) | |
| return result | |
| def skip_smoke_checkpoint(self, global_step): | |
| print(json.dumps({"event": "smoke_checkpoint_intentionally_skipped", "step": global_step}), flush=True) | |
| return None | |
| normal.PosttrainV1VlmTrainer.training_step = measured_step | |
| normal.PosttrainV1VlmTrainer.save_checkpoint = skip_smoke_checkpoint | |
| if __name__ == "__main__": | |
| normal.main() | |