Buckets:
| """Profile one GRPO training step: where does the time go? | |
| Runs train.py unchanged (same arguments), with timers wrapped around each stage of a step. Every timer | |
| synchronises the GPU first, so GPU work is charged to the stage that launched it. | |
| Stages: | |
| rollout our rollout_func: vLLM generation, target calls, judging (includes the weight sync) | |
| weight sync LoRA merged into the base and pushed into vLLM (part of rollout) | |
| vllm sleep/wake freeing and reclaiming vLLM's memory around the rollout (part of rollout) | |
| old-logp forward TRL's extra no-gradient forward over the whole batch (vLLM mismatch correction) | |
| loss forward forward with gradients, per micro-batch | |
| backward backward per micro-batch (re-runs the forward too when gradient checkpointing is on) | |
| optimizer AdamW step on the LoRA weights | |
| Also counts tokens per forward and how many of them are padding. | |
| python profile_train.py --max-steps 1 --prompts-per-step 8 --train-micro-batch 4 --vllm-gpu-frac 0.6 --run-name profile-9b | |
| """ | |
| import collections | |
| import sys | |
| import time | |
| import torch | |
| from accelerate.optimizer import AcceleratedOptimizer | |
| from accelerate import Accelerator | |
| from trl import GRPOTrainer | |
| from trl.generation.vllm_generation import VLLMGeneration | |
| import train | |
| seconds = collections.defaultdict(float) | |
| calls = collections.Counter() | |
| tokens = collections.Counter() # per stage: real tokens and padded positions | |
| def timed(stage, fn): | |
| """Wrap fn so its wall time (after a GPU sync on both ends) adds to seconds[stage].""" | |
| def wrapper(*args, **kwargs): | |
| torch.cuda.synchronize() | |
| start = time.time() | |
| try: | |
| return fn(*args, **kwargs) | |
| finally: | |
| torch.cuda.synchronize() | |
| seconds[stage] += time.time() - start | |
| calls[stage] += 1 | |
| return wrapper | |
| # The log-prob pass runs both without gradients (old log-probs) and with them (the loss); tell them apart. | |
| original_logps = GRPOTrainer._get_per_token_logps_and_entropies | |
| def logps_with_stage(self, model, input_ids, attention_mask, *args, **kwargs): | |
| stage = "loss forward" if torch.is_grad_enabled() else "old-logp forward" | |
| tokens[f"{stage} real"] += int(attention_mask.sum()) | |
| tokens[f"{stage} padded"] += int(attention_mask.numel()) | |
| return timed(stage, original_logps)(self, model, input_ids, attention_mask, *args, **kwargs) | |
| GRPOTrainer._get_per_token_logps_and_entropies = logps_with_stage | |
| GRPOTrainer.training_step = timed("training_step total", GRPOTrainer.training_step) | |
| train.AttackerRollout.__call__ = timed("rollout", train.AttackerRollout.__call__) | |
| VLLMGeneration.sync_weights = timed("weight sync", VLLMGeneration.sync_weights) | |
| Accelerator.backward = timed("backward", Accelerator.backward) | |
| AcceleratedOptimizer.step = timed("optimizer", AcceleratedOptimizer.step) | |
| # vLLM sleep / wake_up are methods of the LLM object TRL creates; patch them when the trainer is built. | |
| original_init = VLLMGeneration.__init__ | |
| def init_and_patch(self, *args, **kwargs): | |
| original_init(self, *args, **kwargs) | |
| if getattr(self, "llm", None) is not None: | |
| self.llm.sleep = timed("vllm sleep", self.llm.sleep) | |
| self.llm.wake_up = timed("vllm wake", self.llm.wake_up) | |
| VLLMGeneration.__init__ = init_and_patch | |
| def report(): | |
| total = seconds["training_step total"] | |
| print("\n========== training step profile ==========") | |
| order = ["rollout", "weight sync", "vllm wake", "vllm sleep", "old-logp forward", "loss forward", "backward", "optimizer"] | |
| for stage in order: | |
| share = f"{100 * seconds[stage] / total:5.1f}%" if total else " - " | |
| print(f"{stage:<18}{seconds[stage]:>9.1f} s {share} ({calls[stage]} calls)") | |
| print(f"{'step total':<18}{total:>9.1f} s") | |
| other = total - seconds["rollout"] - seconds["old-logp forward"] - seconds["loss forward"] - seconds["backward"] - seconds["optimizer"] | |
| print(f"{'other':<18}{other:>9.1f} s (scoring, padding, logging)") | |
| for stage in ("old-logp forward", "loss forward"): | |
| real, padded = tokens[f"{stage} real"], tokens[f"{stage} padded"] | |
| if padded: | |
| print(f"{stage}: {real} real tokens in {padded} positions ({100 * (1 - real / padded):.0f}% padding), " | |
| f"{1000 * seconds[stage] / real:.2f} s per 1k real tokens") | |
| real = tokens["loss forward real"] | |
| if real: | |
| print(f"backward: {1000 * seconds['backward'] / real:.2f} s per 1k real tokens") | |
| if __name__ == "__main__": | |
| try: | |
| train.main(train.parse_args(sys.argv[1:])) | |
| finally: | |
| report() | |
Xet Storage Details
- Size:
- 4.59 kB
- Xet hash:
- 0d79541c209675768dd24fc567eea7d4982a07748e282ce15d985b35b20dceeb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.