Buckets:

Mercity/red-teaming / 23-10-progress /profile_train.py
Rishik001's picture
download
raw
4.59 kB
"""Profile one GRPO training step: where does the time go?
Runs train.py unchanged (same arguments), with timers wrapped around each stage of a step. Every timer
synchronises the GPU first, so GPU work is charged to the stage that launched it.
Stages:
rollout our rollout_func: vLLM generation, target calls, judging (includes the weight sync)
weight sync LoRA merged into the base and pushed into vLLM (part of rollout)
vllm sleep/wake freeing and reclaiming vLLM's memory around the rollout (part of rollout)
old-logp forward TRL's extra no-gradient forward over the whole batch (vLLM mismatch correction)
loss forward forward with gradients, per micro-batch
backward backward per micro-batch (re-runs the forward too when gradient checkpointing is on)
optimizer AdamW step on the LoRA weights
Also counts tokens per forward and how many of them are padding.
python profile_train.py --max-steps 1 --prompts-per-step 8 --train-micro-batch 4 --vllm-gpu-frac 0.6 --run-name profile-9b
"""
import collections
import sys
import time
import torch
from accelerate.optimizer import AcceleratedOptimizer
from accelerate import Accelerator
from trl import GRPOTrainer
from trl.generation.vllm_generation import VLLMGeneration
import train
seconds = collections.defaultdict(float)
calls = collections.Counter()
tokens = collections.Counter() # per stage: real tokens and padded positions
def timed(stage, fn):
"""Wrap fn so its wall time (after a GPU sync on both ends) adds to seconds[stage]."""
def wrapper(*args, **kwargs):
torch.cuda.synchronize()
start = time.time()
try:
return fn(*args, **kwargs)
finally:
torch.cuda.synchronize()
seconds[stage] += time.time() - start
calls[stage] += 1
return wrapper
# The log-prob pass runs both without gradients (old log-probs) and with them (the loss); tell them apart.
original_logps = GRPOTrainer._get_per_token_logps_and_entropies
def logps_with_stage(self, model, input_ids, attention_mask, *args, **kwargs):
stage = "loss forward" if torch.is_grad_enabled() else "old-logp forward"
tokens[f"{stage} real"] += int(attention_mask.sum())
tokens[f"{stage} padded"] += int(attention_mask.numel())
return timed(stage, original_logps)(self, model, input_ids, attention_mask, *args, **kwargs)
GRPOTrainer._get_per_token_logps_and_entropies = logps_with_stage
GRPOTrainer.training_step = timed("training_step total", GRPOTrainer.training_step)
train.AttackerRollout.__call__ = timed("rollout", train.AttackerRollout.__call__)
VLLMGeneration.sync_weights = timed("weight sync", VLLMGeneration.sync_weights)
Accelerator.backward = timed("backward", Accelerator.backward)
AcceleratedOptimizer.step = timed("optimizer", AcceleratedOptimizer.step)
# vLLM sleep / wake_up are methods of the LLM object TRL creates; patch them when the trainer is built.
original_init = VLLMGeneration.__init__
def init_and_patch(self, *args, **kwargs):
original_init(self, *args, **kwargs)
if getattr(self, "llm", None) is not None:
self.llm.sleep = timed("vllm sleep", self.llm.sleep)
self.llm.wake_up = timed("vllm wake", self.llm.wake_up)
VLLMGeneration.__init__ = init_and_patch
def report():
total = seconds["training_step total"]
print("\n========== training step profile ==========")
order = ["rollout", "weight sync", "vllm wake", "vllm sleep", "old-logp forward", "loss forward", "backward", "optimizer"]
for stage in order:
share = f"{100 * seconds[stage] / total:5.1f}%" if total else " - "
print(f"{stage:<18}{seconds[stage]:>9.1f} s {share} ({calls[stage]} calls)")
print(f"{'step total':<18}{total:>9.1f} s")
other = total - seconds["rollout"] - seconds["old-logp forward"] - seconds["loss forward"] - seconds["backward"] - seconds["optimizer"]
print(f"{'other':<18}{other:>9.1f} s (scoring, padding, logging)")
for stage in ("old-logp forward", "loss forward"):
real, padded = tokens[f"{stage} real"], tokens[f"{stage} padded"]
if padded:
print(f"{stage}: {real} real tokens in {padded} positions ({100 * (1 - real / padded):.0f}% padding), "
f"{1000 * seconds[stage] / real:.2f} s per 1k real tokens")
real = tokens["loss forward real"]
if real:
print(f"backward: {1000 * seconds['backward'] / real:.2f} s per 1k real tokens")
if __name__ == "__main__":
try:
train.main(train.parse_args(sys.argv[1:]))
finally:
report()

Xet Storage Details

Size:
4.59 kB
·
Xet hash:
0d79541c209675768dd24fc567eea7d4982a07748e282ce15d985b35b20dceeb

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.