Download src/trainer/grpo_loss.py from Hooshaai/AsyncTensorRLHF: direct link, hf CLI and curl.
- Browser
- Download file 875 Bytes
-
https://huggingface.co/Hooshaai/AsyncTensorRLHF/resolve/main/src/trainer/grpo_loss.py
- Command line
-
hf download hf://Hooshaai/AsyncTensorRLHF/src/trainer/grpo_loss.py
-
curl -L -o grpo_loss.py https://huggingface.co/Hooshaai/AsyncTensorRLHF/resolve/main/src/trainer/grpo_loss.py
875 Bytes
| # GRPO loss implementation (group‑aware advantage normalization) | |
| import torch | |
| from torch.nn import functional as F | |
| def compute_grpo_loss( | |
| policy_log_probs: torch.Tensor, # (B, G, seq_len) where G is group size | |
| old_log_probs: torch.Tensor, # same shape as above | |
| advantages: torch.Tensor, # (B, G, seq_len) – already normalized per group | |
| clip_eps: float = 0.2, | |
| ) -> torch.Tensor: | |
| """Compute GRPO loss. | |
| The loss is applied per‑token, per‑response within each group. | |
| Advantages are assumed to have been computed by standardizing rewards | |
| across the G responses of a prompt (see `GroupAwareReplayBuffer`). | |
| """ | |
| ratio = torch.exp(policy_log_probs - old_log_probs) | |
| clipped = torch.clamp(ratio, 1.0 - clip_eps, 1.0 + clip_eps) | |
| loss = -torch.min(ratio * advantages, clipped * advantages).mean() | |
| return loss | |