TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
7.26 kB
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
def probe_grouped_mm() -> bool:
"""Checks whether torch._grouped_mm is present and actually runs on this
build/device. This is a private, experimental PyTorch API whose signature
has shifted across versions — never trust hasattr() alone, always probe
with a real call. Run this once at startup (see scripts/train.py) to
resolve moe.backend: "auto" to "grouped_mm" or "fallback".
"""
if not hasattr(torch, "_grouped_mm"):
return False
try:
device = "cuda" if torch.cuda.is_available() else "cpu"
a = torch.randn(4, 8, device=device, dtype=torch.bfloat16)
b = torch.randn(2, 8, 8, device=device, dtype=torch.bfloat16)
offs = torch.tensor([2, 4], dtype=torch.int32, device=device)
torch._grouped_mm(a, b, offs=offs)
return True
except Exception:
return False
class ExpertLoadTracker:
"""Running, detached per-expert token counts accumulated across the
grad-accumulation micro-batches of one optimizer step ("global_batch"
load balancing). The training loop must call reset() after each
optimizer step.
"""
def __init__(self, n_layers: int, n_experts: int, device):
self.counts = torch.zeros(n_layers, n_experts, device=device)
def update(self, layer_idx: int, hard_counts: torch.Tensor) -> None:
self.counts[layer_idx] += hard_counts.detach()
def reset(self) -> None:
self.counts.zero_()
class MoEFFN(nn.Module):
"""Dropless top-k MoE FFN (SwiGLU experts) with two interchangeable
compute backends:
- "grouped_mm": torch._grouped_mm fast path.
- "fallback": gather + per-expert-loop. The Python loop runs only in
this forward pass, never in the optimizer — it is NOT
the same constraint as Muon's batched Newton-Schulz.
Experts are stored as stacked nn.Parameter tensors (not an nn.ModuleList)
so both backends and the Muon optimizer's batched per-expert step can
treat them as a single (n_experts, ...) tensor.
"""
def __init__(
self,
d_model: int,
n_experts: int,
top_k: int,
expert_hidden: int,
backend: str,
router_init_std: float,
):
super().__init__()
self.d_model = d_model
self.n_experts = n_experts
self.top_k = top_k
self.backend = backend
self.router = nn.Linear(d_model, n_experts, bias=False)
nn.init.normal_(self.router.weight, std=router_init_std)
# Default std=0.02 init so MoEFFN is sane to use/test standalone;
# Transformer-level init_weights rescales W_down to residual_proj_std
# when this module is used inside a full model.
self.W_gate = nn.Parameter(torch.empty(n_experts, d_model, expert_hidden).normal_(std=0.02))
self.W_up = nn.Parameter(torch.empty(n_experts, d_model, expert_hidden).normal_(std=0.02))
self.W_down = nn.Parameter(torch.empty(n_experts, expert_hidden, d_model).normal_(std=0.02))
def forward(
self,
x: torch.Tensor,
load_tracker: ExpertLoadTracker | None = None,
layer_idx: int | None = None,
):
bsz, seq_len, d = x.shape
x_flat = x.reshape(-1, d)
n_tokens = x_flat.shape[0]
logits = self.router(x_flat) # (N, E), compute in the router's own dtype
probs = F.softmax(logits.float(), dim=-1)
topk_probs, topk_idx = probs.topk(self.top_k, dim=-1)
topk_probs = (topk_probs / topk_probs.sum(dim=-1, keepdim=True)).to(x.dtype)
if self.backend == "grouped_mm":
out_flat = self._forward_grouped_mm(x_flat, topk_idx, topk_probs)
elif self.backend == "fallback":
out_flat = self._forward_fallback(x_flat, topk_idx, topk_probs)
else:
raise ValueError(f"Unknown MoE backend: {self.backend}")
z_loss = torch.logsumexp(logits.float(), dim=-1).pow(2).mean()
flat_expert = topk_idx.reshape(-1)
hard_counts = torch.zeros(self.n_experts, device=x.device, dtype=torch.float32)
hard_counts.scatter_add_(
0, flat_expert, torch.ones_like(flat_expert, dtype=torch.float32)
)
if load_tracker is not None and layer_idx is not None:
load_tracker.update(layer_idx, hard_counts)
running = load_tracker.counts[layer_idx]
f_i = running / running.sum().clamp_min(1.0)
else:
f_i = hard_counts / hard_counts.sum().clamp_min(1.0)
p_i = probs.mean(dim=0) # differentiable, this micro-batch only
lb_loss = self.n_experts * (f_i.detach() * p_i).sum()
return out_flat.view(bsz, seq_len, d), lb_loss, z_loss, hard_counts.detach()
def _forward_fallback(
self, x_flat: torch.Tensor, topk_idx: torch.Tensor, topk_probs: torch.Tensor
) -> torch.Tensor:
n_tokens, d = x_flat.shape
flat_expert = topk_idx.reshape(-1)
flat_token = torch.arange(n_tokens, device=x_flat.device).repeat_interleave(self.top_k)
flat_weight = topk_probs.reshape(-1)
out = torch.zeros_like(x_flat)
for e in range(self.n_experts):
mask = flat_expert == e
if not mask.any():
continue
tok_idx = flat_token[mask]
w = flat_weight[mask].unsqueeze(-1)
xe = x_flat[tok_idx]
h = F.silu(xe @ self.W_gate[e]) * (xe @ self.W_up[e])
down = h @ self.W_down[e]
out.index_add_(0, tok_idx, down * w)
return out
def _forward_grouped_mm(
self, x_flat: torch.Tensor, topk_idx: torch.Tensor, topk_probs: torch.Tensor
) -> torch.Tensor:
n_tokens, d = x_flat.shape
flat_expert = topk_idx.reshape(-1)
flat_token = torch.arange(n_tokens, device=x_flat.device).repeat_interleave(self.top_k)
flat_weight = topk_probs.reshape(-1)
sort_idx = torch.argsort(flat_expert, stable=True)
sorted_expert = flat_expert[sort_idx]
sorted_token = flat_token[sort_idx]
sorted_weight = flat_weight[sort_idx]
counts = torch.bincount(sorted_expert, minlength=self.n_experts)
offs = counts.cumsum(0).to(torch.int32)
# torch._grouped_mm is a private op that torch.autocast does not know
# to cast — it requires bf16/fp16 inputs explicitly, unlike ordinary
# matmul which autocast handles automatically under the training
# loop's autocast context.
gathered_x = x_flat[sorted_token].to(torch.bfloat16)
w_gate = self.W_gate.to(torch.bfloat16)
w_up = self.W_up.to(torch.bfloat16)
w_down = self.W_down.to(torch.bfloat16)
gate = torch._grouped_mm(gathered_x, w_gate, offs=offs)
up = torch._grouped_mm(gathered_x, w_up, offs=offs)
h = F.silu(gate) * up
down = torch._grouped_mm(h, w_down, offs=offs)
down = down * sorted_weight.unsqueeze(-1).to(down.dtype)
out = torch.zeros(n_tokens, d, device=x_flat.device, dtype=down.dtype)
out.index_add_(0, sorted_token, down)
return out.to(x_flat.dtype)