sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
6.15 kB
"""Feed-forward variants: a dense MLP and a Mixture-of-Experts drop-in.
Both classes have identical signatures -- ``(B, N_ctx, D_model) ->
(B, N_ctx, D_model)`` -- so the transformer block picks one and is otherwise
unaware of which it got. The MoE stashes its auxiliary losses and routing
statistics on ``self`` after each forward; the trainer collects them.
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from .config import TransformerConfig
class Expert(nn.Module):
"""The 2-layer MLP primitive that every variant is built from."""
def __init__(self, d_model: int, d_ff: int, dropout: float = 0.0):
super().__init__()
self.fc1 = nn.Linear(d_model, d_ff, bias=False)
self.fc2 = nn.Linear(d_ff, d_model, bias=False)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
return self.dropout(self.fc2(F.gelu(self.fc1(x))))
class MLP(nn.Module):
"""Variant 1: a standard 2-layer MLP feed-forward network."""
def __init__(self, config: TransformerConfig):
super().__init__()
self.expert = Expert(config.d_model, config.d_ff_expert, config.dropout)
# Kept so the trainer can read the same attributes off either class.
self.aux_loss = torch.zeros(())
self.z_loss = torch.zeros(())
self.last_counts = None
def forward(self, x):
# Input shape: (B, N_ctx, D_model); output shape: the same.
self.aux_loss = x.new_zeros(())
self.z_loss = x.new_zeros(())
return self.expert(x)
class MoE(nn.Module):
"""Variants 2-5: top-k routed experts, optionally with a shared expert.
Routing is per token. Every token is sent to ``n_active`` of the
``n_experts`` routed experts, chosen by a linear router, and the expert
outputs are combined with the renormalised router probabilities. The
``n_shared`` experts (0 or 1 here) run on every token unconditionally and
are not routed, which is the DeepSeek-style "shared expert" of variant 4:
it gives every token a common pathway so the routed experts are free to
specialise instead of all having to relearn generic features.
"""
def __init__(self, config: TransformerConfig):
super().__init__()
self.n_experts = config.n_experts
self.n_active = config.n_active
self.n_shared = config.n_shared
self.aux_coef = config.router_aux_coef
self.z_coef = config.router_z_coef
self.noise_std = config.router_noise
d_ff_e = config.d_ff_expert
self.experts = nn.ModuleList(
[Expert(config.d_model, d_ff_e, config.dropout)
for _ in range(config.n_experts)])
self.shared = nn.ModuleList(
[Expert(config.d_model, d_ff_e, config.dropout)
for _ in range(config.n_shared)])
self.router = nn.Linear(config.d_model, config.n_experts, bias=False)
self.aux_loss = torch.zeros(())
self.z_loss = torch.zeros(())
self.last_counts = None # (n_experts,) tokens routed, for analysis
self.last_assign = None # (B, N, k) expert ids, for the heatmap
def forward(self, x):
# Input shape: (B, N_ctx, D_model); output shape: the same.
B, N, D = x.shape
flat = x.reshape(-1, D) # (T, D)
logits = self.router(flat) # (T, E)
if self.training and self.noise_std > 0:
logits = logits + torch.randn_like(logits) * self.noise_std
logits = logits.float()
probs = logits.softmax(dim=-1) # (T, E)
topk_p, topk_i = probs.topk(self.n_active, dim=-1) # (T, k)
# Renormalise so the k gates sum to 1 -- keeps the block's output scale
# independent of k, which matters when comparing top-1 against top-2.
gates = (topk_p / topk_p.sum(dim=-1, keepdim=True)).to(x.dtype)
out = torch.zeros_like(flat)
counts = torch.zeros(self.n_experts, device=x.device, dtype=torch.long)
for e, expert in enumerate(self.experts):
# Tokens for which expert e appears anywhere in the top-k.
sel, slot = (topk_i == e).nonzero(as_tuple=True)
if sel.numel() == 0:
continue
counts[e] = sel.numel()
out.index_add_(0, sel,
expert(flat[sel]) * gates[sel, slot].unsqueeze(-1))
for expert in self.shared:
out = out + expert(flat)
self._compute_aux(logits, probs, topk_i)
self.last_counts = counts
self.last_assign = topk_i.view(B, N, self.n_active)
return out.view(B, N, D)
def _compute_aux(self, logits, probs, topk_i):
"""Switch-Transformer load balancing plus a router z-loss.
``aux`` is E * sum_i f_i * P_i, where f_i is expert i's share of the
dispatch slots and P_i the mean router probability for it. It is
minimised when both are uniform, so it pushes back against the
rich-get-richer collapse where one expert wins every token. The
z-loss penalises large router logits, which keeps the top-k selection
from turning brittle once the logits start drifting apart.
f is divided by k as well as by the token count, so it sums to 1 for
any k and the loss bottoms out at 1.0 whatever the value of k. Without
that, top-2 would carry twice the balancing pressure of top-1 at the
same coefficient and the ablation would confound two changes.
"""
E = self.n_experts
mask = torch.zeros_like(probs)
mask.scatter_(1, topk_i, 1.0)
f = mask.sum(dim=0) / mask.sum().clamp_min(1.0) # dispatch share
P = probs.mean(dim=0) # mean gate mass per expert
self.aux_loss = self.aux_coef * E * (f * P).sum()
self.z_loss = self.z_coef * torch.logsumexp(logits, dim=-1).pow(2).mean()
def build_ffn(config: TransformerConfig) -> nn.Module:
"""The drop-in switch used by the transformer block."""
return MoE(config) if config.use_moe else MLP(config)