"""Feed-forward variants: a dense MLP and a Mixture-of-Experts drop-in. Both classes have identical signatures -- ``(B, N_ctx, D_model) -> (B, N_ctx, D_model)`` -- so the transformer block picks one and is otherwise unaware of which it got. The MoE stashes its auxiliary losses and routing statistics on ``self`` after each forward; the trainer collects them. """ import torch import torch.nn as nn import torch.nn.functional as F from .config import TransformerConfig class Expert(nn.Module): """The 2-layer MLP primitive that every variant is built from.""" def __init__(self, d_model: int, d_ff: int, dropout: float = 0.0): super().__init__() self.fc1 = nn.Linear(d_model, d_ff, bias=False) self.fc2 = nn.Linear(d_ff, d_model, bias=False) self.dropout = nn.Dropout(dropout) def forward(self, x): return self.dropout(self.fc2(F.gelu(self.fc1(x)))) class MLP(nn.Module): """Variant 1: a standard 2-layer MLP feed-forward network.""" def __init__(self, config: TransformerConfig): super().__init__() self.expert = Expert(config.d_model, config.d_ff_expert, config.dropout) # Kept so the trainer can read the same attributes off either class. self.aux_loss = torch.zeros(()) self.z_loss = torch.zeros(()) self.last_counts = None def forward(self, x): # Input shape: (B, N_ctx, D_model); output shape: the same. self.aux_loss = x.new_zeros(()) self.z_loss = x.new_zeros(()) return self.expert(x) class MoE(nn.Module): """Variants 2-5: top-k routed experts, optionally with a shared expert. Routing is per token. Every token is sent to ``n_active`` of the ``n_experts`` routed experts, chosen by a linear router, and the expert outputs are combined with the renormalised router probabilities. The ``n_shared`` experts (0 or 1 here) run on every token unconditionally and are not routed, which is the DeepSeek-style "shared expert" of variant 4: it gives every token a common pathway so the routed experts are free to specialise instead of all having to relearn generic features. """ def __init__(self, config: TransformerConfig): super().__init__() self.n_experts = config.n_experts self.n_active = config.n_active self.n_shared = config.n_shared self.aux_coef = config.router_aux_coef self.z_coef = config.router_z_coef self.noise_std = config.router_noise d_ff_e = config.d_ff_expert self.experts = nn.ModuleList( [Expert(config.d_model, d_ff_e, config.dropout) for _ in range(config.n_experts)]) self.shared = nn.ModuleList( [Expert(config.d_model, d_ff_e, config.dropout) for _ in range(config.n_shared)]) self.router = nn.Linear(config.d_model, config.n_experts, bias=False) self.aux_loss = torch.zeros(()) self.z_loss = torch.zeros(()) self.last_counts = None # (n_experts,) tokens routed, for analysis self.last_assign = None # (B, N, k) expert ids, for the heatmap def forward(self, x): # Input shape: (B, N_ctx, D_model); output shape: the same. B, N, D = x.shape flat = x.reshape(-1, D) # (T, D) logits = self.router(flat) # (T, E) if self.training and self.noise_std > 0: logits = logits + torch.randn_like(logits) * self.noise_std logits = logits.float() probs = logits.softmax(dim=-1) # (T, E) topk_p, topk_i = probs.topk(self.n_active, dim=-1) # (T, k) # Renormalise so the k gates sum to 1 -- keeps the block's output scale # independent of k, which matters when comparing top-1 against top-2. gates = (topk_p / topk_p.sum(dim=-1, keepdim=True)).to(x.dtype) out = torch.zeros_like(flat) counts = torch.zeros(self.n_experts, device=x.device, dtype=torch.long) for e, expert in enumerate(self.experts): # Tokens for which expert e appears anywhere in the top-k. sel, slot = (topk_i == e).nonzero(as_tuple=True) if sel.numel() == 0: continue counts[e] = sel.numel() out.index_add_(0, sel, expert(flat[sel]) * gates[sel, slot].unsqueeze(-1)) for expert in self.shared: out = out + expert(flat) self._compute_aux(logits, probs, topk_i) self.last_counts = counts self.last_assign = topk_i.view(B, N, self.n_active) return out.view(B, N, D) def _compute_aux(self, logits, probs, topk_i): """Switch-Transformer load balancing plus a router z-loss. ``aux`` is E * sum_i f_i * P_i, where f_i is expert i's share of the dispatch slots and P_i the mean router probability for it. It is minimised when both are uniform, so it pushes back against the rich-get-richer collapse where one expert wins every token. The z-loss penalises large router logits, which keeps the top-k selection from turning brittle once the logits start drifting apart. f is divided by k as well as by the token count, so it sums to 1 for any k and the loss bottoms out at 1.0 whatever the value of k. Without that, top-2 would carry twice the balancing pressure of top-1 at the same coefficient and the ablation would confound two changes. """ E = self.n_experts mask = torch.zeros_like(probs) mask.scatter_(1, topk_i, 1.0) f = mask.sum(dim=0) / mask.sum().clamp_min(1.0) # dispatch share P = probs.mean(dim=0) # mean gate mass per expert self.aux_loss = self.aux_coef * E * (f * P).sum() self.z_loss = self.z_coef * torch.logsumexp(logits, dim=-1).pow(2).mean() def build_ffn(config: TransformerConfig) -> nn.Module: """The drop-in switch used by the transformer block.""" return MoE(config) if config.use_moe else MLP(config)