Download src/part1/ffn.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 6.15 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part1/ffn.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part1/ffn.py
-
curl -L -o ffn.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part1/ffn.py
6.15 kB
| """Feed-forward variants: a dense MLP and a Mixture-of-Experts drop-in. | |
| Both classes have identical signatures -- ``(B, N_ctx, D_model) -> | |
| (B, N_ctx, D_model)`` -- so the transformer block picks one and is otherwise | |
| unaware of which it got. The MoE stashes its auxiliary losses and routing | |
| statistics on ``self`` after each forward; the trainer collects them. | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .config import TransformerConfig | |
| class Expert(nn.Module): | |
| """The 2-layer MLP primitive that every variant is built from.""" | |
| def __init__(self, d_model: int, d_ff: int, dropout: float = 0.0): | |
| super().__init__() | |
| self.fc1 = nn.Linear(d_model, d_ff, bias=False) | |
| self.fc2 = nn.Linear(d_ff, d_model, bias=False) | |
| self.dropout = nn.Dropout(dropout) | |
| def forward(self, x): | |
| return self.dropout(self.fc2(F.gelu(self.fc1(x)))) | |
| class MLP(nn.Module): | |
| """Variant 1: a standard 2-layer MLP feed-forward network.""" | |
| def __init__(self, config: TransformerConfig): | |
| super().__init__() | |
| self.expert = Expert(config.d_model, config.d_ff_expert, config.dropout) | |
| # Kept so the trainer can read the same attributes off either class. | |
| self.aux_loss = torch.zeros(()) | |
| self.z_loss = torch.zeros(()) | |
| self.last_counts = None | |
| def forward(self, x): | |
| # Input shape: (B, N_ctx, D_model); output shape: the same. | |
| self.aux_loss = x.new_zeros(()) | |
| self.z_loss = x.new_zeros(()) | |
| return self.expert(x) | |
| class MoE(nn.Module): | |
| """Variants 2-5: top-k routed experts, optionally with a shared expert. | |
| Routing is per token. Every token is sent to ``n_active`` of the | |
| ``n_experts`` routed experts, chosen by a linear router, and the expert | |
| outputs are combined with the renormalised router probabilities. The | |
| ``n_shared`` experts (0 or 1 here) run on every token unconditionally and | |
| are not routed, which is the DeepSeek-style "shared expert" of variant 4: | |
| it gives every token a common pathway so the routed experts are free to | |
| specialise instead of all having to relearn generic features. | |
| """ | |
| def __init__(self, config: TransformerConfig): | |
| super().__init__() | |
| self.n_experts = config.n_experts | |
| self.n_active = config.n_active | |
| self.n_shared = config.n_shared | |
| self.aux_coef = config.router_aux_coef | |
| self.z_coef = config.router_z_coef | |
| self.noise_std = config.router_noise | |
| d_ff_e = config.d_ff_expert | |
| self.experts = nn.ModuleList( | |
| [Expert(config.d_model, d_ff_e, config.dropout) | |
| for _ in range(config.n_experts)]) | |
| self.shared = nn.ModuleList( | |
| [Expert(config.d_model, d_ff_e, config.dropout) | |
| for _ in range(config.n_shared)]) | |
| self.router = nn.Linear(config.d_model, config.n_experts, bias=False) | |
| self.aux_loss = torch.zeros(()) | |
| self.z_loss = torch.zeros(()) | |
| self.last_counts = None # (n_experts,) tokens routed, for analysis | |
| self.last_assign = None # (B, N, k) expert ids, for the heatmap | |
| def forward(self, x): | |
| # Input shape: (B, N_ctx, D_model); output shape: the same. | |
| B, N, D = x.shape | |
| flat = x.reshape(-1, D) # (T, D) | |
| logits = self.router(flat) # (T, E) | |
| if self.training and self.noise_std > 0: | |
| logits = logits + torch.randn_like(logits) * self.noise_std | |
| logits = logits.float() | |
| probs = logits.softmax(dim=-1) # (T, E) | |
| topk_p, topk_i = probs.topk(self.n_active, dim=-1) # (T, k) | |
| # Renormalise so the k gates sum to 1 -- keeps the block's output scale | |
| # independent of k, which matters when comparing top-1 against top-2. | |
| gates = (topk_p / topk_p.sum(dim=-1, keepdim=True)).to(x.dtype) | |
| out = torch.zeros_like(flat) | |
| counts = torch.zeros(self.n_experts, device=x.device, dtype=torch.long) | |
| for e, expert in enumerate(self.experts): | |
| # Tokens for which expert e appears anywhere in the top-k. | |
| sel, slot = (topk_i == e).nonzero(as_tuple=True) | |
| if sel.numel() == 0: | |
| continue | |
| counts[e] = sel.numel() | |
| out.index_add_(0, sel, | |
| expert(flat[sel]) * gates[sel, slot].unsqueeze(-1)) | |
| for expert in self.shared: | |
| out = out + expert(flat) | |
| self._compute_aux(logits, probs, topk_i) | |
| self.last_counts = counts | |
| self.last_assign = topk_i.view(B, N, self.n_active) | |
| return out.view(B, N, D) | |
| def _compute_aux(self, logits, probs, topk_i): | |
| """Switch-Transformer load balancing plus a router z-loss. | |
| ``aux`` is E * sum_i f_i * P_i, where f_i is expert i's share of the | |
| dispatch slots and P_i the mean router probability for it. It is | |
| minimised when both are uniform, so it pushes back against the | |
| rich-get-richer collapse where one expert wins every token. The | |
| z-loss penalises large router logits, which keeps the top-k selection | |
| from turning brittle once the logits start drifting apart. | |
| f is divided by k as well as by the token count, so it sums to 1 for | |
| any k and the loss bottoms out at 1.0 whatever the value of k. Without | |
| that, top-2 would carry twice the balancing pressure of top-1 at the | |
| same coefficient and the ablation would confound two changes. | |
| """ | |
| E = self.n_experts | |
| mask = torch.zeros_like(probs) | |
| mask.scatter_(1, topk_i, 1.0) | |
| f = mask.sum(dim=0) / mask.sum().clamp_min(1.0) # dispatch share | |
| P = probs.mean(dim=0) # mean gate mass per expert | |
| self.aux_loss = self.aux_coef * E * (f * P).sum() | |
| self.z_loss = self.z_coef * torch.logsumexp(logits, dim=-1).pow(2).mean() | |
| def build_ffn(config: TransformerConfig) -> nn.Module: | |
| """The drop-in switch used by the transformer block.""" | |
| return MoE(config) if config.use_moe else MLP(config) | |