Download src/model/moe.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 7.26 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/moe.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/moe.py
-
curl -L -o moe.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/moe.py
7.26 kB
| from __future__ import annotations | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| def probe_grouped_mm() -> bool: | |
| """Checks whether torch._grouped_mm is present and actually runs on this | |
| build/device. This is a private, experimental PyTorch API whose signature | |
| has shifted across versions — never trust hasattr() alone, always probe | |
| with a real call. Run this once at startup (see scripts/train.py) to | |
| resolve moe.backend: "auto" to "grouped_mm" or "fallback". | |
| """ | |
| if not hasattr(torch, "_grouped_mm"): | |
| return False | |
| try: | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| a = torch.randn(4, 8, device=device, dtype=torch.bfloat16) | |
| b = torch.randn(2, 8, 8, device=device, dtype=torch.bfloat16) | |
| offs = torch.tensor([2, 4], dtype=torch.int32, device=device) | |
| torch._grouped_mm(a, b, offs=offs) | |
| return True | |
| except Exception: | |
| return False | |
| class ExpertLoadTracker: | |
| """Running, detached per-expert token counts accumulated across the | |
| grad-accumulation micro-batches of one optimizer step ("global_batch" | |
| load balancing). The training loop must call reset() after each | |
| optimizer step. | |
| """ | |
| def __init__(self, n_layers: int, n_experts: int, device): | |
| self.counts = torch.zeros(n_layers, n_experts, device=device) | |
| def update(self, layer_idx: int, hard_counts: torch.Tensor) -> None: | |
| self.counts[layer_idx] += hard_counts.detach() | |
| def reset(self) -> None: | |
| self.counts.zero_() | |
| class MoEFFN(nn.Module): | |
| """Dropless top-k MoE FFN (SwiGLU experts) with two interchangeable | |
| compute backends: | |
| - "grouped_mm": torch._grouped_mm fast path. | |
| - "fallback": gather + per-expert-loop. The Python loop runs only in | |
| this forward pass, never in the optimizer — it is NOT | |
| the same constraint as Muon's batched Newton-Schulz. | |
| Experts are stored as stacked nn.Parameter tensors (not an nn.ModuleList) | |
| so both backends and the Muon optimizer's batched per-expert step can | |
| treat them as a single (n_experts, ...) tensor. | |
| """ | |
| def __init__( | |
| self, | |
| d_model: int, | |
| n_experts: int, | |
| top_k: int, | |
| expert_hidden: int, | |
| backend: str, | |
| router_init_std: float, | |
| ): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.n_experts = n_experts | |
| self.top_k = top_k | |
| self.backend = backend | |
| self.router = nn.Linear(d_model, n_experts, bias=False) | |
| nn.init.normal_(self.router.weight, std=router_init_std) | |
| # Default std=0.02 init so MoEFFN is sane to use/test standalone; | |
| # Transformer-level init_weights rescales W_down to residual_proj_std | |
| # when this module is used inside a full model. | |
| self.W_gate = nn.Parameter(torch.empty(n_experts, d_model, expert_hidden).normal_(std=0.02)) | |
| self.W_up = nn.Parameter(torch.empty(n_experts, d_model, expert_hidden).normal_(std=0.02)) | |
| self.W_down = nn.Parameter(torch.empty(n_experts, expert_hidden, d_model).normal_(std=0.02)) | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| load_tracker: ExpertLoadTracker | None = None, | |
| layer_idx: int | None = None, | |
| ): | |
| bsz, seq_len, d = x.shape | |
| x_flat = x.reshape(-1, d) | |
| n_tokens = x_flat.shape[0] | |
| logits = self.router(x_flat) # (N, E), compute in the router's own dtype | |
| probs = F.softmax(logits.float(), dim=-1) | |
| topk_probs, topk_idx = probs.topk(self.top_k, dim=-1) | |
| topk_probs = (topk_probs / topk_probs.sum(dim=-1, keepdim=True)).to(x.dtype) | |
| if self.backend == "grouped_mm": | |
| out_flat = self._forward_grouped_mm(x_flat, topk_idx, topk_probs) | |
| elif self.backend == "fallback": | |
| out_flat = self._forward_fallback(x_flat, topk_idx, topk_probs) | |
| else: | |
| raise ValueError(f"Unknown MoE backend: {self.backend}") | |
| z_loss = torch.logsumexp(logits.float(), dim=-1).pow(2).mean() | |
| flat_expert = topk_idx.reshape(-1) | |
| hard_counts = torch.zeros(self.n_experts, device=x.device, dtype=torch.float32) | |
| hard_counts.scatter_add_( | |
| 0, flat_expert, torch.ones_like(flat_expert, dtype=torch.float32) | |
| ) | |
| if load_tracker is not None and layer_idx is not None: | |
| load_tracker.update(layer_idx, hard_counts) | |
| running = load_tracker.counts[layer_idx] | |
| f_i = running / running.sum().clamp_min(1.0) | |
| else: | |
| f_i = hard_counts / hard_counts.sum().clamp_min(1.0) | |
| p_i = probs.mean(dim=0) # differentiable, this micro-batch only | |
| lb_loss = self.n_experts * (f_i.detach() * p_i).sum() | |
| return out_flat.view(bsz, seq_len, d), lb_loss, z_loss, hard_counts.detach() | |
| def _forward_fallback( | |
| self, x_flat: torch.Tensor, topk_idx: torch.Tensor, topk_probs: torch.Tensor | |
| ) -> torch.Tensor: | |
| n_tokens, d = x_flat.shape | |
| flat_expert = topk_idx.reshape(-1) | |
| flat_token = torch.arange(n_tokens, device=x_flat.device).repeat_interleave(self.top_k) | |
| flat_weight = topk_probs.reshape(-1) | |
| out = torch.zeros_like(x_flat) | |
| for e in range(self.n_experts): | |
| mask = flat_expert == e | |
| if not mask.any(): | |
| continue | |
| tok_idx = flat_token[mask] | |
| w = flat_weight[mask].unsqueeze(-1) | |
| xe = x_flat[tok_idx] | |
| h = F.silu(xe @ self.W_gate[e]) * (xe @ self.W_up[e]) | |
| down = h @ self.W_down[e] | |
| out.index_add_(0, tok_idx, down * w) | |
| return out | |
| def _forward_grouped_mm( | |
| self, x_flat: torch.Tensor, topk_idx: torch.Tensor, topk_probs: torch.Tensor | |
| ) -> torch.Tensor: | |
| n_tokens, d = x_flat.shape | |
| flat_expert = topk_idx.reshape(-1) | |
| flat_token = torch.arange(n_tokens, device=x_flat.device).repeat_interleave(self.top_k) | |
| flat_weight = topk_probs.reshape(-1) | |
| sort_idx = torch.argsort(flat_expert, stable=True) | |
| sorted_expert = flat_expert[sort_idx] | |
| sorted_token = flat_token[sort_idx] | |
| sorted_weight = flat_weight[sort_idx] | |
| counts = torch.bincount(sorted_expert, minlength=self.n_experts) | |
| offs = counts.cumsum(0).to(torch.int32) | |
| # torch._grouped_mm is a private op that torch.autocast does not know | |
| # to cast — it requires bf16/fp16 inputs explicitly, unlike ordinary | |
| # matmul which autocast handles automatically under the training | |
| # loop's autocast context. | |
| gathered_x = x_flat[sorted_token].to(torch.bfloat16) | |
| w_gate = self.W_gate.to(torch.bfloat16) | |
| w_up = self.W_up.to(torch.bfloat16) | |
| w_down = self.W_down.to(torch.bfloat16) | |
| gate = torch._grouped_mm(gathered_x, w_gate, offs=offs) | |
| up = torch._grouped_mm(gathered_x, w_up, offs=offs) | |
| h = F.silu(gate) * up | |
| down = torch._grouped_mm(h, w_down, offs=offs) | |
| down = down * sorted_weight.unsqueeze(-1).to(down.dtype) | |
| out = torch.zeros(n_tokens, d, device=x_flat.device, dtype=down.dtype) | |
| out.index_add_(0, sorted_token, down) | |
| return out.to(x_flat.dtype) | |