Download src/model/block.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 1.41 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/block.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/block.py
-
curl -L -o block.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/block.py
1.41 kB
| import torch | |
| import torch.nn as nn | |
| from .attention import Attention | |
| from .moe import ExpertLoadTracker, MoEFFN | |
| from .norm import ZeroCenteredRMSNorm | |
| class TransformerBlock(nn.Module): | |
| def __init__(self, d_model: int, attn_cfg, moe_cfg, norm_eps: float): | |
| super().__init__() | |
| self.norm1 = ZeroCenteredRMSNorm(d_model, eps=norm_eps) | |
| self.attn = Attention( | |
| d_model=d_model, | |
| n_heads=attn_cfg.n_heads, | |
| head_dim=attn_cfg.head_dim, | |
| n_kv_heads=attn_cfg.n_kv_heads, | |
| norm_eps=norm_eps, | |
| output_gate=attn_cfg.output_gate, | |
| bias=attn_cfg.bias, | |
| ) | |
| self.norm2 = ZeroCenteredRMSNorm(d_model, eps=norm_eps) | |
| self.moe = MoEFFN( | |
| d_model=d_model, | |
| n_experts=moe_cfg.n_experts, | |
| top_k=moe_cfg.top_k, | |
| expert_hidden=moe_cfg.expert_hidden, | |
| backend=moe_cfg.backend, | |
| router_init_std=moe_cfg.router_init_std, | |
| ) | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| cos: torch.Tensor, | |
| sin: torch.Tensor, | |
| load_tracker: ExpertLoadTracker | None, | |
| layer_idx: int, | |
| ): | |
| x = x + self.attn(self.norm1(x), cos, sin) | |
| moe_out, lb_loss, z_loss, hard_counts = self.moe(self.norm2(x), load_tracker, layer_idx) | |
| x = x + moe_out | |
| return x, lb_loss, z_loss, hard_counts | |