Download src/model/transformer.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 2.23 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/transformer.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/transformer.py
-
curl -L -o transformer.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/transformer.py
2.23 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .block import TransformerBlock | |
| from .init import init_weights | |
| from .moe import ExpertLoadTracker | |
| from .norm import ZeroCenteredRMSNorm | |
| from .rope import precompute_rope | |
| class Transformer(nn.Module): | |
| def __init__(self, model_cfg): | |
| super().__init__() | |
| self.cfg = model_cfg | |
| self.n_layers = model_cfg.n_layers | |
| self.embed = nn.Embedding(model_cfg.vocab_size, model_cfg.d_model) | |
| self.blocks = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| d_model=model_cfg.d_model, | |
| attn_cfg=model_cfg.attention, | |
| moe_cfg=model_cfg.moe, | |
| norm_eps=model_cfg.norm_eps, | |
| ) | |
| for _ in range(model_cfg.n_layers) | |
| ] | |
| ) | |
| self.final_norm = ZeroCenteredRMSNorm(model_cfg.d_model, eps=model_cfg.norm_eps) | |
| cos, sin = precompute_rope( | |
| model_cfg.attention.head_dim, model_cfg.max_seq_len, model_cfg.attention.rope_theta | |
| ) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| init_weights(self, model_cfg.init.std, model_cfg.n_layers) | |
| def make_load_tracker(self, device) -> ExpertLoadTracker: | |
| return ExpertLoadTracker(self.n_layers, self.cfg.moe.n_experts, device) | |
| def forward(self, idx: torch.Tensor, load_tracker: ExpertLoadTracker | None = None): | |
| bsz, seq_len = idx.shape | |
| x = self.embed(idx) | |
| cos = self.rope_cos[:seq_len].to(x.device) | |
| sin = self.rope_sin[:seq_len].to(x.device) | |
| lb_loss_sum = x.new_zeros(()) | |
| z_loss_sum = x.new_zeros(()) | |
| hard_counts = [] | |
| for i, block in enumerate(self.blocks): | |
| x, lb, z, hc = block(x, cos, sin, load_tracker, i) | |
| lb_loss_sum = lb_loss_sum + lb | |
| z_loss_sum = z_loss_sum + z | |
| hard_counts.append(hc) | |
| x = self.final_norm(x) | |
| logits = F.linear(x, self.embed.weight) | |
| lb_loss = lb_loss_sum / self.n_layers | |
| z_loss = z_loss_sum / self.n_layers | |
| return logits, lb_loss, z_loss, hard_counts | |