bubblspace-moe0.9b-a0.2b / src /model /transformer.py
TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
2.23 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from .block import TransformerBlock
from .init import init_weights
from .moe import ExpertLoadTracker
from .norm import ZeroCenteredRMSNorm
from .rope import precompute_rope
class Transformer(nn.Module):
def __init__(self, model_cfg):
super().__init__()
self.cfg = model_cfg
self.n_layers = model_cfg.n_layers
self.embed = nn.Embedding(model_cfg.vocab_size, model_cfg.d_model)
self.blocks = nn.ModuleList(
[
TransformerBlock(
d_model=model_cfg.d_model,
attn_cfg=model_cfg.attention,
moe_cfg=model_cfg.moe,
norm_eps=model_cfg.norm_eps,
)
for _ in range(model_cfg.n_layers)
]
)
self.final_norm = ZeroCenteredRMSNorm(model_cfg.d_model, eps=model_cfg.norm_eps)
cos, sin = precompute_rope(
model_cfg.attention.head_dim, model_cfg.max_seq_len, model_cfg.attention.rope_theta
)
self.register_buffer("rope_cos", cos, persistent=False)
self.register_buffer("rope_sin", sin, persistent=False)
init_weights(self, model_cfg.init.std, model_cfg.n_layers)
def make_load_tracker(self, device) -> ExpertLoadTracker:
return ExpertLoadTracker(self.n_layers, self.cfg.moe.n_experts, device)
def forward(self, idx: torch.Tensor, load_tracker: ExpertLoadTracker | None = None):
bsz, seq_len = idx.shape
x = self.embed(idx)
cos = self.rope_cos[:seq_len].to(x.device)
sin = self.rope_sin[:seq_len].to(x.device)
lb_loss_sum = x.new_zeros(())
z_loss_sum = x.new_zeros(())
hard_counts = []
for i, block in enumerate(self.blocks):
x, lb, z, hc = block(x, cos, sin, load_tracker, i)
lb_loss_sum = lb_loss_sum + lb
z_loss_sum = z_loss_sum + z
hard_counts.append(hc)
x = self.final_norm(x)
logits = F.linear(x, self.embed.weight)
lb_loss = lb_loss_sum / self.n_layers
z_loss = z_loss_sum / self.n_layers
return logits, lb_loss, z_loss, hard_counts