File size: 1,405 Bytes
27cd743
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
import torch
import torch.nn as nn

from .attention import Attention
from .moe import ExpertLoadTracker, MoEFFN
from .norm import ZeroCenteredRMSNorm


class TransformerBlock(nn.Module):
    def __init__(self, d_model: int, attn_cfg, moe_cfg, norm_eps: float):
        super().__init__()
        self.norm1 = ZeroCenteredRMSNorm(d_model, eps=norm_eps)
        self.attn = Attention(
            d_model=d_model,
            n_heads=attn_cfg.n_heads,
            head_dim=attn_cfg.head_dim,
            n_kv_heads=attn_cfg.n_kv_heads,
            norm_eps=norm_eps,
            output_gate=attn_cfg.output_gate,
            bias=attn_cfg.bias,
        )
        self.norm2 = ZeroCenteredRMSNorm(d_model, eps=norm_eps)
        self.moe = MoEFFN(
            d_model=d_model,
            n_experts=moe_cfg.n_experts,
            top_k=moe_cfg.top_k,
            expert_hidden=moe_cfg.expert_hidden,
            backend=moe_cfg.backend,
            router_init_std=moe_cfg.router_init_std,
        )

    def forward(
        self,
        x: torch.Tensor,
        cos: torch.Tensor,
        sin: torch.Tensor,
        load_tracker: ExpertLoadTracker | None,
        layer_idx: int,
    ):
        x = x + self.attn(self.norm1(x), cos, sin)
        moe_out, lb_loss, z_loss, hard_counts = self.moe(self.norm2(x), load_tracker, layer_idx)
        x = x + moe_out
        return x, lb_loss, z_loss, hard_counts