File size: 4,203 Bytes
ad63418
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import torch
import torch.nn as nn
import torch.nn.functional as F

class PositionWiseFFN(nn.Module):
    def __init__(self, d_model, d_ff, dropout=0.1):
        super().__init__()
        self.linear1 = nn.Linear(d_model, d_ff)
        self.linear2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        x = self.linear1(x)
        x = F.relu(x)
        x = self.dropout(x)
        x = self.linear2(x)
        return x

class TopKRouter(nn.Module):
    def __init__(self, d_model, num_experts, top_k):
        super().__init__()
        self.num_experts = num_experts
        self.top_k = top_k
        self.gate = nn.Linear(d_model, num_experts, bias = False)

    def forward(self, x):
        logits = self.gate(x)
        values, indices = torch.topk(logits, self.top_k, dim = -1)
        routing_weights = F.softmax(values, dim = -1)
        return routing_weights, indices

class MoELayer(nn.Module):
    def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1):
        super().__init__()
        self.d_model = d_model
        self.d_expert = d_expert
        self.num_experts = num_experts
        self.top_k = top_k
        self.use_shared_expert = use_shared_expert

        self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long))
        self.track_usage = False

        if self.use_shared_expert == True:
            self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout)
            self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)])
            self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1)

        else:
            self.shared_expert = None
            self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)])
            self.router = TopKRouter(self.d_model, self.num_experts, self.top_k)

    def reset_usage_counts(self):
        self.expert_counts.zero_()

    def forward(self, x):
        batch_size, seq_len, d_model = x.shape
        x_flat = x.view(-1, d_model)
        routing_weights, selected_experts = self.router(x_flat)

        if self.track_usage:
            with torch.no_grad():
                counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts)
                self.expert_counts += counts

        out = torch.zeros_like(x_flat)

        for i, expert in enumerate(self.experts):
            token_idx, top_k_idx = torch.where(selected_experts == i)
            if token_idx.numel() == 0:
                continue
            tokens_for_expert = x_flat[token_idx]
            expert_out = expert(tokens_for_expert)
            weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1)
            weighted_out = expert_out * weights
            out[token_idx] += weighted_out

        if self.use_shared_expert:
            shared_out = self.shared_expert(x_flat)
            out = out + shared_out
            if self.track_usage:
                with torch.no_grad():
                    self.expert_counts[-1] += x_flat.size(0)

        out = out.view(batch_size, seq_len, d_model)
        return out

def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1):
    if variant == 1:
        return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout)
    elif variant == 2:
        return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout)
    elif variant == 3:
        return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
    elif variant == 4:
        return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout)
    elif variant == 5:
        return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
    else:
        raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5")