File size: 4,203 Bytes
ad63418 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | import torch
import torch.nn as nn
import torch.nn.functional as F
class PositionWiseFFN(nn.Module):
def __init__(self, d_model, d_ff, dropout=0.1):
super().__init__()
self.linear1 = nn.Linear(d_model, d_ff)
self.linear2 = nn.Linear(d_ff, d_model)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
x = self.linear1(x)
x = F.relu(x)
x = self.dropout(x)
x = self.linear2(x)
return x
class TopKRouter(nn.Module):
def __init__(self, d_model, num_experts, top_k):
super().__init__()
self.num_experts = num_experts
self.top_k = top_k
self.gate = nn.Linear(d_model, num_experts, bias = False)
def forward(self, x):
logits = self.gate(x)
values, indices = torch.topk(logits, self.top_k, dim = -1)
routing_weights = F.softmax(values, dim = -1)
return routing_weights, indices
class MoELayer(nn.Module):
def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1):
super().__init__()
self.d_model = d_model
self.d_expert = d_expert
self.num_experts = num_experts
self.top_k = top_k
self.use_shared_expert = use_shared_expert
self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long))
self.track_usage = False
if self.use_shared_expert == True:
self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout)
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)])
self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1)
else:
self.shared_expert = None
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)])
self.router = TopKRouter(self.d_model, self.num_experts, self.top_k)
def reset_usage_counts(self):
self.expert_counts.zero_()
def forward(self, x):
batch_size, seq_len, d_model = x.shape
x_flat = x.view(-1, d_model)
routing_weights, selected_experts = self.router(x_flat)
if self.track_usage:
with torch.no_grad():
counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts)
self.expert_counts += counts
out = torch.zeros_like(x_flat)
for i, expert in enumerate(self.experts):
token_idx, top_k_idx = torch.where(selected_experts == i)
if token_idx.numel() == 0:
continue
tokens_for_expert = x_flat[token_idx]
expert_out = expert(tokens_for_expert)
weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1)
weighted_out = expert_out * weights
out[token_idx] += weighted_out
if self.use_shared_expert:
shared_out = self.shared_expert(x_flat)
out = out + shared_out
if self.track_usage:
with torch.no_grad():
self.expert_counts[-1] += x_flat.size(0)
out = out.view(batch_size, seq_len, d_model)
return out
def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1):
if variant == 1:
return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout)
elif variant == 2:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout)
elif variant == 3:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
elif variant == 4:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout)
elif variant == 5:
return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
else:
raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5")
|