walke007's picture
Upload Part 2 checkpoints and recorded results
a3baf17 verified
Raw History Blame Contribute Delete
406 Bytes
import torch
from torch import nn
class FFN(nn.Module):
def __init__(self, model_dimensions):
super().__init__()
self.d = model_dimensions
self.FFN_stack = nn.Sequential(
nn.Linear(self.d, 4 * self.d),
nn.ReLU(),
nn.Linear(4 * self.d, self.d)
)
def forward(self, x):
logits = self.FFN_stack(x)
return logits