File size: 657 Bytes
37d8122 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | """FeedForward GELU (position-wise) — écrit à la main."""
from __future__ import annotations
import torch.nn as nn
class FeedForward(nn.Module):
"""MLP à deux couches : ``n_embd -> mult*n_embd -> n_embd`` avec GELU."""
def __init__(self, n_embd: int, mult: int = 4, dropout: float = 0.0, bias: bool = False):
super().__init__()
hidden = mult * n_embd
self.fc1 = nn.Linear(n_embd, hidden, bias=bias)
self.fc2 = nn.Linear(hidden, n_embd, bias=bias)
self.act = nn.GELU()
self.dropout = nn.Dropout(dropout)
def forward(self, x):
return self.dropout(self.fc2(self.act(self.fc1(x))))
|