HPL-rus / model.py
bashkirtsevich's picture
Upload 4 files
581d38d verified
Raw History Blame Contribute Delete
1.01 kB
class MiniTransformer(nn.Module):
def __init__(self):
super().__init__()
self.token_embedding = nn.Embedding(vocab_size, embed_size)
self.position_embedding = nn.Embedding(block_size, embed_size)
encoder_layer = nn.TransformerEncoderLayer(
d_model=embed_size,
nhead=num_heads,
dim_feedforward=embed_size*4,
batch_first=True,
norm_first=True
)
self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
self.language_head = nn.Linear(embed_size, vocab_size)
def forward(self, idx):
B, T = idx.shape
tok_emb = self.token_embedding(idx)
pos_emb = self.position_embedding(torch.arange(T, device=device))
x = tok_emb + pos_emb
mask = torch.triu(torch.ones(T, T, device=device), diagonal=1).bool()
x = self.transformer(x, mask=mask, is_causal=True)
logits = self.language_head(x)
return logits