class MiniTransformer(nn.Module): def __init__(self): super().__init__() self.token_embedding = nn.Embedding(vocab_size, embed_size) self.position_embedding = nn.Embedding(block_size, embed_size) encoder_layer = nn.TransformerEncoderLayer( d_model=embed_size, nhead=num_heads, dim_feedforward=embed_size*4, batch_first=True, norm_first=True ) self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers) self.language_head = nn.Linear(embed_size, vocab_size) def forward(self, idx): B, T = idx.shape tok_emb = self.token_embedding(idx) pos_emb = self.position_embedding(torch.arange(T, device=device)) x = tok_emb + pos_emb mask = torch.triu(torch.ones(T, T, device=device), diagonal=1).bool() x = self.transformer(x, mask=mask, is_causal=True) logits = self.language_head(x) return logits