Download model.py from bashkirtsevich/HPL-rus: direct link, hf CLI and curl.
- Browser
- Download file 1.01 kB
-
https://huggingface.co/bashkirtsevich/HPL-rus/resolve/main/model.py
- Command line
-
hf download hf://bashkirtsevich/HPL-rus/model.py
-
curl -L -o model.py https://huggingface.co/bashkirtsevich/HPL-rus/resolve/main/model.py
1.01 kB
| class MiniTransformer(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.token_embedding = nn.Embedding(vocab_size, embed_size) | |
| self.position_embedding = nn.Embedding(block_size, embed_size) | |
| encoder_layer = nn.TransformerEncoderLayer( | |
| d_model=embed_size, | |
| nhead=num_heads, | |
| dim_feedforward=embed_size*4, | |
| batch_first=True, | |
| norm_first=True | |
| ) | |
| self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers) | |
| self.language_head = nn.Linear(embed_size, vocab_size) | |
| def forward(self, idx): | |
| B, T = idx.shape | |
| tok_emb = self.token_embedding(idx) | |
| pos_emb = self.position_embedding(torch.arange(T, device=device)) | |
| x = tok_emb + pos_emb | |
| mask = torch.triu(torch.ones(T, T, device=device), diagonal=1).bool() | |
| x = self.transformer(x, mask=mask, is_causal=True) | |
| logits = self.language_head(x) | |
| return logits |