Download model.py from Adam25613/AMAIA: direct link, hf CLI and curl.
- Browser
- Download file 3.8 kB
-
https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/model.py
- Command line
-
hf download hf://spaces/Adam25613/AMAIA/model.py
-
curl -L -o model.py https://huggingface.co/spaces/Adam25613/AMAIA/resolve/main/model.py
3.8 kB
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class CausalSelfAttention(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.n_heads = cfg.n_heads | |
| self.d_head = cfg.d_model // cfg.n_heads | |
| self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=False) | |
| self.proj = nn.Linear(cfg.d_model, cfg.d_model, bias=False) | |
| self.dropout = nn.Dropout(cfg.dropout) | |
| def forward(self, x): | |
| B, T, C = x.shape | |
| q, k, v = self.qkv(x).split(C, dim=2) | |
| q = q.view(B, T, self.n_heads, self.d_head).transpose(1, 2) | |
| k = k.view(B, T, self.n_heads, self.d_head).transpose(1, 2) | |
| v = v.view(B, T, self.n_heads, self.d_head).transpose(1, 2) | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=True, | |
| dropout_p=self.dropout.p if self.training else 0.0) | |
| y = y.transpose(1, 2).contiguous().view(B, T, C) | |
| return self.dropout(self.proj(y)) | |
| class FFN(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.fc1 = nn.Linear(cfg.d_model, 4 * cfg.d_model) | |
| self.fc2 = nn.Linear(4 * cfg.d_model, cfg.d_model) | |
| self.dropout = nn.Dropout(cfg.dropout) | |
| def forward(self, x): | |
| return self.dropout(self.fc2(F.gelu(self.fc1(x)))) | |
| class Block(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.ln1 = nn.LayerNorm(cfg.d_model) | |
| self.attn = CausalSelfAttention(cfg) | |
| self.ln2 = nn.LayerNorm(cfg.d_model) | |
| self.ffn = FFN(cfg) | |
| def forward(self, x): | |
| x = x + self.attn(self.ln1(x)) | |
| x = x + self.ffn(self.ln2(x)) | |
| return x | |
| class Amaia(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.tok_emb = nn.Embedding(cfg.vocab_size, cfg.d_model) | |
| self.pos_emb = nn.Embedding(cfg.max_seq, cfg.d_model) | |
| self.drop = nn.Dropout(cfg.dropout) | |
| self.blocks = nn.Sequential(*[Block(cfg) for _ in range(cfg.n_layers)]) | |
| self.ln_f = nn.LayerNorm(cfg.d_model) | |
| self.head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False) | |
| self.tok_emb.weight = self.head.weight | |
| self.apply(self._init_weights) | |
| def _init_weights(self, m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, std=0.02) | |
| if m.bias is not None: | |
| nn.init.zeros_(m.bias) | |
| elif isinstance(m, nn.Embedding): | |
| nn.init.normal_(m.weight, std=0.02) | |
| def forward(self, idx, targets=None): | |
| B, T = idx.shape | |
| pos = torch.arange(T, device=idx.device) | |
| x = self.drop(self.tok_emb(idx) + self.pos_emb(pos)) | |
| x = self.blocks(x) | |
| x = self.ln_f(x) | |
| logits = self.head(x) | |
| loss = None | |
| if targets is not None: | |
| loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) | |
| return logits, loss | |
| def n_params(self): | |
| return sum(p.numel() for p in self.parameters()) | |
| def generate(self, idx, max_new_tokens, temperature=0.8, top_p=0.92): | |
| for _ in range(max_new_tokens): | |
| idx_cond = idx[:, -self.cfg.max_seq:] | |
| logits, _ = self(idx_cond) | |
| logits = logits[:, -1, :] / temperature | |
| probs = F.softmax(logits, dim=-1) | |
| sorted_probs, sorted_idx = torch.sort(probs, descending=True) | |
| cumsum = torch.cumsum(sorted_probs, dim=-1) | |
| mask = cumsum - sorted_probs > top_p | |
| sorted_probs[mask] = 0.0 | |
| sorted_probs /= sorted_probs.sum() | |
| next_token = sorted_idx.gather(-1, torch.multinomial(sorted_probs, 1)) | |
| idx = torch.cat([idx, next_token], dim=1) | |
| return idx | |