Download ml/model.py from paralaif/tiny-scene-three: direct link, hf CLI and curl.
- Browser
- Download file 2.23 kB
-
https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/model.py
- Command line
-
hf download hf://paralaif/tiny-scene-three/ml/model.py
-
curl -L -o model.py https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/model.py
2.23 kB
| """Small decoder-only Transformer (pre-LayerNorm GPT). web/js/nn/model.js mirrors this exactly.""" | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class Block(nn.Module): | |
| def __init__(self, d, heads): | |
| super().__init__() | |
| self.h = heads | |
| self.ln1 = nn.LayerNorm(d) | |
| self.qkv = nn.Linear(d, 3 * d) | |
| self.proj = nn.Linear(d, d) | |
| self.ln2 = nn.LayerNorm(d) | |
| self.fc1 = nn.Linear(d, 4 * d) | |
| self.fc2 = nn.Linear(4 * d, d) | |
| def forward(self, x): | |
| B, T, C = x.shape | |
| q, k, v = self.qkv(self.ln1(x)).view(B, T, 3, self.h, C // self.h).permute(2, 0, 3, 1, 4) | |
| a = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| x = x + self.proj(a.transpose(1, 2).reshape(B, T, C)) | |
| return x + self.fc2(F.gelu(self.fc1(self.ln2(x)), approximate="tanh")) | |
| class TinyGPT(nn.Module): | |
| def __init__(self, vocab, ctx, d=256, layers=6, heads=8): | |
| super().__init__() | |
| self.cfg = dict(vocab=vocab, ctx=ctx, d=d, layers=layers, heads=heads) | |
| self.tok = nn.Embedding(vocab, d) | |
| self.pos = nn.Embedding(ctx, d) | |
| self.blocks = nn.ModuleList(Block(d, heads) for _ in range(layers)) | |
| self.ln_f = nn.LayerNorm(d) | |
| self.apply(self._init) | |
| def _init(m): | |
| if isinstance(m, (nn.Linear, nn.Embedding)): | |
| nn.init.normal_(m.weight, std=0.02) | |
| if getattr(m, "bias", None) is not None: | |
| nn.init.zeros_(m.bias) | |
| def forward(self, idx): | |
| x = self.tok(idx) + self.pos(torch.arange(idx.shape[1], device=idx.device)) | |
| for b in self.blocks: | |
| x = b(x) | |
| return self.ln_f(x) @ self.tok.weight.T # tied output head | |
| def generate(self, prompt_ids, eos, max_new=40): | |
| idx = torch.tensor([prompt_ids], device=self.tok.weight.device) | |
| out = [] | |
| for _ in range(max_new): | |
| if idx.shape[1] >= self.cfg["ctx"]: | |
| break | |
| nxt = int(self(idx)[0, -1].argmax()) | |
| if nxt == eos: | |
| break | |
| out.append(nxt) | |
| idx = torch.cat([idx, torch.tensor([[nxt]], device=idx.device)], 1) | |
| return out | |