"""Small decoder-only Transformer (pre-LayerNorm GPT). web/js/nn/model.js mirrors this exactly.""" import torch import torch.nn as nn import torch.nn.functional as F class Block(nn.Module): def __init__(self, d, heads): super().__init__() self.h = heads self.ln1 = nn.LayerNorm(d) self.qkv = nn.Linear(d, 3 * d) self.proj = nn.Linear(d, d) self.ln2 = nn.LayerNorm(d) self.fc1 = nn.Linear(d, 4 * d) self.fc2 = nn.Linear(4 * d, d) def forward(self, x): B, T, C = x.shape q, k, v = self.qkv(self.ln1(x)).view(B, T, 3, self.h, C // self.h).permute(2, 0, 3, 1, 4) a = F.scaled_dot_product_attention(q, k, v, is_causal=True) x = x + self.proj(a.transpose(1, 2).reshape(B, T, C)) return x + self.fc2(F.gelu(self.fc1(self.ln2(x)), approximate="tanh")) class TinyGPT(nn.Module): def __init__(self, vocab, ctx, d=256, layers=6, heads=8): super().__init__() self.cfg = dict(vocab=vocab, ctx=ctx, d=d, layers=layers, heads=heads) self.tok = nn.Embedding(vocab, d) self.pos = nn.Embedding(ctx, d) self.blocks = nn.ModuleList(Block(d, heads) for _ in range(layers)) self.ln_f = nn.LayerNorm(d) self.apply(self._init) @staticmethod def _init(m): if isinstance(m, (nn.Linear, nn.Embedding)): nn.init.normal_(m.weight, std=0.02) if getattr(m, "bias", None) is not None: nn.init.zeros_(m.bias) def forward(self, idx): x = self.tok(idx) + self.pos(torch.arange(idx.shape[1], device=idx.device)) for b in self.blocks: x = b(x) return self.ln_f(x) @ self.tok.weight.T # tied output head @torch.no_grad() def generate(self, prompt_ids, eos, max_new=40): idx = torch.tensor([prompt_ids], device=self.tok.weight.device) out = [] for _ in range(max_new): if idx.shape[1] >= self.cfg["ctx"]: break nxt = int(self(idx)[0, -1].argmax()) if nxt == eos: break out.append(nxt) idx = torch.cat([idx, torch.tensor([[nxt]], device=idx.device)], 1) return out