Lucasgitguy-co's picture
wiki-model 0.1.0
787f661 verified
Raw History Blame Contribute Delete
5.64 kB
"""wiki-model-tiny: a ~12M-param Llama-style GPT in plain PyTorch (RMSNorm, RoPE, SwiGLU, tied embeddings)."""
import json, math
from dataclasses import dataclass, asdict
import torch
import torch.nn as nn
import torch.nn.functional as F
@dataclass
class Config:
vocab: int = 8192
dim: int = 320
layers: int = 8
heads: int = 8
hidden: int = 864
ctx: int = 384
rope_base: float = 10000.0
class RMSNorm(nn.Module):
def __init__(self, d, eps=1e-5):
super().__init__(); self.w = nn.Parameter(torch.ones(d)); self.eps = eps
def forward(self, x):
return self.w * x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps).to(x.dtype)
def rope(x, cos, sin):
x1, x2 = x[..., ::2], x[..., 1::2]
return torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), -1).flatten(-2)
class Block(nn.Module):
def __init__(self, c):
super().__init__()
self.h, self.hd = c.heads, c.dim // c.heads
self.n1, self.n2 = RMSNorm(c.dim), RMSNorm(c.dim)
self.qkv = nn.Linear(c.dim, 3 * c.dim, bias=False)
self.o = nn.Linear(c.dim, c.dim, bias=False)
self.gate_up = nn.Linear(c.dim, 2 * c.hidden, bias=False)
self.down = nn.Linear(c.hidden, c.dim, bias=False)
def forward(self, x, cos, sin, cache=None):
B, T, D = x.shape
q, k, v = self.qkv(self.n1(x)).view(B, T, 3, self.h, self.hd).permute(2, 0, 3, 1, 4)
q, k = rope(q, cos, sin), rope(k, cos, sin)
if cache is not None:
if "k" in cache:
k, v = torch.cat([cache["k"], k], 2), torch.cat([cache["v"], v], 2)
cache["k"], cache["v"] = k, v
y = F.scaled_dot_product_attention(q, k, v, is_causal=T > 1)
x = x + self.o(y.transpose(1, 2).reshape(B, T, D))
g, u = self.gate_up(self.n2(x)).chunk(2, -1)
return x + self.down(F.silu(g) * u)
class TinyGPT(nn.Module):
def __init__(self, c: Config):
super().__init__()
self.c = c
self.emb = nn.Embedding(c.vocab, c.dim)
self.blocks = nn.ModuleList(Block(c) for _ in range(c.layers))
self.norm = RMSNorm(c.dim)
hd = c.dim // c.heads
inv = 1.0 / (c.rope_base ** (torch.arange(0, hd, 2).float() / hd))
f = torch.outer(torch.arange(c.ctx).float(), inv)
self.register_buffer("cos", f.cos(), persistent=False)
self.register_buffer("sin", f.sin(), persistent=False)
self.apply(self._init)
for b in self.blocks: # scaled residual init
nn.init.normal_(b.o.weight, std=0.02 / math.sqrt(2 * c.layers))
nn.init.normal_(b.down.weight, std=0.02 / math.sqrt(2 * c.layers))
@staticmethod
def _init(m):
if isinstance(m, (nn.Linear, nn.Embedding)):
nn.init.normal_(m.weight, std=0.02)
def forward(self, idx, targets=None, caches=None, pos=0, sel=None):
T = idx.shape[1]
cos, sin = self.cos[pos:pos + T], self.sin[pos:pos + T]
x = self.emb(idx)
for i, b in enumerate(self.blocks):
x = b(x, cos, sin, None if caches is None else caches[i])
h = self.norm(x)
if targets is None:
return F.linear(h, self.emb.weight)
if sel is None: # all positions (pretraining): fixed shape
return F.cross_entropy(F.linear(h, self.emb.weight).float().flatten(0, 1), targets.flatten(), ignore_index=-100)
# SFT: LM head only at the given flat positions (padded to a bucket by the caller so MPS sees few shapes)
hs = h.flatten(0, 1).index_select(0, sel)
return F.cross_entropy(F.linear(hs, self.emb.weight).float(), targets.flatten().index_select(0, sel), ignore_index=-100)
@torch.no_grad()
def generate(self, ids, max_new=80, eos=1, temperature=0.0, top_p=0.9, gen=None, no_repeat=4):
ids = ids[-(self.c.ctx - max_new):]
caches = [{} for _ in self.blocks]
x = torch.tensor([ids], device=self.emb.weight.device)
logits = self(x, caches=caches)[:, -1]
out, pos = [], len(ids)
for _ in range(max_new):
if no_repeat and len(out) >= no_repeat - 1: # block tokens that would repeat an n-gram of the answer
tail = tuple(out[-(no_repeat - 1):])
banned = [out[i + no_repeat - 1] for i in range(len(out) - no_repeat + 1) if tuple(out[i:i + no_repeat - 1]) == tail]
if banned: logits[0, banned] = float("-inf")
if temperature <= 0:
nxt = int(logits.argmax(-1))
else:
p = torch.softmax(logits.float() / temperature, -1)[0]
sp, si = p.sort(descending=True)
keep = sp.cumsum(0) - sp < top_p
nxt = int(si[keep][torch.multinomial(sp[keep], 1, generator=gen)])
if nxt == eos or pos >= self.c.ctx: break
out.append(nxt)
logits = self(torch.tensor([[nxt]], device=x.device), caches=caches, pos=pos)[:, -1]; pos += 1
return out
def save(self, path):
from safetensors.torch import save_file
import os
os.makedirs(path, exist_ok=True)
save_file({k: v.contiguous() for k, v in self.state_dict().items() if k != "emb_out"}, f"{path}/model.safetensors")
json.dump(asdict(self.c), open(f"{path}/config.json", "w"), indent=1)
@classmethod
def load(cls, path, device="cpu"):
from safetensors.torch import load_file
m = cls(Config(**json.load(open(f"{path}/config.json"))))
m.load_state_dict(load_file(f"{path}/model.safetensors"), strict=True)
return m.to(device).eval()