Download wiki_model/model.py from Lucasgitguy-co/wiki-model-tiny: direct link, hf CLI and curl.
- Browser
- Download file 5.64 kB
-
https://huggingface.co/Lucasgitguy-co/wiki-model-tiny/resolve/main/wiki_model/model.py
- Command line
-
hf download hf://Lucasgitguy-co/wiki-model-tiny/wiki_model/model.py
-
curl -L -o model.py https://huggingface.co/Lucasgitguy-co/wiki-model-tiny/resolve/main/wiki_model/model.py
5.64 kB
| """wiki-model-tiny: a ~12M-param Llama-style GPT in plain PyTorch (RMSNorm, RoPE, SwiGLU, tied embeddings).""" | |
| import json, math | |
| from dataclasses import dataclass, asdict | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class Config: | |
| vocab: int = 8192 | |
| dim: int = 320 | |
| layers: int = 8 | |
| heads: int = 8 | |
| hidden: int = 864 | |
| ctx: int = 384 | |
| rope_base: float = 10000.0 | |
| class RMSNorm(nn.Module): | |
| def __init__(self, d, eps=1e-5): | |
| super().__init__(); self.w = nn.Parameter(torch.ones(d)); self.eps = eps | |
| def forward(self, x): | |
| return self.w * x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps).to(x.dtype) | |
| def rope(x, cos, sin): | |
| x1, x2 = x[..., ::2], x[..., 1::2] | |
| return torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), -1).flatten(-2) | |
| class Block(nn.Module): | |
| def __init__(self, c): | |
| super().__init__() | |
| self.h, self.hd = c.heads, c.dim // c.heads | |
| self.n1, self.n2 = RMSNorm(c.dim), RMSNorm(c.dim) | |
| self.qkv = nn.Linear(c.dim, 3 * c.dim, bias=False) | |
| self.o = nn.Linear(c.dim, c.dim, bias=False) | |
| self.gate_up = nn.Linear(c.dim, 2 * c.hidden, bias=False) | |
| self.down = nn.Linear(c.hidden, c.dim, bias=False) | |
| def forward(self, x, cos, sin, cache=None): | |
| B, T, D = x.shape | |
| q, k, v = self.qkv(self.n1(x)).view(B, T, 3, self.h, self.hd).permute(2, 0, 3, 1, 4) | |
| q, k = rope(q, cos, sin), rope(k, cos, sin) | |
| if cache is not None: | |
| if "k" in cache: | |
| k, v = torch.cat([cache["k"], k], 2), torch.cat([cache["v"], v], 2) | |
| cache["k"], cache["v"] = k, v | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=T > 1) | |
| x = x + self.o(y.transpose(1, 2).reshape(B, T, D)) | |
| g, u = self.gate_up(self.n2(x)).chunk(2, -1) | |
| return x + self.down(F.silu(g) * u) | |
| class TinyGPT(nn.Module): | |
| def __init__(self, c: Config): | |
| super().__init__() | |
| self.c = c | |
| self.emb = nn.Embedding(c.vocab, c.dim) | |
| self.blocks = nn.ModuleList(Block(c) for _ in range(c.layers)) | |
| self.norm = RMSNorm(c.dim) | |
| hd = c.dim // c.heads | |
| inv = 1.0 / (c.rope_base ** (torch.arange(0, hd, 2).float() / hd)) | |
| f = torch.outer(torch.arange(c.ctx).float(), inv) | |
| self.register_buffer("cos", f.cos(), persistent=False) | |
| self.register_buffer("sin", f.sin(), persistent=False) | |
| self.apply(self._init) | |
| for b in self.blocks: # scaled residual init | |
| nn.init.normal_(b.o.weight, std=0.02 / math.sqrt(2 * c.layers)) | |
| nn.init.normal_(b.down.weight, std=0.02 / math.sqrt(2 * c.layers)) | |
| def _init(m): | |
| if isinstance(m, (nn.Linear, nn.Embedding)): | |
| nn.init.normal_(m.weight, std=0.02) | |
| def forward(self, idx, targets=None, caches=None, pos=0, sel=None): | |
| T = idx.shape[1] | |
| cos, sin = self.cos[pos:pos + T], self.sin[pos:pos + T] | |
| x = self.emb(idx) | |
| for i, b in enumerate(self.blocks): | |
| x = b(x, cos, sin, None if caches is None else caches[i]) | |
| h = self.norm(x) | |
| if targets is None: | |
| return F.linear(h, self.emb.weight) | |
| if sel is None: # all positions (pretraining): fixed shape | |
| return F.cross_entropy(F.linear(h, self.emb.weight).float().flatten(0, 1), targets.flatten(), ignore_index=-100) | |
| # SFT: LM head only at the given flat positions (padded to a bucket by the caller so MPS sees few shapes) | |
| hs = h.flatten(0, 1).index_select(0, sel) | |
| return F.cross_entropy(F.linear(hs, self.emb.weight).float(), targets.flatten().index_select(0, sel), ignore_index=-100) | |
| def generate(self, ids, max_new=80, eos=1, temperature=0.0, top_p=0.9, gen=None, no_repeat=4): | |
| ids = ids[-(self.c.ctx - max_new):] | |
| caches = [{} for _ in self.blocks] | |
| x = torch.tensor([ids], device=self.emb.weight.device) | |
| logits = self(x, caches=caches)[:, -1] | |
| out, pos = [], len(ids) | |
| for _ in range(max_new): | |
| if no_repeat and len(out) >= no_repeat - 1: # block tokens that would repeat an n-gram of the answer | |
| tail = tuple(out[-(no_repeat - 1):]) | |
| banned = [out[i + no_repeat - 1] for i in range(len(out) - no_repeat + 1) if tuple(out[i:i + no_repeat - 1]) == tail] | |
| if banned: logits[0, banned] = float("-inf") | |
| if temperature <= 0: | |
| nxt = int(logits.argmax(-1)) | |
| else: | |
| p = torch.softmax(logits.float() / temperature, -1)[0] | |
| sp, si = p.sort(descending=True) | |
| keep = sp.cumsum(0) - sp < top_p | |
| nxt = int(si[keep][torch.multinomial(sp[keep], 1, generator=gen)]) | |
| if nxt == eos or pos >= self.c.ctx: break | |
| out.append(nxt) | |
| logits = self(torch.tensor([[nxt]], device=x.device), caches=caches, pos=pos)[:, -1]; pos += 1 | |
| return out | |
| def save(self, path): | |
| from safetensors.torch import save_file | |
| import os | |
| os.makedirs(path, exist_ok=True) | |
| save_file({k: v.contiguous() for k, v in self.state_dict().items() if k != "emb_out"}, f"{path}/model.safetensors") | |
| json.dump(asdict(self.c), open(f"{path}/config.json", "w"), indent=1) | |
| def load(cls, path, device="cpu"): | |
| from safetensors.torch import load_file | |
| m = cls(Config(**json.load(open(f"{path}/config.json")))) | |
| m.load_state_dict(load_file(f"{path}/model.safetensors"), strict=True) | |
| return m.to(device).eval() | |