Add tokenizer_config.json, model.py, .gitattributes and README (renamed copy of gpt-s2.5-5m)

README.md

@@ -0,0 +1,85 @@
+---
+license: apache-2.0
+pipeline_tag: text-generation
+language: en
+tags:
+  - tiny
+  - tiny-lm
+  - tiny-model
+  - slm
+  - small-language-model
+  - from-scratch
+  - gpt
+  - gqa
+  - swiglu
+  - rope
+  - rmsnorm
+  - cpu-trained
+library_name: transformers
+metrics:
+  - accuracy
+model-index:
+  - name: HellaSwag
+    type: text-generation
+    results: []
+---
+
+# compacttest-5m
+
+**5.11M-parameter subword language model, trained from scratch on CPU.**
+
+> This repo is a **renamed copy** of [`Compactbot/gpt-s2.5-5m`](https://huggingface.co/Compactbot/gpt-s2.5-5m),
+> renamed at the request of @Datdanboi25 (see `Compactbot/model-requests` #2).
+> Weights, config and architecture are **identical** to the original.
+
+## What it is
+
+A small but genuine from-scratch causal LM in the GPT-X2.5 style:
+
+- **Architecture:** GQA (8 query / 2 key-value heads) + SwiGLU FFN + RoPE + RMSNorm,
+  pre-norm, no biases, **weight-tied** input/output embeddings (the embedding matrix
+  doubles as the LM head — no separate `lm_head` tensor).
+- **Params:** 5,114,112 (verified against the checkpoint).
+- **Tokenizer:** 8,192-vocab byte-level BPE (custom, not a standard HF tokenizer).
+- **Trained:** on CPU, ~94M TinyStories tokens, cosine LR schedule with warmup.
+
+## How to load
+
+This is a self-contained `nn.Module`, not a transformers-native architecture.
+
+```python
+import torch
+from model import Model
+
+m = Model()
+from safetensors.torch import load_file
+sd = load_file("model.safetensors")
+m.load_state_dict(sd)
+m.eval()
+
+# greedy generation
+with torch.no_grad():
+    x = torch.tensor([[1]])
+    for _ in range(60):
+        logits = m(x[:, -512:])[:, -1, :]
+        x = torch.cat([x, logits.argmax(-1, keepdim=True)], dim=1)
+```
+
+## Honest scope
+
+- **Greedy-coherent, sampling-fragile.** At 5M params the model produces readable
+  prose under greedy decoding but degrades noticeably under sampling. That is the
+  expected behaviour at this scale, not a bug.
+- **Intelligence index:** 0.032 (see the original card for the full eval breakdown).
+- It is a reference build demonstrating that a 5M-param from-scratch subword LM is
+  trainable and coherent on CPU. It is not a chat model.
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `config.json` | Architecture config (matches `model.py` exactly) |
+| `model.py` | Self-contained architecture definition |
+| `model.safetensors` | Weights (5,114,112 params, F32) |
+| `tokenizer.json` | 8192-vocab BPE tokenizer |
+| `tokenizer_config.json` | Tokenizer metadata |

model.py

@@ -0,0 +1,153 @@
+"""
+GPT-S2.5 — 5.11M-param subword language model (from scratch).
+Custom GPT-2-style architecture: GQA + SwiGLU + RoPE + RMSNorm, weight-tied head.
+
+This is NOT a transformers-native architecture. It is a self-contained
+nn.Module you can load directly:
+
+    import torch
+    from model import Model
+    m = Model()
+    sd = torch.load("best.pt", map_location="cpu")["model"]   # or load safetensors
+    m.load_state_dict(sd); m.eval()
+
+    # generate (greedy):
+    with torch.no_grad():
+        x = torch.tensor([[1]])
+        for _ in range(60):
+            logits = m(x[:, -512:])[:, -1, :]
+            x = torch.cat([x, logits.argmax(-1, keepdim=True)], dim=1)
+
+Architecture (matches config.json exactly):
+  vocab 8192 (BPE), n_embd 256, 4 layers, 8 q-heads / 2 kv-heads (GQA 4:1),
+  head_dim 32, SwiGLU FFN intermediate 768, RoPE (base 10000), RMSNorm (eps 1e-6),
+  pre-norm, no biases, weight-tied head (tok.weight reused as lm_head).
+  Total: 5,114,112 parameters.
+"""
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+VOCAB      = 8192
+N_EMBD      = 256
+N_LAYERS    = 4
+N_Q_HEADS   = 8
+N_KV_HEADS  = 2
+HEAD_DIM    = 32
+INTER       = 768
+MAX_SEQ     = 512
+ROPE_BASE   = 10000.0
+
+Q_DIM  = N_Q_HEADS * HEAD_DIM   # 256
+KV_DIM = N_KV_HEADS * HEAD_DIM  # 64
+assert Q_DIM == N_EMBD
+assert N_Q_HEADS % N_KV_HEADS == 0
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
+
+
+def precompute_rope(head_dim, max_seq, base):
+    freqs = 1.0 / (base ** (torch.arange(0, head_dim, 2).float() / head_dim))
+    t = torch.arange(max_seq, dtype=torch.float)
+    ang = torch.outer(t, freqs)
+    return ang.cos(), ang.sin()  # [T, head_dim/2]
+
+
+def apply_rope(x, cos, sin):
+    # x: [B, H, T, D]
+    T = x.size(2)
+    cos = cos[:T].view(1, 1, T, -1)
+    sin = sin[:T].view(1, 1, T, -1)
+    x1, x2 = x[..., 0::2], x[..., 1::2]
+    out = torch.empty_like(x)
+    out[..., 0::2] = x1 * cos - x2 * sin
+    out[..., 1::2] = x2 * cos + x1 * sin
+    return out
+
+
+class Attention(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.wq = nn.Linear(N_EMBD, Q_DIM, bias=False)
+        self.wk = nn.Linear(N_EMBD, KV_DIM, bias=False)
+        self.wv = nn.Linear(N_EMBD, KV_DIM, bias=False)
+        self.wo = nn.Linear(Q_DIM, N_EMBD, bias=False)
+
+    def forward(self, x, cos, sin):
+        B, T, _ = x.shape
+        q = self.wq(x).view(B, T, N_Q_HEADS, HEAD_DIM).transpose(1, 2)
+        k = self.wk(x).view(B, T, N_KV_HEADS, HEAD_DIM).transpose(1, 2)
+        v = self.wv(x).view(B, T, N_KV_HEADS, HEAD_DIM).transpose(1, 2)
+        q = apply_rope(q, cos, sin)
+        k = apply_rope(k, cos, sin)
+        # GQA: repeat kv heads to match q heads
+        rep = N_Q_HEADS // N_KV_HEADS
+        k = k.repeat_interleave(rep, dim=1)
+        v = v.repeat_interleave(rep, dim=1)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        y = y.transpose(1, 2).contiguous().view(B, T, Q_DIM)
+        return self.wo(y)
+
+
+class MLP(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.wgate = nn.Linear(N_EMBD, INTER, bias=False)
+        self.wup   = nn.Linear(N_EMBD, INTER, bias=False)
+        self.wdown = nn.Linear(INTER, N_EMBD, bias=False)
+
+    def forward(self, x):
+        return self.wdown(F.silu(self.wgate(x)) * self.wup(x))
+
+
+class Block(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.ln1 = RMSNorm(N_EMBD)
+        self.attn = Attention()
+        self.ln2 = RMSNorm(N_EMBD)
+        self.mlp = MLP()
+
+    def forward(self, x, cos, sin):
+        x = x + self.attn(self.ln1(x), cos, sin)
+        x = x + self.mlp(self.ln2(x))
+        return x
+
+
+class Model(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.tok = nn.Embedding(VOCAB, N_EMBD)
+        self.blocks = nn.ModuleList([Block() for _ in range(N_LAYERS)])
+        self.ln_f = RMSNorm(N_EMBD)
+        self.cos, self.sin = precompute_rope(HEAD_DIM, MAX_SEQ, ROPE_BASE)
+
+    def forward(self, idx, targets=None):
+        B, T = idx.shape
+        h = self.tok(idx)
+        cos, sin = self.cos, self.sin
+        for blk in self.blocks:
+            h = blk(h, cos, sin)
+        h = self.ln_f(h)
+        logits = F.linear(h, self.tok.weight)  # weight-tied head
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, VOCAB), targets.view(-1))
+            return loss
+        return logits
+
+
+if __name__ == "__main__":
+    m = Model()
+    total = sum(p.numel() for p in m.parameters())
+    print(f"params = {total}  (expected 5114112)")
+    assert total == 5114112, "param count mismatch"
+    print("OK")

tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "tokenizer_class": "GPTS25BPE",
+  "model_type": "gpt-s2.5",
+  "vocab_size": 8192,
+  "add_prefix_space": false,
+  "note": "8192-vocab BPE (byte-level). This is a custom BPE, not a standard HF tokenizer; load bpe_8k.json / tokenizer.json directly."
+}