Add 3B-Chat-v2, switch large models to GGUF, preload default

- Add GPUburnout-3B-75K-Chat-v2 (proper recipe, beats 2B on 5/6 benchmarks)
- Mark old GPUburnout-3B-75K-Chat as RETIRED with explanatory description
- Switch 1B, 2B, 3B-old, 3B-v2 to llama-cpp-python GGUF Q4_K_M loaders
(3-5x faster CPU inference than fp32 transformers)
- Preload default model at startup to eliminate first-click lag
- Drop transformers + accelerate (no longer needed for large models)

README.md

@@ -16,9 +16,13 @@ Compare language models trained from scratch across multiple seasons:
 
 - **Tiny Shakespeare** (3.2M params) — Character-level, trained on Shakespeare
 - **GPT-2 Small** (134M params) — BPE tokenizer, trained on 2.8B tokens
-- **Llama 1B** (1.04B params) — Llama architecture, trained on 11.8B tokens for $175
-- **GPUburnout-2B-Chat-DPO** (2.06B params) — 2B parameter chat model fine-tuned with DPO on 75K examples
+- **Llama 1B** (1.04B params) — Llama architecture, Chinchilla-optimal at 20.97B tokens
+- **GPUburnout-2B-Chat-DPO** (2.06B params) — 2B chat model with SFT + DPO
+- **GPUburnout-3B-Chat-v2** (3.12B params) — 3B chat model with proper chat-template SFT
+- **GPUburnout-3B-Chat (RETIRED)** — Original 3B with the format-slip bug. Kept as a historical artifact.
+
+Large models run via llama-cpp-python on Q4_K_M GGUF for fast CPU inference.
 
 Built by [Jun Park](https://gpuburnout.com/about/) | [Read the blog](https://gpuburnout.com) | [GitHub](https://github.com/GPUburnout)
 
-Last updated: April 5, 2026
+Last updated: June 21, 2026

app.py

@@ -1,6 +1,11 @@
 """
 GPUburnout Models — Unified Demo
-Compare models trained from scratch: GPUburnout-3M → GPUburnout-134M → GPUburnout-1B
+Compare models trained from scratch, 3.2M → 3.12B parameters.
+
+Architecture notes:
+- s1: tiny custom GPT-2 (3M and 134M) — kept in PyTorch
+- gguf: all Llama-family models (1B, 2B, 3B old + new) run via llama-cpp-python
+  for fast CPU inference. Q4_K_M quantization.
 """
 
 import gc
@@ -37,8 +42,9 @@ MODELS = {
     },
     "GPUburnout-1B (1.04B)": {
         "path": "GPUburnout/GPUburnout-1B-160K",
-        "arch": "s2_hf",
-        "description": "Season 2. Llama architecture, 11.8B tokens, $175 total. Final loss 2.494.",
+        "gguf_file": "GPUburnout-1B-160K-Q4_K_M.gguf",
+        "arch": "gguf",
+        "description": "Season 2 base model. Llama architecture, 20.97B tokens (Chinchilla-optimal), $175 total.",
         "examples": [
             "The capital of France is",
             "In a shocking discovery, scientists found that",
@@ -48,8 +54,9 @@ MODELS = {
     },
     "GPUburnout-2B-75K-Chat-DPO (2.06B)": {
         "path": "GPUburnout/GPUburnout-2B-75K-Chat-DPO",
-        "arch": "s2_hf",
-        "description": "2B parameter chat model fine-tuned with DPO on 75K examples. Optimized for conversational AI.",
+        "gguf_file": "GPUburnout-2B-Chat-DPO-Q4_K_M.gguf",
+        "arch": "gguf",
+        "description": "2B chat model fine-tuned with SFT + DPO on 75K examples. Clean conversational output.",
         "examples": [
             "What is the meaning of life?",
             "Explain quantum computing in simple terms.",
@@ -57,8 +64,40 @@ MODELS = {
             "Tell me about the history of artificial intelligence.",
         ],
     },
+    "GPUburnout-3B-75K-Chat-v2 (3.12B)": {
+        "path": "GPUburnout/GPUburnout-3B-75K-Chat-v2",
+        "gguf_file": "GPUburnout-3B-75K-Chat-v2-Q4_K_M.gguf",
+        "arch": "gguf",
+        "description": (
+            "The 3B chat model with the CORRECT chat-template SFT recipe. "
+            "Beats the 2B on 5/6 benchmarks. See blog Ch.4 for the backstory."
+        ),
+        "examples": [
+            "What is the capital of France?",
+            "Explain how photosynthesis works to a 10-year-old.",
+            "Write a Python function to check if a string is a palindrome.",
+            "What is 247 x 18?",
+        ],
+    },
+    "GPUburnout-3B-75K-Chat (3.12B) — RETIRED": {
+        "path": "GPUburnout/GPUburnout-3B-75K-Chat",
+        "gguf_file": "GPUburnout-3B-75K-Chat-Q4_K_M.gguf",
+        "arch": "gguf",
+        "description": (
+            "RETIRED. Original 3B chat model trained with a buggy plain-concat SFT formatter. "
+            "Loops and underperforms the 2B on ARC-Easy. Kept as a historical artifact — use v2 instead."
+        ),
+        "examples": [
+            "What is the capital of France?",
+            "Write a Python function to check if a string is a palindrome.",
+            "Explain relativity in three sentences.",
+            "Which of two trains traveled farther by noon?",
+        ],
+    },
 }
 
+DEFAULT_MODEL = "GPUburnout-134M (134M)"
+
 # ── Current model state (one at a time) ─────────────────────────────────────
 
 current = {"name": None, "model": None, "tokenizer": None, "config": None}
@@ -85,25 +124,20 @@ def load_model(model_name):
     unload_current()
 
     info = MODELS[model_name]
-    model_dir = info["path"]
-
-    # Handle HuggingFace Hub models differently
-    if info["arch"] == "s2_hf":
-        model, tokenizer = _load_s2_hf(model_dir)
-        config = {"arch": "s2_hf"}
-    else:
-        config_path = os.path.join(model_dir, "config.json")
+    arch = info["arch"]
 
+    if arch == "gguf":
+        model, tokenizer = _load_gguf(info)
+        config = {"arch": "gguf"}
+    elif arch == "s1":
+        config_path = os.path.join(info["path"], "config.json")
         if not os.path.exists(config_path):
-            raise FileNotFoundError(f"Model not found: {model_dir}")
-
+            raise FileNotFoundError(f"Model not found: {info['path']}")
         with open(config_path) as f:
             config = json.load(f)
-
-        if info["arch"] == "s1":
-            model, tokenizer = _load_s1(model_dir, config)
-        else:
-            model, tokenizer = _load_s2(model_dir, config)
+        model, tokenizer = _load_s1(info["path"], config)
+    else:
+        raise ValueError(f"Unknown arch: {arch}")
 
     current["name"] = model_name
     current["model"] = model
@@ -129,10 +163,8 @@ def _load_s1(model_dir, config):
     model.load_state_dict(torch.load(weights_path, map_location="cpu", weights_only=True))
     model.eval()
 
-    # Load tokenizer
     tokenizer_type = config.get("tokenizer_type", "character")
     tokenizer_path = os.path.join(model_dir, "tokenizer.json")
-
     if tokenizer_type == "bpe":
         from s1_tokenizer_bpe import BPETokenizer
         tokenizer = BPETokenizer()
@@ -145,59 +177,21 @@ def _load_s1(model_dir, config):
     return model, tokenizer
 
 
-def _load_s2(model_dir, config):
-    """Load Season 2 Llama style model."""
-    from s2_model import LlamaModel, ModelConfig
+def _load_gguf(info):
+    """Load a GGUF model via llama-cpp-python, pulling from HF Hub on first use."""
+    from llama_cpp import Llama
 
-    model_config = ModelConfig(
-        vocab_size=config.get("vocab_size", 32005),
-        d_model=config.get("d_model", 2048),
-        n_layers=config.get("n_layers", 16),
-        n_heads=config.get("n_heads", 32),
-        n_kv_heads=config.get("n_kv_heads", 8),
-        d_ff=config.get("d_ff", 8192),
-        max_seq_len=config.get("max_seq_len", 2048),
+    repo_id = info["path"]
+    filename = info["gguf_file"]
+    print(f"Loading GGUF {filename} from {repo_id}...")
+    model = Llama.from_pretrained(
+        repo_id=repo_id,
+        filename=filename,
+        n_ctx=2048,
+        n_threads=os.cpu_count() or 2,
+        verbose=False,
     )
-
-    model = LlamaModel(model_config).to("cpu")
-    weights_path = os.path.join(model_dir, "pytorch_model.bin")
-
-    # Download from HF model repo if not present locally (Space LFS limit workaround)
-    if not os.path.exists(weights_path):
-        from huggingface_hub import hf_hub_download
-        print("Downloading Llama 1B weights from GPUburnout/GPUburnout-1B-160K...")
-        weights_path = hf_hub_download(
-            repo_id="GPUburnout/GPUburnout-1B-160K",
-            filename="pytorch_model.bin",
-            local_dir=model_dir,
-        )
-
-    state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
-    model.load_state_dict(state_dict)
-    model.eval()
-
-    # S2 uses HuggingFace tokenizers library
-    from tokenizers import Tokenizer
-    tokenizer = Tokenizer.from_file("tokenizer/bpe_tokenizer.json")
-
-    return model, tokenizer
-
-
-def _load_s2_hf(repo_id):
-    """Load Season 2 model from HuggingFace Hub using transformers."""
-    from transformers import AutoModelForCausalLM, AutoTokenizer
-
-    print(f"Loading model from HuggingFace Hub: {repo_id}...")
-    tokenizer = AutoTokenizer.from_pretrained(repo_id)
-    model = AutoModelForCausalLM.from_pretrained(
-        repo_id,
-        torch_dtype=torch.float32,
-        device_map="cpu",
-        low_cpu_mem_usage=True,
-    )
-    model.eval()
-
-    return model, tokenizer
+    return model, None
 
 
 # ── Generation ──────────────────────────────────────────────────────────────
@@ -224,37 +218,17 @@ def generate_s1(model, tokenizer, config, prompt, max_tokens, temperature, top_k
     return tokenizer.decode(tokens[0].tolist())
 
 
-def generate_s2(model, tokenizer, prompt, max_tokens, temperature, top_k):
-    """Generate text with S2 (Llama) model."""
-    encoded = tokenizer.encode(prompt)
-    input_ids = torch.tensor([encoded.ids], dtype=torch.long)
-
-    with torch.no_grad():
-        output_ids = model.generate(
-            input_ids,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            top_k=top_k if top_k > 0 else None,
-        )
-
-    return tokenizer.decode(output_ids[0].tolist())
-
-
-def generate_s2_hf(model, tokenizer, prompt, max_tokens, temperature, top_k):
-    """Generate text with S2 HuggingFace model."""
-    inputs = tokenizer(prompt, return_tensors="pt")
-
-    with torch.no_grad():
-        output_ids = model.generate(
-            inputs.input_ids,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            top_k=top_k if top_k > 0 else 50,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id,
-        )
-
-    return tokenizer.decode(output_ids[0], skip_special_tokens=True)
+def generate_gguf(model, prompt, max_tokens, temperature, top_k):
+    """Generate text with a GGUF model via llama-cpp-python."""
+    out = model(
+        prompt,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_k=top_k if top_k > 0 else 50,
+        stop=["<|im_end|>", "<|endoftext|>"],
+        echo=True,
+    )
+    return out["choices"][0]["text"]
 
 
 def generate_text(model_name, prompt, max_tokens, temperature, top_k):
@@ -273,12 +247,12 @@ def generate_text(model_name, prompt, max_tokens, temperature, top_k):
     try:
         if info["arch"] == "s1":
             return generate_s1(model, tokenizer, config, prompt, int(max_tokens), temperature, int(top_k))
-        elif info["arch"] == "s2_hf":
-            return generate_s2_hf(model, tokenizer, prompt, int(max_tokens), temperature, int(top_k))
+        elif info["arch"] == "gguf":
+            return generate_gguf(model, prompt, int(max_tokens), temperature, int(top_k))
         else:
-            return generate_s2(model, tokenizer, prompt, int(max_tokens), temperature, int(top_k))
+            return f"Unknown arch: {info['arch']}"
     except Exception as e:
-        return f"Error during generation: {e}\n\nPlease try with different parameters or a different model."
+        return f"Error during generation: {e}\n\nTry different parameters or a different model."
 
 
 def get_status(model_name):
@@ -372,6 +346,15 @@ dark_theme = gr.themes.Base(
     slider_color_dark="#22d3ee",
 )
 
+# ── Preload default model at startup ────────────────────────────────────────
+
+print(f"Preloading default model: {DEFAULT_MODEL}...")
+try:
+    load_model(DEFAULT_MODEL)
+    print(f"  Loaded.")
+except Exception as e:
+    print(f"  Preload failed (will lazy-load on first generate): {e}")
+
 # ── Gradio UI ───────────────────────────────────────────────────────────────
 
 with gr.Blocks(
@@ -383,7 +366,7 @@ with gr.Blocks(
     gr.HTML("""
     <div class="header-text">
         <h1>GPUburnout Models</h1>
-        <p>Compare language models I trained from scratch — from 3.2M to 2 billion parameters.</p>
+        <p>Compare language models I trained from scratch, 3.2M to 3.12 billion parameters.</p>
         <p>
             <a href="https://gpuburnout.com" target="_blank">Read the blog</a> ·
             <a href="https://github.com/GPUburnout" target="_blank">GitHub</a> ·
@@ -396,7 +379,7 @@ with gr.Blocks(
         with gr.Column(scale=1):
             model_selector = gr.Dropdown(
                 choices=list(MODELS.keys()),
-                value="GPUburnout-134M (134M)",
+                value=DEFAULT_MODEL,
                 label="Select Model",
             )
 

requirements.txt

@@ -3,6 +3,5 @@ torch>=2.0.0
 tokenizers>=0.13.0
 gradio>=4.0.0
 huggingface_hub
-transformers>=4.30.0
 numpy
-accelerate
+llama-cpp-python