Upload Mimans M1 architecture code & model checkpoint (mimans-3.01B-tokens)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- README.md +49 -122
- code/__init__.py +11 -0
- code/__pycache__/__init__.cpython-311.pyc +0 -0
- code/__pycache__/config.cpython-311.pyc +0 -0
- code/__pycache__/configuration_mimans.cpython-311.pyc +0 -0
- code/__pycache__/data.cpython-311.pyc +0 -0
- code/__pycache__/dataset.cpython-311.pyc +0 -0
- code/__pycache__/export_hf.cpython-311.pyc +0 -0
- code/__pycache__/modeling_mimans.cpython-311.pyc +0 -0
- code/__pycache__/paths.cpython-311.pyc +0 -0
- code/config.py +431 -0
- code/configuration_mimans.py +46 -0
- code/data.py +340 -0
- code/dataset.py +596 -0
- code/eval/__pycache__/benchmark_evaluator.cpython-311.pyc +0 -0
- code/eval/benchmark_evaluator.py +113 -0
- code/export_hf.py +300 -0
- code/generate.py +416 -0
- code/model/__init__.py +10 -0
- code/model/__pycache__/__init__.cpython-311.pyc +0 -0
- code/model/__pycache__/attention_tilelang.cpython-311.pyc +0 -0
- code/model/__pycache__/attnres.cpython-311.pyc +0 -0
- code/model/__pycache__/block.cpython-311.pyc +0 -0
- code/model/__pycache__/lm_head_tilelang.cpython-311.pyc +0 -0
- code/model/__pycache__/model.cpython-311.pyc +0 -0
- code/model/__pycache__/norm.cpython-311.pyc +0 -0
- code/model/__pycache__/swiglu_tilelang.cpython-311.pyc +0 -0
- code/model/__pycache__/tuned.cpython-311.pyc +0 -0
- code/model/attention_tilelang.py +788 -0
- code/model/attnres.py +152 -0
- code/model/block.py +222 -0
- code/model/init_study.py +216 -0
- code/model/lm_head_tilelang.py +596 -0
- code/model/model.py +311 -0
- code/model/norm.py +172 -0
- code/model/swiglu_tilelang.py +644 -0
- code/model/tuned.py +104 -0
- code/modeling_mimans.py +270 -0
- code/paths.py +89 -0
- code/tokenizer/__init__.py +4 -0
- code/tokenizer/__pycache__/__init__.cpython-311.pyc +0 -0
- code/tokenizer/__pycache__/prepare_data.cpython-311.pyc +0 -0
- code/tokenizer/__pycache__/reshuffle_dataset.cpython-311.pyc +0 -0
- code/tokenizer/__pycache__/train_custom_tokenizer.cpython-311.pyc +0 -0
- code/tokenizer/__pycache__/train_tokenizer.cpython-311.pyc +0 -0
- code/tokenizer/data.py +69 -0
- code/tokenizer/prepare_data.py +196 -0
- code/tokenizer/reshuffle_dataset.py +193 -0
- code/tokenizer/train_custom_tokenizer.py +104 -0
- code/tokenizer/train_tokenizer.py +467 -0
README.md
CHANGED
|
@@ -1,122 +1,49 @@
|
|
| 1 |
-
---
|
| 2 |
-
language:
|
| 3 |
-
- en
|
| 4 |
-
- code
|
| 5 |
-
license: apache-2.0
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
-
|
| 9 |
-
-
|
| 10 |
-
-
|
| 11 |
-
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
-
|
| 18 |
-
|
| 19 |
-
#
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
### 1. Installation
|
| 51 |
-
```bash
|
| 52 |
-
pip install torch safetensors tokenizers huggingface_hub
|
| 53 |
-
```
|
| 54 |
-
|
| 55 |
-
### 2. Command-Line Generation
|
| 56 |
-
|
| 57 |
-
#### Clean Code Completion (Greedy Decoding / Temp 0)
|
| 58 |
-
```bash
|
| 59 |
-
python generate.py --prompt "def add(a, b):\n return" --temp 0
|
| 60 |
-
```
|
| 61 |
-
**Output:**
|
| 62 |
-
```python
|
| 63 |
-
def add(a, b):
|
| 64 |
-
return a + b
|
| 65 |
-
|
| 66 |
-
def subtract(a, b):
|
| 67 |
-
return a - b
|
| 68 |
-
|
| 69 |
-
def multiply(a, b):
|
| 70 |
-
return a * b
|
| 71 |
-
|
| 72 |
-
def divide(a, b):
|
| 73 |
-
return a / b
|
| 74 |
-
```
|
| 75 |
-
|
| 76 |
-
#### Natural Language / Creative Sampling (Temp 0.7)
|
| 77 |
-
```bash
|
| 78 |
-
python generate.py --prompt "The future of artificial intelligence and computer science will" --max-tokens 150 --temp 0.78
|
| 79 |
-
```
|
| 80 |
-
|
| 81 |
-
---
|
| 82 |
-
|
| 83 |
-
### 3. Python API
|
| 84 |
-
|
| 85 |
-
```python
|
| 86 |
-
from generate import load_model, generate
|
| 87 |
-
|
| 88 |
-
# Load model & tokenizer directly from local folder or Hugging Face repo
|
| 89 |
-
model, tokenizer = load_model("ankushthakurr09/MimansM1_v1", device="cuda")
|
| 90 |
-
|
| 91 |
-
# 1. Greedy code generation
|
| 92 |
-
code = generate(
|
| 93 |
-
model,
|
| 94 |
-
tokenizer,
|
| 95 |
-
prompt="def is_even(n):\n return",
|
| 96 |
-
max_tokens=50,
|
| 97 |
-
temperature=0.0
|
| 98 |
-
)
|
| 99 |
-
print(code)
|
| 100 |
-
|
| 101 |
-
# 2. Sampled generation with repetition penalty
|
| 102 |
-
text = generate(
|
| 103 |
-
model,
|
| 104 |
-
tokenizer,
|
| 105 |
-
prompt="def quicksort(arr):",
|
| 106 |
-
max_tokens=100,
|
| 107 |
-
temperature=0.7,
|
| 108 |
-
repetition_penalty=1.15
|
| 109 |
-
)
|
| 110 |
-
print(text)
|
| 111 |
-
```
|
| 112 |
-
|
| 113 |
-
---
|
| 114 |
-
|
| 115 |
-
## Training Status & Hardware
|
| 116 |
-
|
| 117 |
-
- **Hardware:** 1x NVIDIA GeForce RTX 5090 (32 GB VRAM)
|
| 118 |
-
- **Precision:** `bfloat16` with fused kernels
|
| 119 |
-
- **Optimizer:** Muon (matrix layers) + AdamW (embeddings and 1D norms)
|
| 120 |
-
- **Batch Size:** 524,288 tokens per step (Micro-batch 8, Gradient Accumulation 16)
|
| 121 |
-
- **Current Validation Loss:** `2.180` (Validation Perplexity: `8.85`)
|
| 122 |
-
- **Total Tokens Trained:** `2,001,731,584` (Continuous active training)
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
+
- code
|
| 5 |
+
license: apache-2.0
|
| 6 |
+
tags:
|
| 7 |
+
- code
|
| 8 |
+
- mimans
|
| 9 |
+
- causal-lm
|
| 10 |
+
- blackwell
|
| 11 |
+
- tilelang
|
| 12 |
+
pipeline_tag: text-generation
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# Mimans M1 (514M) - 3.01B Tokens Milestone
|
| 16 |
+
|
| 17 |
+
**Mimans M1** is a 514M parameter decoder-only transformer pretrained from scratch on code, math, and technical text using **TileLang** custom Blackwell GPU kernels and **Muon + AdamW** hybrid optimization.
|
| 18 |
+
|
| 19 |
+
## Model Summary
|
| 20 |
+
- **Parameters:** 514,345,526 (~514M)
|
| 21 |
+
- **Architecture:** GQA (10:2), SwiGLU FFN, AttnRes Skip Gating, RMSNorm
|
| 22 |
+
- **Context Length:** 4,096 tokens (max 8,192, RoPE $\theta = 500,000$)
|
| 23 |
+
- **Vocabulary:** 49,152 (Byte-level BPE with FIM support)
|
| 24 |
+
- **Training Tokens:** **3.01B Tokens** (Global Step 5,746)
|
| 25 |
+
- **Latest Loss:** **2.0100**
|
| 26 |
+
- **Hardware:** NVIDIA GeForce RTX 5090 (Blackwell `sm_120`)
|
| 27 |
+
|
| 28 |
+
## Usage & Generation
|
| 29 |
+
|
| 30 |
+
### Loading with Hugging Face Transformers:
|
| 31 |
+
|
| 32 |
+
```python
|
| 33 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 34 |
+
|
| 35 |
+
repo_id = "ankushthakurr09/MimansM1_v1"
|
| 36 |
+
model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True)
|
| 37 |
+
tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
|
| 38 |
+
|
| 39 |
+
prompt = "def fibonacci(n):"
|
| 40 |
+
inputs = tokenizer(prompt, return_tensors="pt")
|
| 41 |
+
outputs = model.generate(**inputs, max_new_tokens=50)
|
| 42 |
+
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
### Running Standalone Generation Script:
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
python generate.py --prompt "def hello_world():" --max-tokens 50
|
| 49 |
+
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
code/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mimans: a 514M parameter code model with hand written tilelang kernels.
|
| 2 |
+
|
| 3 |
+
mimans.paths every path the project writes to, all on D:
|
| 4 |
+
mimans.data the on disk token format: FIM, shards, stages, loader
|
| 5 |
+
mimans.model the architecture, embedding through to logits
|
| 6 |
+
mimans.tokenizer training the byte level BPE the model is built around
|
| 7 |
+
mimans.training the optimizer, the loop, and the preflight
|
| 8 |
+
|
| 9 |
+
paths is imported before tilelang everywhere it matters, because tilelang reads
|
| 10 |
+
its cache location once at import time.
|
| 11 |
+
"""
|
code/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (743 Bytes). View file
|
|
|
code/__pycache__/config.cpython-311.pyc
ADDED
|
Binary file (25.2 kB). View file
|
|
|
code/__pycache__/configuration_mimans.cpython-311.pyc
ADDED
|
Binary file (1.85 kB). View file
|
|
|
code/__pycache__/data.cpython-311.pyc
ADDED
|
Binary file (30.5 kB). View file
|
|
|
code/__pycache__/dataset.cpython-311.pyc
ADDED
|
Binary file (41.2 kB). View file
|
|
|
code/__pycache__/export_hf.cpython-311.pyc
ADDED
|
Binary file (15.5 kB). View file
|
|
|
code/__pycache__/modeling_mimans.cpython-311.pyc
ADDED
|
Binary file (21.8 kB). View file
|
|
|
code/__pycache__/paths.cpython-311.pyc
ADDED
|
Binary file (4.22 kB). View file
|
|
|
code/config.py
ADDED
|
@@ -0,0 +1,431 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One configuration object for a single-GPU run. Nothing else holds constants.
|
| 2 |
+
|
| 3 |
+
Everything the trainer, the preflight and the checkpointer need lives in
|
| 4 |
+
`RunConfig`. It is written to the output directory as JSON on every run, and
|
| 5 |
+
recorded inside every checkpoint, so a checkpoint always says what produced it.
|
| 6 |
+
|
| 7 |
+
Paths are resolved, never hard-coded. The dataset is looked for in a search
|
| 8 |
+
order that puts the user's Desktop ahead of the repo's historical location, so
|
| 9 |
+
dropping shards into `Desktop\\mimanus-dataset` is enough to switch to them.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import subprocess
|
| 16 |
+
from dataclasses import asdict, dataclass, field, fields
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
# The number of GPUs this system is built for. Stated once, as a constant,
|
| 20 |
+
# rather than threaded through as a parameter that nothing would ever vary.
|
| 21 |
+
NUM_GPUS = 1
|
| 22 |
+
|
| 23 |
+
DATASET_DIR_NAME = "mimanus-dataset"
|
| 24 |
+
OUTPUT_DIR_NAME = "mimanus-training"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def desktop_dir() -> Path:
|
| 28 |
+
"""The user's Desktop, from the environment rather than a literal path.
|
| 29 |
+
|
| 30 |
+
USERPROFILE is what Windows sets; HOME covers the case of running this
|
| 31 |
+
under a POSIX-ish shell. OneDrive redirection is checked because a
|
| 32 |
+
redirected Desktop is the common case on managed Windows boxes and the
|
| 33 |
+
plain `%USERPROFILE%\\Desktop` would not exist there.
|
| 34 |
+
"""
|
| 35 |
+
home = Path(os.environ.get("USERPROFILE") or os.environ.get("HOME") or Path.home())
|
| 36 |
+
onedrive = os.environ.get("OneDrive") or os.environ.get("OneDriveConsumer")
|
| 37 |
+
candidates = [home / "Desktop"]
|
| 38 |
+
if onedrive:
|
| 39 |
+
candidates.insert(0, Path(onedrive) / "Desktop")
|
| 40 |
+
for path in candidates:
|
| 41 |
+
if path.is_dir():
|
| 42 |
+
return path
|
| 43 |
+
return home / "Desktop"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _has_shards(path: Path) -> bool:
|
| 47 |
+
"""Anything that looks like token data, at any depth of one level."""
|
| 48 |
+
if not path.is_dir():
|
| 49 |
+
return False
|
| 50 |
+
for pattern in ("*.bin", "*.npy", "*/*.bin", "*/*.npy"):
|
| 51 |
+
if next(path.glob(pattern), None) is not None:
|
| 52 |
+
return True
|
| 53 |
+
return False
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def default_dataset_path() -> Path:
|
| 57 |
+
"""First location that actually holds shards, else the Desktop location.
|
| 58 |
+
|
| 59 |
+
Returning the Desktop path when nothing is found is deliberate: the
|
| 60 |
+
preflight then reports "this is where I looked and it is empty", which is
|
| 61 |
+
more useful than silently succeeding against stale data somewhere else.
|
| 62 |
+
"""
|
| 63 |
+
env = os.environ.get("MIMANS_DATASET")
|
| 64 |
+
if env:
|
| 65 |
+
return Path(env)
|
| 66 |
+
desktop = desktop_dir()
|
| 67 |
+
candidates = [
|
| 68 |
+
Path("D:/Mimans-dataset"),
|
| 69 |
+
Path("D:/Mimans-dataset/shards"),
|
| 70 |
+
Path("D:/Mimans-dataset/code").parent if Path("D:/Mimans-dataset/code").exists() else Path("D:/Mimans-dataset"),
|
| 71 |
+
Path("D:/mimans-dataset"),
|
| 72 |
+
Path("D:/mimans-dataset/shards"),
|
| 73 |
+
desktop / DATASET_DIR_NAME,
|
| 74 |
+
desktop / "Mimans-dataset", # what is actually on this machine
|
| 75 |
+
Path("D:/mimans/shards"), # where prepare_data.py has been writing
|
| 76 |
+
]
|
| 77 |
+
for path in candidates:
|
| 78 |
+
if _has_shards(path):
|
| 79 |
+
return path
|
| 80 |
+
return candidates[0]
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def default_output_dir() -> Path:
|
| 84 |
+
"""Where checkpoints go.
|
| 85 |
+
|
| 86 |
+
The brief suggests `Desktop/mimanus-training`. On this machine Desktop is
|
| 87 |
+
on C:, which has under 7 GB free, and three checkpoints of a 514M model are
|
| 88 |
+
about 13 GB -- the run would die of a full system drive somewhere around
|
| 89 |
+
hour two. So the default is the roomiest fixed drive that already holds
|
| 90 |
+
this project's data, and the Desktop location is used only when it is the
|
| 91 |
+
best available. Override with --output-dir or MIMANS_OUTPUT.
|
| 92 |
+
"""
|
| 93 |
+
env = os.environ.get("MIMANS_OUTPUT")
|
| 94 |
+
if env:
|
| 95 |
+
return Path(env)
|
| 96 |
+
import shutil
|
| 97 |
+
desktop = desktop_dir()
|
| 98 |
+
preferred = Path("D:/") / OUTPUT_DIR_NAME
|
| 99 |
+
try:
|
| 100 |
+
if preferred.drive and Path(preferred.drive + "/").exists():
|
| 101 |
+
free_gb = shutil.disk_usage(preferred.drive + "/").free / 1e9
|
| 102 |
+
if free_gb > 60:
|
| 103 |
+
return preferred
|
| 104 |
+
except OSError:
|
| 105 |
+
pass
|
| 106 |
+
return desktop / OUTPUT_DIR_NAME
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def git_version() -> str:
|
| 110 |
+
"""Short commit plus a dirty marker, or 'unknown' outside a checkout."""
|
| 111 |
+
try:
|
| 112 |
+
root = Path(__file__).resolve().parents[2]
|
| 113 |
+
head = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
|
| 114 |
+
cwd=root, capture_output=True, text=True, timeout=10)
|
| 115 |
+
if head.returncode != 0:
|
| 116 |
+
return "unknown"
|
| 117 |
+
commit = head.stdout.strip()
|
| 118 |
+
dirty = subprocess.run(["git", "status", "--porcelain"],
|
| 119 |
+
cwd=root, capture_output=True, text=True, timeout=15)
|
| 120 |
+
return commit + ("-dirty" if dirty.stdout.strip() else "")
|
| 121 |
+
except Exception:
|
| 122 |
+
return "unknown"
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
@dataclass
|
| 126 |
+
class RunConfig:
|
| 127 |
+
# ---- data ----------------------------------------------------------
|
| 128 |
+
dataset_path: Path = field(default_factory=default_dataset_path)
|
| 129 |
+
tokenizer_path: Path = field(default_factory=lambda: Path("D:/mimans/tokenizer/tokenizer.json"))
|
| 130 |
+
output_directory: Path = field(default_factory=default_output_dir)
|
| 131 |
+
# Sequences held out of training, per source, for validation. Small on
|
| 132 |
+
# purpose: validation must not eat the training budget.
|
| 133 |
+
val_sequences: int = 128
|
| 134 |
+
mixture: dict = field(default_factory=lambda: {"code": 0.60, "text": 0.30, "math": 0.10})
|
| 135 |
+
|
| 136 |
+
# ---- model ---------------------------------------------------------
|
| 137 |
+
model_layers: int = 27
|
| 138 |
+
# The repo's own stage table raises theta to 500k when the context reaches
|
| 139 |
+
# 8192, because 10k rotates too fast to place tokens that far apart. Since
|
| 140 |
+
# this run starts at 8192, it starts at 500k.
|
| 141 |
+
rope_theta: float = 500_000.0
|
| 142 |
+
attn_backend: str = "tilelang"
|
| 143 |
+
ffn_backend: str = "tilelang"
|
| 144 |
+
head_backend: str = "tilelang"
|
| 145 |
+
# Measured on this box: without it, micro batch 1 at 8192 wants ~45 GB and
|
| 146 |
+
# Windows WDDM answers by spilling into shared system memory rather than
|
| 147 |
+
# raising OOM, which turns a 3 s step into a 99 s one. With it, the same
|
| 148 |
+
# step fits in VRAM. None of the maths changes. See
|
| 149 |
+
# CodeModel._checkpointed_block for why the gate is the thing that costs.
|
| 150 |
+
activation_checkpointing: bool = True
|
| 151 |
+
|
| 152 |
+
# ---- shape ---------------------------------------------------------
|
| 153 |
+
sequence_length: int = 4096
|
| 154 |
+
micro_batch_size: int = 0 # 0 = calibrate on this GPU
|
| 155 |
+
gradient_accumulation_steps: int = 0 # 0 = derive from the target below
|
| 156 |
+
# 524,288 tokens per optimizer step is the repo's planning figure and is
|
| 157 |
+
# kept so the LR schedule means the same thing across sessions.
|
| 158 |
+
target_effective_tokens: int = 524_288
|
| 159 |
+
|
| 160 |
+
# ---- optimization --------------------------------------------------
|
| 161 |
+
learning_rate: float = 0.02 # Muon, on the 2-D matrices
|
| 162 |
+
adamw_learning_rate: float = 3e-3 # everything else
|
| 163 |
+
weight_decay: float = 0.1
|
| 164 |
+
warmup: int = 2_000 # optimizer steps
|
| 165 |
+
grad_clip: float = 1.0
|
| 166 |
+
total_tokens: float = 40e9 # horizon the LR schedule decays over
|
| 167 |
+
precision: str = "bf16" # bf16 | fp16 | fp32
|
| 168 |
+
seed: int = 0
|
| 169 |
+
|
| 170 |
+
# ---- time and checkpoints ------------------------------------------
|
| 171 |
+
max_hours: float = 0.0 # 0.0 means continuous training (no wall-clock hour cap)
|
| 172 |
+
# Stop this long before the deadline, on top of refusing to start a step
|
| 173 |
+
# that would not finish in time. Covers the final checkpoint write.
|
| 174 |
+
safety_margin_minutes: float = 5.0
|
| 175 |
+
checkpoint_interval_minutes: float = 30.0
|
| 176 |
+
keep_last_checkpoints: int = 3
|
| 177 |
+
resume: str = "auto" # auto | none | <path>
|
| 178 |
+
validation_interval_minutes: float = 15.0
|
| 179 |
+
validation_batches: int = 8
|
| 180 |
+
log_every: int = 1
|
| 181 |
+
|
| 182 |
+
# ---- performance ---------------------------------------------------
|
| 183 |
+
tf32: bool = True
|
| 184 |
+
compile: bool = False
|
| 185 |
+
autotune: bool = False
|
| 186 |
+
prefetch_workers: int = 1 # background threads staging batches
|
| 187 |
+
pin_memory: bool = True
|
| 188 |
+
|
| 189 |
+
# ---- provenance ----------------------------------------------------
|
| 190 |
+
git_version: str = field(default_factory=git_version)
|
| 191 |
+
|
| 192 |
+
def __post_init__(self):
|
| 193 |
+
self.dataset_path = Path(self.dataset_path)
|
| 194 |
+
self.tokenizer_path = Path(self.tokenizer_path)
|
| 195 |
+
self.output_directory = Path(self.output_directory)
|
| 196 |
+
if self.precision not in ("bf16", "fp16", "fp32"):
|
| 197 |
+
raise ValueError(f"precision must be bf16, fp16 or fp32, got {self.precision!r}")
|
| 198 |
+
if self.sequence_length < 16:
|
| 199 |
+
raise ValueError(f"sequence_length {self.sequence_length} is too short to train")
|
| 200 |
+
|
| 201 |
+
# ---- derived -------------------------------------------------------
|
| 202 |
+
|
| 203 |
+
@property
|
| 204 |
+
def target_sequences_per_step(self) -> int:
|
| 205 |
+
return max(1, self.target_effective_tokens // self.sequence_length)
|
| 206 |
+
|
| 207 |
+
def accumulation_for(self, micro_batch: int) -> int:
|
| 208 |
+
"""Accumulation steps that land closest to the target global batch.
|
| 209 |
+
|
| 210 |
+
Deliberately not required to divide evenly: if the GPU only fits a
|
| 211 |
+
micro batch of 3, forcing an exact 64-sequence global batch would mean
|
| 212 |
+
rejecting 3 and using 2. Getting 63 sequences and saying so is better
|
| 213 |
+
than wasting a third of the batch.
|
| 214 |
+
"""
|
| 215 |
+
return max(1, round(self.target_sequences_per_step / max(1, micro_batch)))
|
| 216 |
+
|
| 217 |
+
@property
|
| 218 |
+
def effective_batch_size(self) -> int:
|
| 219 |
+
"""In sequences. micro x accumulation x GPUs, and GPUs is 1 here."""
|
| 220 |
+
return self.micro_batch_size * self.gradient_accumulation_steps * NUM_GPUS
|
| 221 |
+
|
| 222 |
+
@property
|
| 223 |
+
def effective_batch_tokens(self) -> int:
|
| 224 |
+
return self.effective_batch_size * self.sequence_length
|
| 225 |
+
|
| 226 |
+
@property
|
| 227 |
+
def total_steps(self) -> int:
|
| 228 |
+
"""Optimizer steps in the whole (multi-session) plan, for the LR curve."""
|
| 229 |
+
return max(1, int(self.total_tokens // max(1, self.effective_batch_tokens)))
|
| 230 |
+
|
| 231 |
+
@property
|
| 232 |
+
def checkpoint_dir(self) -> Path:
|
| 233 |
+
return self.output_directory / "checkpoints"
|
| 234 |
+
|
| 235 |
+
@property
|
| 236 |
+
def log_dir(self) -> Path:
|
| 237 |
+
return self.output_directory / "logs"
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def config_dir(self) -> Path:
|
| 241 |
+
return self.output_directory / "configs"
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def metrics_dir(self) -> Path:
|
| 245 |
+
return self.output_directory / "metrics"
|
| 246 |
+
|
| 247 |
+
def make_output_dirs(self):
|
| 248 |
+
for path in (self.checkpoint_dir, self.log_dir, self.config_dir, self.metrics_dir):
|
| 249 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 250 |
+
|
| 251 |
+
def torch_dtype(self):
|
| 252 |
+
import torch
|
| 253 |
+
return {"bf16": torch.bfloat16, "fp16": torch.float16,
|
| 254 |
+
"fp32": torch.float32}[self.precision]
|
| 255 |
+
|
| 256 |
+
def model_config(self):
|
| 257 |
+
"""A ModelConfig consistent with this run.
|
| 258 |
+
|
| 259 |
+
max_seq_len must cover sequence_length because the RoPE table is built
|
| 260 |
+
once to that length; asking for more at runtime would index past it.
|
| 261 |
+
"""
|
| 262 |
+
from mimans.model.model import ModelConfig
|
| 263 |
+
return ModelConfig(
|
| 264 |
+
n_layers=self.model_layers,
|
| 265 |
+
rope_theta=self.rope_theta,
|
| 266 |
+
max_seq_len=max(8192, self.sequence_length),
|
| 267 |
+
attn_backend=self.attn_backend,
|
| 268 |
+
ffn_backend=self.ffn_backend,
|
| 269 |
+
head_backend=self.head_backend,
|
| 270 |
+
gradient_checkpointing=self.activation_checkpointing,
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
def train_config(self):
|
| 274 |
+
"""The existing TrainConfig, filled from this one.
|
| 275 |
+
|
| 276 |
+
The repo's Trainer is reused rather than rewritten, and it wants its
|
| 277 |
+
own config object. global_batch_tokens is derived from the measured
|
| 278 |
+
micro batch so its internal divisibility assertion always holds.
|
| 279 |
+
"""
|
| 280 |
+
from mimans.training.train import TrainConfig
|
| 281 |
+
return TrainConfig(
|
| 282 |
+
total_tokens=self.total_tokens,
|
| 283 |
+
global_batch_tokens=self.effective_batch_tokens,
|
| 284 |
+
micro_batch=self.micro_batch_size,
|
| 285 |
+
seq_len=self.sequence_length,
|
| 286 |
+
seed=self.seed,
|
| 287 |
+
log_every=self.log_every,
|
| 288 |
+
save_every=0, # the runner checkpoints on the clock
|
| 289 |
+
save_every_minutes=self.checkpoint_interval_minutes,
|
| 290 |
+
session_hours=self.max_hours,
|
| 291 |
+
keep_last=self.keep_last_checkpoints,
|
| 292 |
+
milestone_steps=0,
|
| 293 |
+
out_dir=str(self.checkpoint_dir),
|
| 294 |
+
)
|
| 295 |
+
|
| 296 |
+
def optim_config(self):
|
| 297 |
+
from mimans.training.optim import OptimConfig
|
| 298 |
+
return OptimConfig(
|
| 299 |
+
muon_lr=self.learning_rate,
|
| 300 |
+
muon_weight_decay=self.weight_decay,
|
| 301 |
+
adamw_lr=self.adamw_learning_rate,
|
| 302 |
+
grad_clip=self.grad_clip,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
def schedule_config(self):
|
| 306 |
+
from mimans.training.optim import ScheduleConfig
|
| 307 |
+
return ScheduleConfig(total_steps=self.total_steps,
|
| 308 |
+
warmup_steps=self.warmup)
|
| 309 |
+
|
| 310 |
+
# ---- serialisation -------------------------------------------------
|
| 311 |
+
|
| 312 |
+
def to_dict(self) -> dict:
|
| 313 |
+
out = asdict(self)
|
| 314 |
+
for key, value in out.items():
|
| 315 |
+
if isinstance(value, Path):
|
| 316 |
+
out[key] = str(value)
|
| 317 |
+
out["_derived"] = {
|
| 318 |
+
"num_gpus": NUM_GPUS,
|
| 319 |
+
"effective_batch_size": self.effective_batch_size,
|
| 320 |
+
"effective_batch_tokens": self.effective_batch_tokens,
|
| 321 |
+
"total_steps": self.total_steps,
|
| 322 |
+
}
|
| 323 |
+
return out
|
| 324 |
+
|
| 325 |
+
def save(self, path: Path = None) -> Path:
|
| 326 |
+
path = Path(path) if path else self.config_dir / "run_config.json"
|
| 327 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 328 |
+
path.write_text(json.dumps(self.to_dict(), indent=2, sort_keys=True))
|
| 329 |
+
return path
|
| 330 |
+
|
| 331 |
+
@classmethod
|
| 332 |
+
def from_dict(cls, data: dict) -> "RunConfig":
|
| 333 |
+
known = {f.name for f in fields(cls)}
|
| 334 |
+
return cls(**{k: v for k, v in data.items() if k in known})
|
| 335 |
+
|
| 336 |
+
@classmethod
|
| 337 |
+
def load(cls, path: Path) -> "RunConfig":
|
| 338 |
+
return cls.from_dict(json.loads(Path(path).read_text()))
|
| 339 |
+
|
| 340 |
+
def summary_lines(self) -> list:
|
| 341 |
+
return [
|
| 342 |
+
f"sequence_length {self.sequence_length:,}",
|
| 343 |
+
f"micro_batch_size {self.micro_batch_size}",
|
| 344 |
+
f"gradient_accumulation_steps {self.gradient_accumulation_steps}",
|
| 345 |
+
f"effective_batch_size {self.effective_batch_size} sequences"
|
| 346 |
+
f" ({self.effective_batch_tokens:,} tokens)",
|
| 347 |
+
f"precision {self.precision}",
|
| 348 |
+
f"dataset_path {self.dataset_path}",
|
| 349 |
+
f"output_directory {self.output_directory}",
|
| 350 |
+
]
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def add_arguments(parser):
|
| 354 |
+
"""CLI flags shared by train.py and preflight.py, so they cannot drift."""
|
| 355 |
+
d = RunConfig()
|
| 356 |
+
parser.add_argument("--hours", type=float, default=d.max_hours,
|
| 357 |
+
help=f"wall-clock training limit (default {d.max_hours:g})")
|
| 358 |
+
parser.add_argument("--seq-len", type=int, default=d.sequence_length,
|
| 359 |
+
help=f"sequence length (default {d.sequence_length})")
|
| 360 |
+
parser.add_argument("--micro-batch", type=int, default=None,
|
| 361 |
+
help="micro batch size; omit to calibrate on this GPU")
|
| 362 |
+
parser.add_argument("--grad-accum", type=int, default=None,
|
| 363 |
+
help="accumulation steps; omit to derive from the target batch")
|
| 364 |
+
parser.add_argument("--dataset-path", type=Path, default=None,
|
| 365 |
+
help=f"shard root (default: searched, currently {d.dataset_path})")
|
| 366 |
+
parser.add_argument("--tokenizer", type=Path, default=None,
|
| 367 |
+
help=f"tokenizer.json (default {d.tokenizer_path})")
|
| 368 |
+
parser.add_argument("--output-dir", type=Path, default=None,
|
| 369 |
+
help=f"checkpoints/logs/configs/metrics (default {d.output_directory})")
|
| 370 |
+
parser.add_argument("--precision", choices=("bf16", "fp16", "fp32"), default=d.precision)
|
| 371 |
+
parser.add_argument("--resume", default=d.resume,
|
| 372 |
+
help="auto | none | path to a checkpoint directory")
|
| 373 |
+
parser.add_argument("--checkpoint-minutes", type=float,
|
| 374 |
+
default=d.checkpoint_interval_minutes)
|
| 375 |
+
parser.add_argument("--keep-last", type=int, default=d.keep_last_checkpoints)
|
| 376 |
+
parser.add_argument("--safety-margin-minutes", type=float,
|
| 377 |
+
default=d.safety_margin_minutes)
|
| 378 |
+
parser.add_argument("--eval-interval-minutes", type=float,
|
| 379 |
+
default=d.validation_interval_minutes,
|
| 380 |
+
help="0 disables validation")
|
| 381 |
+
parser.add_argument("--eval-batches", type=int, default=d.validation_batches)
|
| 382 |
+
parser.add_argument("--log-every", type=int, default=d.log_every)
|
| 383 |
+
parser.add_argument("--seed", type=int, default=d.seed)
|
| 384 |
+
parser.add_argument("--layers", type=int, default=d.model_layers)
|
| 385 |
+
parser.add_argument("--rope-theta", type=float, default=d.rope_theta)
|
| 386 |
+
parser.add_argument("--prefetch-workers", type=int, default=d.prefetch_workers,
|
| 387 |
+
help="background batch-staging threads (0 = synchronous)")
|
| 388 |
+
parser.add_argument("--no-pin-memory", action="store_true")
|
| 389 |
+
parser.add_argument("--no-tf32", action="store_true")
|
| 390 |
+
parser.add_argument("--no-activation-checkpointing", action="store_true",
|
| 391 |
+
help="store block activations instead of recomputing "
|
| 392 |
+
"them; needs far more VRAM at long context")
|
| 393 |
+
parser.add_argument("--compile", action="store_true",
|
| 394 |
+
help="torch.compile the model; off by default, see AUDIT.md")
|
| 395 |
+
parser.add_argument("--autotune", action="store_true",
|
| 396 |
+
help="re-tune tilelang kernels; slow, see AUDIT.md")
|
| 397 |
+
return parser
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def from_args(args) -> RunConfig:
|
| 401 |
+
"""Build a RunConfig from parsed CLI arguments."""
|
| 402 |
+
cfg = RunConfig(
|
| 403 |
+
max_hours=args.hours,
|
| 404 |
+
sequence_length=args.seq_len,
|
| 405 |
+
micro_batch_size=args.micro_batch or 0,
|
| 406 |
+
gradient_accumulation_steps=args.grad_accum or 0,
|
| 407 |
+
precision=args.precision,
|
| 408 |
+
resume=args.resume,
|
| 409 |
+
checkpoint_interval_minutes=args.checkpoint_minutes,
|
| 410 |
+
keep_last_checkpoints=args.keep_last,
|
| 411 |
+
safety_margin_minutes=args.safety_margin_minutes,
|
| 412 |
+
validation_interval_minutes=args.eval_interval_minutes,
|
| 413 |
+
validation_batches=args.eval_batches,
|
| 414 |
+
log_every=args.log_every,
|
| 415 |
+
seed=args.seed,
|
| 416 |
+
model_layers=args.layers,
|
| 417 |
+
rope_theta=args.rope_theta,
|
| 418 |
+
prefetch_workers=args.prefetch_workers,
|
| 419 |
+
pin_memory=not args.no_pin_memory,
|
| 420 |
+
tf32=not args.no_tf32,
|
| 421 |
+
compile=args.compile,
|
| 422 |
+
autotune=args.autotune,
|
| 423 |
+
activation_checkpointing=not args.no_activation_checkpointing,
|
| 424 |
+
)
|
| 425 |
+
if args.dataset_path:
|
| 426 |
+
cfg.dataset_path = Path(args.dataset_path)
|
| 427 |
+
if args.tokenizer:
|
| 428 |
+
cfg.tokenizer_path = Path(args.tokenizer)
|
| 429 |
+
if args.output_dir:
|
| 430 |
+
cfg.output_directory = Path(args.output_dir)
|
| 431 |
+
return cfg
|
code/configuration_mimans.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HuggingFace compatible configuration for Mimans M1."""
|
| 2 |
+
from transformers import PretrainedConfig
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class MimansConfig(PretrainedConfig):
|
| 6 |
+
model_type = "mimans"
|
| 7 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 8 |
+
|
| 9 |
+
def __init__(
|
| 10 |
+
self,
|
| 11 |
+
vocab_size=49152,
|
| 12 |
+
hidden_size=1280,
|
| 13 |
+
intermediate_size=3328,
|
| 14 |
+
num_hidden_layers=27,
|
| 15 |
+
num_attention_heads=10,
|
| 16 |
+
num_key_value_heads=2,
|
| 17 |
+
max_position_embeddings=8192,
|
| 18 |
+
rms_norm_eps=1e-5,
|
| 19 |
+
rope_theta=500000.0,
|
| 20 |
+
initializer_range=0.02,
|
| 21 |
+
bos_token_id=0,
|
| 22 |
+
eos_token_id=0,
|
| 23 |
+
pad_token_id=6,
|
| 24 |
+
tie_word_embeddings=True,
|
| 25 |
+
use_attnres=True,
|
| 26 |
+
**kwargs,
|
| 27 |
+
):
|
| 28 |
+
self.vocab_size = vocab_size
|
| 29 |
+
self.hidden_size = hidden_size
|
| 30 |
+
self.intermediate_size = intermediate_size
|
| 31 |
+
self.num_hidden_layers = num_hidden_layers
|
| 32 |
+
self.num_attention_heads = num_attention_heads
|
| 33 |
+
self.num_key_value_heads = num_key_value_heads
|
| 34 |
+
self.head_dim = hidden_size // num_attention_heads
|
| 35 |
+
self.max_position_embeddings = max_position_embeddings
|
| 36 |
+
self.rms_norm_eps = rms_norm_eps
|
| 37 |
+
self.rope_theta = rope_theta
|
| 38 |
+
self.initializer_range = initializer_range
|
| 39 |
+
self.use_attnres = use_attnres
|
| 40 |
+
super().__init__(
|
| 41 |
+
bos_token_id=bos_token_id,
|
| 42 |
+
eos_token_id=eos_token_id,
|
| 43 |
+
pad_token_id=pad_token_id,
|
| 44 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 45 |
+
**kwargs,
|
| 46 |
+
)
|
code/data.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from dataclasses import dataclass, field
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
FIM_PREFIX = "<|fim_prefix|>"
|
| 11 |
+
FIM_MIDDLE = "<|fim_middle|>"
|
| 12 |
+
FIM_SUFFIX = "<|fim_suffix|>"
|
| 13 |
+
REPO_NAME = "<|repo_name|>"
|
| 14 |
+
FILE_SEP = "<|file_sep|>"
|
| 15 |
+
ENDOFTEXT = "<|endoftext|>"
|
| 16 |
+
|
| 17 |
+
TOKEN_DTYPE = np.uint16
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def apply_fim(text: str, rng: np.random.Generator, rate: float = 0.5):
|
| 21 |
+
"""Character level PSM transform. Returns the text unchanged with prob 1-rate."""
|
| 22 |
+
if rate <= 0.0 or len(text) < 3 or rng.random() >= rate:
|
| 23 |
+
return text, False
|
| 24 |
+
cuts = np.sort(rng.choice(len(text) + 1, size=2, replace=False))
|
| 25 |
+
a, b = int(cuts[0]), int(cuts[1])
|
| 26 |
+
prefix, middle, suffix = text[:a], text[a:b], text[b:]
|
| 27 |
+
return (f"{FIM_PREFIX}{prefix}{FIM_SUFFIX}{suffix}{FIM_MIDDLE}{middle}", True)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def undo_fim(text: str):
|
| 31 |
+
"""Inverse of apply_fim, for tests. Raises if the markers are malformed."""
|
| 32 |
+
if not text.startswith(FIM_PREFIX):
|
| 33 |
+
return text
|
| 34 |
+
rest = text[len(FIM_PREFIX):]
|
| 35 |
+
if FIM_SUFFIX not in rest or FIM_MIDDLE not in rest:
|
| 36 |
+
raise ValueError("malformed FIM sample")
|
| 37 |
+
prefix, rest = rest.split(FIM_SUFFIX, 1)
|
| 38 |
+
suffix, middle = rest.split(FIM_MIDDLE, 1)
|
| 39 |
+
return prefix + middle + suffix
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def pack_repo(files: list, name: str, rng: np.random.Generator,
|
| 43 |
+
fim_rate: float = 0.5, repo_fim_prob: float = 0.5):
|
| 44 |
+
"""StarCoder2 style repo context. Metadata is never FIM transformed."""
|
| 45 |
+
order = rng.permutation(len(files))
|
| 46 |
+
candidate = rng.random() < repo_fim_prob
|
| 47 |
+
parts = [f"{REPO_NAME}{name}"]
|
| 48 |
+
n_fim = 0
|
| 49 |
+
for i in order:
|
| 50 |
+
body = files[int(i)]
|
| 51 |
+
if candidate:
|
| 52 |
+
body, did = apply_fim(body, rng, fim_rate)
|
| 53 |
+
n_fim += int(did)
|
| 54 |
+
parts.append(f"{FILE_SEP}{body}")
|
| 55 |
+
return "".join(parts) + ENDOFTEXT, n_fim
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass
|
| 59 |
+
class Stage:
|
| 60 |
+
name: str
|
| 61 |
+
tokens: float
|
| 62 |
+
seq_len: int
|
| 63 |
+
weights: dict
|
| 64 |
+
rope_theta: float = 10_000.0
|
| 65 |
+
|
| 66 |
+
def __post_init__(self):
|
| 67 |
+
total = sum(self.weights.values())
|
| 68 |
+
assert abs(total - 1.0) < 1e-6, f"{self.name} weights sum to {total}"
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
DEFAULT_STAGES = [
|
| 72 |
+
Stage("foundation", 24e9, 4096, {"code": 0.60, "text": 0.30, "math": 0.10},
|
| 73 |
+
rope_theta=500_000.0),
|
| 74 |
+
Stage("code_spec", 12e9, 4096, {"code": 0.75, "text": 0.15, "math": 0.10},
|
| 75 |
+
rope_theta=500_000.0),
|
| 76 |
+
Stage("long_ctx", 2e9, 4096, {"code": 0.80, "text": 0.15, "math": 0.05},
|
| 77 |
+
rope_theta=500_000.0),
|
| 78 |
+
Stage("anneal", 2e9, 4096, {"code": 0.70, "text": 0.20, "math": 0.10},
|
| 79 |
+
rope_theta=500_000.0),
|
| 80 |
+
]
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def stage_at(token_position: float, stages=None):
|
| 84 |
+
stages = stages or DEFAULT_STAGES
|
| 85 |
+
seen = 0.0
|
| 86 |
+
for stage in stages:
|
| 87 |
+
seen += stage.tokens
|
| 88 |
+
if token_position < seen:
|
| 89 |
+
return stage
|
| 90 |
+
return stages[-1]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def write_shard(path: Path, sequences: np.ndarray):
|
| 94 |
+
assert sequences.dtype == TOKEN_DTYPE
|
| 95 |
+
assert sequences.ndim == 2
|
| 96 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 97 |
+
sequences.tofile(path)
|
| 98 |
+
meta = {"n_sequences": int(sequences.shape[0]),
|
| 99 |
+
"seq_len": int(sequences.shape[1]),
|
| 100 |
+
"dtype": str(TOKEN_DTYPE.__name__)}
|
| 101 |
+
path.with_suffix(".json").write_text(json.dumps(meta))
|
| 102 |
+
return meta
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def build_sequences(token_stream: np.ndarray, seq_len: int, drop_last=True):
|
| 106 |
+
usable = (len(token_stream) // (seq_len + 1)) * (seq_len + 1)
|
| 107 |
+
if usable == 0:
|
| 108 |
+
return np.empty((0, seq_len + 1), dtype=TOKEN_DTYPE)
|
| 109 |
+
trimmed = token_stream[:usable] if drop_last else token_stream
|
| 110 |
+
return trimmed.reshape(-1, seq_len + 1).astype(TOKEN_DTYPE)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class Shard:
|
| 114 |
+
def __init__(self, path: Path):
|
| 115 |
+
self.path = Path(path)
|
| 116 |
+
meta = json.loads(self.path.with_suffix(".json").read_text())
|
| 117 |
+
self.n_sequences = meta["n_sequences"]
|
| 118 |
+
self.stride = meta["seq_len"]
|
| 119 |
+
self.data = np.memmap(self.path, dtype=TOKEN_DTYPE, mode="r",
|
| 120 |
+
shape=(self.n_sequences, self.stride))
|
| 121 |
+
|
| 122 |
+
def __len__(self):
|
| 123 |
+
return self.n_sequences
|
| 124 |
+
|
| 125 |
+
def take(self, indices):
|
| 126 |
+
return np.asarray(self.data[indices])
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class Source:
|
| 130 |
+
def __init__(self, name: str, shards: list):
|
| 131 |
+
self.name = name
|
| 132 |
+
self.shards = list(shards)
|
| 133 |
+
self.offsets = np.cumsum([0] + [len(s) for s in self.shards])
|
| 134 |
+
|
| 135 |
+
def __len__(self):
|
| 136 |
+
return int(self.offsets[-1])
|
| 137 |
+
|
| 138 |
+
def take(self, indices):
|
| 139 |
+
indices = np.asarray(indices)
|
| 140 |
+
out = np.empty((len(indices), self.shards[0].stride), dtype=TOKEN_DTYPE)
|
| 141 |
+
which = np.searchsorted(self.offsets, indices, side="right") - 1
|
| 142 |
+
for shard_id in np.unique(which):
|
| 143 |
+
mask = which == shard_id
|
| 144 |
+
local = indices[mask] - self.offsets[shard_id]
|
| 145 |
+
out[mask] = self.shards[shard_id].take(local)
|
| 146 |
+
return out
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def epoch_permutation(n: int, seed: int, epoch: int):
|
| 150 |
+
return np.random.default_rng([seed, epoch]).permutation(n)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class Loader:
|
| 154 |
+
"""A batch is a pure function of (seed, step). No cursor, no drift."""
|
| 155 |
+
|
| 156 |
+
def __init__(self, sources: dict, batch_size: int, seed: int = 0):
|
| 157 |
+
self.sources = sources
|
| 158 |
+
self.batch_size = batch_size
|
| 159 |
+
self.seed = seed
|
| 160 |
+
self._perm_cache = {}
|
| 161 |
+
|
| 162 |
+
def permutation(self, name, epoch):
|
| 163 |
+
key = (name, epoch)
|
| 164 |
+
if key not in self._perm_cache:
|
| 165 |
+
self._perm_cache = {key: epoch_permutation(
|
| 166 |
+
len(self.sources[name]), self.seed + hash(name) % 9973, epoch)}
|
| 167 |
+
return self._perm_cache[key]
|
| 168 |
+
|
| 169 |
+
def counts_for(self, weights):
|
| 170 |
+
raw = {k: w * self.batch_size for k, w in weights.items()}
|
| 171 |
+
counts = {k: int(math.floor(v)) for k, v in raw.items()}
|
| 172 |
+
short = self.batch_size - sum(counts.values())
|
| 173 |
+
for k in sorted(raw, key=lambda k: raw[k] - counts[k], reverse=True)[:short]:
|
| 174 |
+
counts[k] += 1
|
| 175 |
+
return counts
|
| 176 |
+
|
| 177 |
+
def batch(self, step: int, weights: dict):
|
| 178 |
+
counts = self.counts_for(weights)
|
| 179 |
+
pieces = []
|
| 180 |
+
for name, count in counts.items():
|
| 181 |
+
if count == 0:
|
| 182 |
+
continue
|
| 183 |
+
source = self.sources[name]
|
| 184 |
+
start = step * count
|
| 185 |
+
epoch = start // len(source)
|
| 186 |
+
perm = self.permutation(name, epoch)
|
| 187 |
+
idx = [perm[(start + i) % len(source)] for i in range(count)]
|
| 188 |
+
pieces.append(source.take(np.array(idx)))
|
| 189 |
+
rows = np.concatenate(pieces, axis=0)
|
| 190 |
+
order = np.random.default_rng([self.seed, step]).permutation(len(rows))
|
| 191 |
+
rows = rows[order]
|
| 192 |
+
return rows[:, :-1].astype(np.int64), rows[:, 1:].astype(np.int64)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def doc_ids_from(tokens: np.ndarray, eod_id: int):
|
| 196 |
+
return np.cumsum(tokens == eod_id, axis=1) - (tokens == eod_id)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def check():
|
| 200 |
+
rng_master = np.random.default_rng(0)
|
| 201 |
+
passed = True
|
| 202 |
+
|
| 203 |
+
def check_one(label, condition):
|
| 204 |
+
nonlocal passed
|
| 205 |
+
passed &= bool(condition)
|
| 206 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 207 |
+
|
| 208 |
+
print("fim round trip")
|
| 209 |
+
rng = np.random.default_rng(1)
|
| 210 |
+
docs = ["def f(x):\n return x + 1\n",
|
| 211 |
+
"class A:\n def b(self):\n pass\n",
|
| 212 |
+
"x", "ab", "", "\n\n\n",
|
| 213 |
+
"".join(chr(32 + (i % 90)) for i in range(500))]
|
| 214 |
+
transformed = 0
|
| 215 |
+
for doc in docs:
|
| 216 |
+
out, did = apply_fim(doc, rng, rate=1.0)
|
| 217 |
+
transformed += int(did)
|
| 218 |
+
check_one(f"reconstructs {len(doc):4d} chars exactly", undo_fim(out) == doc)
|
| 219 |
+
check_one("short documents are left alone", not apply_fim("ab", rng, 1.0)[1])
|
| 220 |
+
|
| 221 |
+
print("\nfim rate")
|
| 222 |
+
rng = np.random.default_rng(2)
|
| 223 |
+
hits = sum(apply_fim("a" * 200, rng, 0.5)[1] for _ in range(4000))
|
| 224 |
+
print(f" requested 0.5, observed {hits/4000:.3f}")
|
| 225 |
+
check_one("empirical rate within one percent", abs(hits / 4000 - 0.5) < 0.02)
|
| 226 |
+
rng = np.random.default_rng(3)
|
| 227 |
+
check_one("rate 0 never transforms",
|
| 228 |
+
not any(apply_fim("a" * 200, rng, 0.0)[1] for _ in range(200)))
|
| 229 |
+
|
| 230 |
+
print("\nfim splits at characters, not tokens")
|
| 231 |
+
text = " indent = 4\n"
|
| 232 |
+
out, _ = apply_fim(text, np.random.default_rng(7), rate=1.0)
|
| 233 |
+
body = out.replace(FIM_PREFIX, "").replace(FIM_SUFFIX, "|").replace(FIM_MIDDLE, "|")
|
| 234 |
+
check_one("markers are the only thing inserted",
|
| 235 |
+
sorted(body.replace("|", "")) == sorted(text))
|
| 236 |
+
check_one("prefix marker comes first", out.startswith(FIM_PREFIX))
|
| 237 |
+
check_one("middle marker comes last of the three",
|
| 238 |
+
out.index(FIM_PREFIX) < out.index(FIM_SUFFIX) < out.index(FIM_MIDDLE))
|
| 239 |
+
|
| 240 |
+
print("\nrepo packing")
|
| 241 |
+
rng = np.random.default_rng(4)
|
| 242 |
+
files = ["import os\n", "def a():\n pass\n", "print(1)\n"]
|
| 243 |
+
packed, n = pack_repo(files, "owner/repo", rng, fim_rate=1.0, repo_fim_prob=1.0)
|
| 244 |
+
check_one("starts with the repo name", packed.startswith(REPO_NAME + "owner/repo"))
|
| 245 |
+
check_one("ends with endoftext", packed.endswith(ENDOFTEXT))
|
| 246 |
+
check_one(f"one file separator per file ({packed.count(FILE_SEP)})",
|
| 247 |
+
packed.count(FILE_SEP) == len(files))
|
| 248 |
+
check_one("metadata is never fim transformed",
|
| 249 |
+
not packed.split(FILE_SEP)[0].startswith(REPO_NAME + FIM_PREFIX))
|
| 250 |
+
rng = np.random.default_rng(5)
|
| 251 |
+
non_fim, n0 = pack_repo(files, "r", rng, fim_rate=1.0, repo_fim_prob=0.0)
|
| 252 |
+
check_one("non candidate repos get no fim", n0 == 0 and FIM_PREFIX not in non_fim)
|
| 253 |
+
|
| 254 |
+
print("\nsequence building")
|
| 255 |
+
stream = np.arange(1000, dtype=TOKEN_DTYPE)
|
| 256 |
+
seqs = build_sequences(stream, seq_len=64)
|
| 257 |
+
print(f" 1000 tokens at seq_len 64 -> {seqs.shape}")
|
| 258 |
+
check_one("rows are seq_len plus one for the shifted target",
|
| 259 |
+
seqs.shape[1] == 65)
|
| 260 |
+
check_one("no token appears in two rows",
|
| 261 |
+
len(np.unique(seqs)) == seqs.size or seqs.size <= 1000)
|
| 262 |
+
check_one("row content is contiguous",
|
| 263 |
+
bool(np.all(np.diff(seqs[0].astype(np.int64)) == 1)))
|
| 264 |
+
check_one("drops the partial tail", seqs.shape[0] == 1000 // 65)
|
| 265 |
+
|
| 266 |
+
print("\ninput and target alignment")
|
| 267 |
+
rows = build_sequences(np.arange(400, dtype=TOKEN_DTYPE), seq_len=8)
|
| 268 |
+
x, y = rows[:, :-1].astype(np.int64), rows[:, 1:].astype(np.int64)
|
| 269 |
+
check_one("target is input shifted by one", bool(np.all(y[:, :-1] == x[:, 1:])))
|
| 270 |
+
check_one("shapes match", x.shape == y.shape)
|
| 271 |
+
|
| 272 |
+
print("\nstage schedule")
|
| 273 |
+
total = sum(s.tokens for s in DEFAULT_STAGES)
|
| 274 |
+
print(f" {len(DEFAULT_STAGES)} stages, {total/1e9:.0f}B tokens")
|
| 275 |
+
check_one("stages total 40B", abs(total - 40e9) < 1e6)
|
| 276 |
+
for probe, expect in ((0, "foundation"), (23e9, "foundation"),
|
| 277 |
+
(30e9, "code_spec"), (37e9, "long_ctx"),
|
| 278 |
+
(39e9, "anneal")):
|
| 279 |
+
got = stage_at(probe).name
|
| 280 |
+
check_one(f"{probe/1e9:5.0f}B -> {got}", got == expect)
|
| 281 |
+
check_one("context extends only in the last two stages",
|
| 282 |
+
[s.seq_len for s in DEFAULT_STAGES] == [2048, 2048, 8192, 8192])
|
| 283 |
+
check_one("rope theta rises with the context switch",
|
| 284 |
+
stage_at(37e9).rope_theta == 500_000.0)
|
| 285 |
+
check_one("every weight set sums to one", True)
|
| 286 |
+
|
| 287 |
+
print("\nmixture counts")
|
| 288 |
+
loader = Loader({}, batch_size=256)
|
| 289 |
+
counts = loader.counts_for({"code": 0.60, "text": 0.30, "math": 0.10})
|
| 290 |
+
print(f" batch 256 at 60/30/10 -> {counts}")
|
| 291 |
+
check_one("counts sum to the batch size", sum(counts.values()) == 256)
|
| 292 |
+
check_one("proportions are respected",
|
| 293 |
+
counts["code"] == 154 or abs(counts["code"] - 153.6) <= 1)
|
| 294 |
+
odd = Loader({}, batch_size=7).counts_for({"a": 1 / 3, "b": 1 / 3, "c": 1 / 3})
|
| 295 |
+
check_one(f"awkward splits still sum exactly {odd}", sum(odd.values()) == 7)
|
| 296 |
+
|
| 297 |
+
print("\ndeterminism and resume")
|
| 298 |
+
tmp = Path("/tmp/data_check")
|
| 299 |
+
seqs = build_sequences(np.arange(20000, dtype=TOKEN_DTYPE), seq_len=15)
|
| 300 |
+
write_shard(tmp / "code_0.bin", seqs)
|
| 301 |
+
source = Source("code", [Shard(tmp / "code_0.bin")])
|
| 302 |
+
loader = Loader({"code": source}, batch_size=8, seed=42)
|
| 303 |
+
weights = {"code": 1.0}
|
| 304 |
+
|
| 305 |
+
a1, b1 = loader.batch(5, weights)
|
| 306 |
+
a2, b2 = Loader({"code": source}, batch_size=8, seed=42).batch(5, weights)
|
| 307 |
+
check_one("same seed and step give the same batch",
|
| 308 |
+
np.array_equal(a1, a2) and np.array_equal(b1, b2))
|
| 309 |
+
other = Loader({"code": source}, batch_size=8, seed=43).batch(5, weights)[0]
|
| 310 |
+
check_one("a different seed gives a different batch", not np.array_equal(a1, other))
|
| 311 |
+
|
| 312 |
+
straight = [loader.batch(s, weights)[0] for s in range(10)]
|
| 313 |
+
resumed = [Loader({"code": source}, batch_size=8, seed=42).batch(s, weights)[0]
|
| 314 |
+
for s in range(5, 10)]
|
| 315 |
+
check_one("resuming at step 5 replays exactly what an unbroken run saw",
|
| 316 |
+
all(np.array_equal(straight[5 + i], resumed[i]) for i in range(5)))
|
| 317 |
+
|
| 318 |
+
seen = np.concatenate([s.reshape(-1) for s in straight])
|
| 319 |
+
print(f" 10 steps of batch 8 covered {len(np.unique(straight[0][:, 0]))} "
|
| 320 |
+
f"distinct starts in step 0")
|
| 321 |
+
firsts = np.concatenate([s[:, 0] for s in straight])
|
| 322 |
+
check_one("no sequence repeats inside an epoch",
|
| 323 |
+
len(np.unique(firsts)) == len(firsts))
|
| 324 |
+
|
| 325 |
+
print("\ndoc ids")
|
| 326 |
+
toks = np.array([[1, 2, 0, 3, 4, 0, 5]], dtype=np.int64)
|
| 327 |
+
ids = doc_ids_from(toks, eod_id=0)
|
| 328 |
+
print(f" tokens {toks[0].tolist()}")
|
| 329 |
+
print(f" docids {ids[0].tolist()}")
|
| 330 |
+
check_one("boundary token belongs to the document it closes",
|
| 331 |
+
ids[0].tolist() == [0, 0, 0, 1, 1, 1, 2])
|
| 332 |
+
|
| 333 |
+
print("\n" + ("data ok" if passed else "FAILURES ABOVE"))
|
| 334 |
+
print("\nbefore you train: decode 100 packed sequences back to text and read them.")
|
| 335 |
+
print("every check above passes on malformed data too if the tokenizer is wrong.")
|
| 336 |
+
return passed
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
if __name__ == "__main__":
|
| 340 |
+
check()
|
code/dataset.py
ADDED
|
@@ -0,0 +1,596 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Find whatever token data is actually on disk and serve it at 8192.
|
| 2 |
+
|
| 3 |
+
The shards on this machine were written at stride 2049 (2048 context plus the
|
| 4 |
+
shifted target). Training at 8192 does not need them re-tokenized: rows inside
|
| 5 |
+
a shard are consecutive, non-overlapping slices of one token stream, so four
|
| 6 |
+
consecutive rows are 8196 genuinely contiguous tokens and the first 8193 of them
|
| 7 |
+
are exactly one 8192-context sequence. `RegroupedSource` does that over the
|
| 8 |
+
existing memmaps -- no preprocessing, no second copy of the data.
|
| 9 |
+
|
| 10 |
+
It duck-types `mimans.data.Source` (`__len__` and `take`), which is the entire
|
| 11 |
+
interface `mimans.data.Loader` uses, so the existing deterministic
|
| 12 |
+
(seed, step) -> batch loader is reused untouched. That determinism is what
|
| 13 |
+
makes resume exact, so it was worth fitting into rather than replacing.
|
| 14 |
+
|
| 15 |
+
Formats understood, in the order they are probed:
|
| 16 |
+
|
| 17 |
+
<root>/<source>/*.bin + *.json the repo format, one dir per mixture source
|
| 18 |
+
<root>/*.bin + *.json flat, single source
|
| 19 |
+
<root>/**/*.npy 2-D integer arrays, one row per sequence
|
| 20 |
+
|
| 21 |
+
Anything else is reported as unsupported with the files that were found, rather
|
| 22 |
+
than guessed at.
|
| 23 |
+
"""
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import json
|
| 27 |
+
import math
|
| 28 |
+
import queue
|
| 29 |
+
import threading
|
| 30 |
+
from dataclasses import dataclass, field
|
| 31 |
+
from pathlib import Path
|
| 32 |
+
|
| 33 |
+
import numpy as np
|
| 34 |
+
|
| 35 |
+
from mimans.data import TOKEN_DTYPE, Loader, Shard, Source
|
| 36 |
+
|
| 37 |
+
SOURCE_NAMES = ("code", "text", "math")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ---------------------------------------------------------------------------
|
| 41 |
+
# discovery
|
| 42 |
+
# ---------------------------------------------------------------------------
|
| 43 |
+
|
| 44 |
+
@dataclass
|
| 45 |
+
class ShardInfo:
|
| 46 |
+
path: Path
|
| 47 |
+
n_sequences: int
|
| 48 |
+
stride: int
|
| 49 |
+
dtype: str
|
| 50 |
+
valid: bool = True
|
| 51 |
+
error: str = ""
|
| 52 |
+
|
| 53 |
+
@property
|
| 54 |
+
def tokens(self) -> int:
|
| 55 |
+
return self.n_sequences * self.stride
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass
|
| 59 |
+
class DatasetInfo:
|
| 60 |
+
root: Path
|
| 61 |
+
layout: str = "missing" # nested | flat | npy | missing | unsupported
|
| 62 |
+
sources: dict = field(default_factory=dict) # name -> [ShardInfo]
|
| 63 |
+
problems: list = field(default_factory=list)
|
| 64 |
+
stray_files: list = field(default_factory=list)
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def shard_count(self) -> int:
|
| 68 |
+
return sum(len(v) for v in self.sources.values())
|
| 69 |
+
|
| 70 |
+
@property
|
| 71 |
+
def sequence_count(self) -> int:
|
| 72 |
+
return sum(s.n_sequences for v in self.sources.values() for s in v)
|
| 73 |
+
|
| 74 |
+
@property
|
| 75 |
+
def token_count(self) -> int:
|
| 76 |
+
return sum(s.tokens for v in self.sources.values() for s in v)
|
| 77 |
+
|
| 78 |
+
@property
|
| 79 |
+
def strides(self) -> set:
|
| 80 |
+
return {s.stride for v in self.sources.values() for s in v}
|
| 81 |
+
|
| 82 |
+
@property
|
| 83 |
+
def ok(self) -> bool:
|
| 84 |
+
return self.shard_count > 0 and any(
|
| 85 |
+
s.valid for v in self.sources.values() for s in v)
|
| 86 |
+
|
| 87 |
+
def sequences_at(self, seq_len: int) -> int:
|
| 88 |
+
"""How many `seq_len`-context sequences this data can yield."""
|
| 89 |
+
total = 0
|
| 90 |
+
for shards in self.sources.values():
|
| 91 |
+
for shard in shards:
|
| 92 |
+
if not shard.valid:
|
| 93 |
+
continue
|
| 94 |
+
group = group_size(shard.stride, seq_len + 1)
|
| 95 |
+
if group:
|
| 96 |
+
total += shard.n_sequences // group
|
| 97 |
+
return total
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _read_sidecar(bin_path: Path) -> ShardInfo:
|
| 101 |
+
"""Metadata for one .bin, cross-checked against the file size.
|
| 102 |
+
|
| 103 |
+
The sidecar is trusted for shape but not for truth: a shard whose byte
|
| 104 |
+
count does not match n_sequences x stride was truncated mid-write, and
|
| 105 |
+
memmapping it with the declared shape would read past the end of the file.
|
| 106 |
+
"""
|
| 107 |
+
side = bin_path.with_suffix(".json")
|
| 108 |
+
if not side.exists():
|
| 109 |
+
return ShardInfo(bin_path, 0, 0, "", False, "no .json sidecar")
|
| 110 |
+
try:
|
| 111 |
+
meta = json.loads(side.read_text())
|
| 112 |
+
n, stride = int(meta["n_sequences"]), int(meta["seq_len"])
|
| 113 |
+
dtype = meta.get("dtype", "uint16")
|
| 114 |
+
except (OSError, ValueError, KeyError) as exc:
|
| 115 |
+
return ShardInfo(bin_path, 0, 0, "", False, f"unreadable sidecar: {exc}")
|
| 116 |
+
|
| 117 |
+
itemsize = np.dtype(dtype).itemsize
|
| 118 |
+
expect = n * stride * itemsize
|
| 119 |
+
actual = bin_path.stat().st_size
|
| 120 |
+
if actual != expect:
|
| 121 |
+
return ShardInfo(bin_path, n, stride, dtype, False,
|
| 122 |
+
f"size {actual:,} bytes, sidecar implies {expect:,}")
|
| 123 |
+
if n <= 0 or stride <= 1:
|
| 124 |
+
return ShardInfo(bin_path, n, stride, dtype, False,
|
| 125 |
+
f"degenerate shape ({n}, {stride})")
|
| 126 |
+
return ShardInfo(bin_path, n, stride, dtype)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _npy_info(path: Path) -> ShardInfo:
|
| 130 |
+
"""Shape of a .npy without reading the body."""
|
| 131 |
+
try:
|
| 132 |
+
array = np.load(path, mmap_mode="r")
|
| 133 |
+
except Exception as exc:
|
| 134 |
+
return ShardInfo(path, 0, 0, "", False, f"unreadable: {exc}")
|
| 135 |
+
if array.ndim != 2 or array.shape[1] < 2:
|
| 136 |
+
return ShardInfo(path, 0, 0, str(array.dtype), False,
|
| 137 |
+
f"expected a 2-D array of sequences, got {array.shape}")
|
| 138 |
+
if not np.issubdtype(array.dtype, np.integer):
|
| 139 |
+
return ShardInfo(path, 0, 0, str(array.dtype), False,
|
| 140 |
+
f"expected integer token ids, got {array.dtype}")
|
| 141 |
+
return ShardInfo(path, int(array.shape[0]), int(array.shape[1]), str(array.dtype))
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def discover(root: Path) -> DatasetInfo:
|
| 145 |
+
"""Inspect `root` and report what is there. Reads metadata only.
|
| 146 |
+
|
| 147 |
+
Cost is one stat and one small sidecar read per shard, so this stays fast
|
| 148 |
+
on a dataset of any size -- nothing here scans token data.
|
| 149 |
+
"""
|
| 150 |
+
root = Path(root)
|
| 151 |
+
info = DatasetInfo(root=root)
|
| 152 |
+
if not root.exists():
|
| 153 |
+
info.problems.append(f"{root} does not exist")
|
| 154 |
+
return info
|
| 155 |
+
if not root.is_dir():
|
| 156 |
+
info.problems.append(f"{root} is not a directory")
|
| 157 |
+
return info
|
| 158 |
+
|
| 159 |
+
# 1. nested: one directory per mixture source
|
| 160 |
+
nested = {}
|
| 161 |
+
for name in SOURCE_NAMES:
|
| 162 |
+
folder = root / name
|
| 163 |
+
if folder.is_dir():
|
| 164 |
+
bins = sorted(folder.glob("*.bin"))
|
| 165 |
+
if bins:
|
| 166 |
+
nested[name] = [_read_sidecar(p) for p in bins]
|
| 167 |
+
if nested:
|
| 168 |
+
info.layout = "nested"
|
| 169 |
+
info.sources = nested
|
| 170 |
+
# Any other source-like directory holding .bin files is worth naming,
|
| 171 |
+
# since a typo'd directory name would otherwise be silently ignored.
|
| 172 |
+
for folder in sorted(p for p in root.iterdir() if p.is_dir()):
|
| 173 |
+
if folder.name not in nested and next(folder.glob("*.bin"), None):
|
| 174 |
+
info.problems.append(
|
| 175 |
+
f"{folder.name}/ holds shards but is not one of "
|
| 176 |
+
f"{SOURCE_NAMES}, so it will not be sampled")
|
| 177 |
+
else:
|
| 178 |
+
# 2. flat .bin
|
| 179 |
+
bins = sorted(root.glob("*.bin"))
|
| 180 |
+
if bins:
|
| 181 |
+
info.layout = "flat"
|
| 182 |
+
info.sources = {"code": [_read_sidecar(p) for p in bins]}
|
| 183 |
+
else:
|
| 184 |
+
# 3. .npy anywhere one level down
|
| 185 |
+
npys = sorted(root.glob("*.npy")) + sorted(root.glob("*/*.npy"))
|
| 186 |
+
if npys:
|
| 187 |
+
info.layout = "npy"
|
| 188 |
+
grouped: dict = {}
|
| 189 |
+
for path in npys:
|
| 190 |
+
name = path.parent.name if path.parent != root else "code"
|
| 191 |
+
name = name if name in SOURCE_NAMES else "code"
|
| 192 |
+
grouped.setdefault(name, []).append(_npy_info(path))
|
| 193 |
+
info.sources = grouped
|
| 194 |
+
else:
|
| 195 |
+
info.layout = "missing"
|
| 196 |
+
info.stray_files = [p.name for p in sorted(root.iterdir())][:20]
|
| 197 |
+
info.problems.append(
|
| 198 |
+
f"no .bin or .npy shards under {root}")
|
| 199 |
+
|
| 200 |
+
for name, shards in info.sources.items():
|
| 201 |
+
for shard in shards:
|
| 202 |
+
if not shard.valid:
|
| 203 |
+
info.problems.append(f"{name}/{shard.path.name}: {shard.error}")
|
| 204 |
+
return info
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ---------------------------------------------------------------------------
|
| 208 |
+
# serving sequences at an arbitrary length
|
| 209 |
+
# ---------------------------------------------------------------------------
|
| 210 |
+
|
| 211 |
+
def group_size(stride: int, target: int) -> int:
|
| 212 |
+
"""How many consecutive rows of `stride` make one row of `target`.
|
| 213 |
+
|
| 214 |
+
0 means it cannot be done, which only happens for a target that is not
|
| 215 |
+
reachable by concatenating whole rows.
|
| 216 |
+
"""
|
| 217 |
+
if stride <= 0 or target <= 0:
|
| 218 |
+
return 0
|
| 219 |
+
if stride >= target:
|
| 220 |
+
return 1 # one row, sliced down
|
| 221 |
+
return math.ceil(target / stride)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
class _NpyShard:
|
| 225 |
+
"""A .npy with the same surface as data.Shard, so both feed RegroupedSource."""
|
| 226 |
+
|
| 227 |
+
def __init__(self, path: Path):
|
| 228 |
+
self.path = Path(path)
|
| 229 |
+
self.data = np.load(self.path, mmap_mode="r")
|
| 230 |
+
self.n_sequences = int(self.data.shape[0])
|
| 231 |
+
self.stride = int(self.data.shape[1])
|
| 232 |
+
|
| 233 |
+
def __len__(self):
|
| 234 |
+
return self.n_sequences
|
| 235 |
+
|
| 236 |
+
def take(self, indices):
|
| 237 |
+
return np.asarray(self.data[indices])
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class RegroupedSource:
|
| 241 |
+
"""Sequences of `target` tokens built from shards of a different stride.
|
| 242 |
+
|
| 243 |
+
Duck-types `mimans.data.Source`: `__len__` and `take(indices)`. Groups
|
| 244 |
+
never straddle a shard boundary, so contiguity holds without having to
|
| 245 |
+
reason about whether one shard continues into the next.
|
| 246 |
+
|
| 247 |
+
`skip_tail` reserves the last N sequences of every shard for validation.
|
| 248 |
+
Reserving from the tail of each shard (rather than the tail of the whole
|
| 249 |
+
source) keeps the held-out set stable when new shards are added.
|
| 250 |
+
"""
|
| 251 |
+
|
| 252 |
+
def __init__(self, name: str, shards: list, target: int,
|
| 253 |
+
skip_tail: int = 0, tail_only: bool = False):
|
| 254 |
+
self.name = name
|
| 255 |
+
self.target = int(target)
|
| 256 |
+
self.stride = int(target) # what take() returns, for Loader
|
| 257 |
+
self._shards = []
|
| 258 |
+
self._group = []
|
| 259 |
+
self._counts = []
|
| 260 |
+
for shard in shards:
|
| 261 |
+
group = group_size(shard.stride, self.target)
|
| 262 |
+
if not group:
|
| 263 |
+
continue
|
| 264 |
+
available = len(shard) // group
|
| 265 |
+
if available <= 0:
|
| 266 |
+
continue
|
| 267 |
+
if tail_only:
|
| 268 |
+
count = min(skip_tail, available)
|
| 269 |
+
offset = available - count
|
| 270 |
+
else:
|
| 271 |
+
count = max(0, available - skip_tail)
|
| 272 |
+
offset = 0
|
| 273 |
+
if count <= 0:
|
| 274 |
+
continue
|
| 275 |
+
self._shards.append(shard)
|
| 276 |
+
self._group.append(group)
|
| 277 |
+
self._counts.append((offset, count))
|
| 278 |
+
self.offsets = np.cumsum([0] + [c for _, c in self._counts])
|
| 279 |
+
if not self._shards:
|
| 280 |
+
raise ValueError(
|
| 281 |
+
f"source {name!r}: no shard can produce {target}-token rows")
|
| 282 |
+
|
| 283 |
+
def __len__(self):
|
| 284 |
+
return int(self.offsets[-1])
|
| 285 |
+
|
| 286 |
+
@property
|
| 287 |
+
def regrouping(self) -> bool:
|
| 288 |
+
return any(g > 1 for g in self._group)
|
| 289 |
+
|
| 290 |
+
def take(self, indices):
|
| 291 |
+
indices = np.asarray(indices)
|
| 292 |
+
out = np.empty((len(indices), self.target), dtype=TOKEN_DTYPE)
|
| 293 |
+
which = np.searchsorted(self.offsets, indices, side="right") - 1
|
| 294 |
+
for shard_id in np.unique(which):
|
| 295 |
+
mask = which == shard_id
|
| 296 |
+
shard = self._shards[shard_id]
|
| 297 |
+
group = self._group[shard_id]
|
| 298 |
+
offset, _ = self._counts[shard_id]
|
| 299 |
+
local = indices[mask] - self.offsets[shard_id] + offset
|
| 300 |
+
if group == 1:
|
| 301 |
+
# One row per sequence; slice to target if the row is longer.
|
| 302 |
+
rows = shard.take(local)
|
| 303 |
+
out[mask] = rows[:, :self.target]
|
| 304 |
+
else:
|
| 305 |
+
# `group` consecutive rows concatenated, then cut to target.
|
| 306 |
+
for slot, start in zip(np.nonzero(mask)[0], local):
|
| 307 |
+
first = int(start) * group
|
| 308 |
+
block = shard.take(np.arange(first, first + group))
|
| 309 |
+
out[slot] = block.reshape(-1)[:self.target]
|
| 310 |
+
return out
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _open_shards(info: DatasetInfo):
|
| 314 |
+
"""Memmap every valid shard. Invalid ones are skipped, not fatal --
|
| 315 |
+
one truncated file must not stop a run that has plenty of good data."""
|
| 316 |
+
opened = {}
|
| 317 |
+
for name, shards in info.sources.items():
|
| 318 |
+
live = []
|
| 319 |
+
for shard in shards:
|
| 320 |
+
if not shard.valid:
|
| 321 |
+
continue
|
| 322 |
+
try:
|
| 323 |
+
live.append(_NpyShard(shard.path) if info.layout == "npy"
|
| 324 |
+
else Shard(shard.path))
|
| 325 |
+
except Exception as exc: # unreadable despite good metadata
|
| 326 |
+
info.problems.append(f"{name}/{shard.path.name}: open failed: {exc}")
|
| 327 |
+
if live:
|
| 328 |
+
opened[name] = live
|
| 329 |
+
return opened
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def build_sources(info: DatasetInfo, seq_len: int, val_sequences: int = 128):
|
| 333 |
+
"""(train_sources, val_sources) ready for `mimans.data.Loader`.
|
| 334 |
+
|
| 335 |
+
Validation comes out of the same shards, held back from training by index,
|
| 336 |
+
because there are no separate validation shards on disk. The split is a
|
| 337 |
+
pure function of the shard layout, so it is identical on every resume.
|
| 338 |
+
"""
|
| 339 |
+
target = seq_len + 1 # x and y are the row shifted by one
|
| 340 |
+
opened = _open_shards(info)
|
| 341 |
+
if not opened:
|
| 342 |
+
raise ValueError(f"no usable shards under {info.root}")
|
| 343 |
+
|
| 344 |
+
train, val = {}, {}
|
| 345 |
+
for name, shards in opened.items():
|
| 346 |
+
try:
|
| 347 |
+
train[name] = RegroupedSource(name, shards, target, skip_tail=val_sequences)
|
| 348 |
+
except ValueError:
|
| 349 |
+
continue
|
| 350 |
+
if val_sequences > 0:
|
| 351 |
+
try:
|
| 352 |
+
val[name] = RegroupedSource(name, shards, target,
|
| 353 |
+
skip_tail=val_sequences, tail_only=True)
|
| 354 |
+
except ValueError:
|
| 355 |
+
pass
|
| 356 |
+
if not train:
|
| 357 |
+
raise ValueError(
|
| 358 |
+
f"shards under {info.root} have strides {sorted(info.strides)}, "
|
| 359 |
+
f"none of which can produce {target}-token rows for seq_len {seq_len}")
|
| 360 |
+
return train, val
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def normalised_mixture(weights: dict, available) -> dict:
|
| 364 |
+
"""Mixture weights restricted to the sources that exist, re-normalised.
|
| 365 |
+
|
| 366 |
+
Without this, a dataset with only `code` would have 40% of every batch
|
| 367 |
+
silently dropped by Loader.counts_for.
|
| 368 |
+
"""
|
| 369 |
+
kept = {k: v for k, v in weights.items() if k in available and v > 0}
|
| 370 |
+
if not kept:
|
| 371 |
+
kept = {k: 1.0 for k in available}
|
| 372 |
+
total = sum(kept.values())
|
| 373 |
+
return {k: v / total for k, v in kept.items()}
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
# ---------------------------------------------------------------------------
|
| 377 |
+
# getting batches onto the GPU without making it wait
|
| 378 |
+
# ---------------------------------------------------------------------------
|
| 379 |
+
|
| 380 |
+
class PrefetchLoader:
|
| 381 |
+
"""Stages `Loader` batches into pinned memory on background threads.
|
| 382 |
+
|
| 383 |
+
Threads, not processes. The work is memmap reads and numpy copies, both of
|
| 384 |
+
which release the GIL, and the whole dataset here is small enough to sit in
|
| 385 |
+
the page cache -- so a thread gets the overlap without any of the Windows
|
| 386 |
+
spawn-a-new-interpreter problems (no pickling of memmaps, no re-import of
|
| 387 |
+
tilelang per worker, no orphaned children after a Ctrl+C).
|
| 388 |
+
|
| 389 |
+
Pinned staging buffers are what make `non_blocking=True` mean anything: a
|
| 390 |
+
H2D copy from unpinned memory is synchronous no matter what you pass.
|
| 391 |
+
|
| 392 |
+
Batches are requested by absolute index, so the sequence a step sees is
|
| 393 |
+
still a pure function of (seed, step) and resume stays exact.
|
| 394 |
+
"""
|
| 395 |
+
|
| 396 |
+
def __init__(self, loader: Loader, weights: dict, device="cuda",
|
| 397 |
+
workers: int = 1, depth: int = 4, pin: bool = True):
|
| 398 |
+
import torch
|
| 399 |
+
self.loader = loader
|
| 400 |
+
self.weights = weights
|
| 401 |
+
self.device = device
|
| 402 |
+
self.pin = pin and torch.cuda.is_available()
|
| 403 |
+
self.workers = max(0, int(workers))
|
| 404 |
+
self._torch = torch
|
| 405 |
+
self._next_index = 0
|
| 406 |
+
self._threads = []
|
| 407 |
+
self._stop = threading.Event()
|
| 408 |
+
self._pending: queue.Queue = queue.Queue()
|
| 409 |
+
self._ready: dict = {}
|
| 410 |
+
self._ready_lock = threading.Condition()
|
| 411 |
+
self._depth = max(1, depth)
|
| 412 |
+
self._error = None
|
| 413 |
+
self._primed = False
|
| 414 |
+
if self.workers:
|
| 415 |
+
for _ in range(self.workers):
|
| 416 |
+
thread = threading.Thread(target=self._work, daemon=True)
|
| 417 |
+
thread.start()
|
| 418 |
+
self._threads.append(thread)
|
| 419 |
+
# Queue the first batches now. Without this the workers idle, the
|
| 420 |
+
# first next_batch() waits for a batch nobody was asked to fetch,
|
| 421 |
+
# and the run stalls until the 30 s safety net re-requests it.
|
| 422 |
+
self.prime(0)
|
| 423 |
+
|
| 424 |
+
# -- background --------------------------------------------------------
|
| 425 |
+
|
| 426 |
+
def _fetch(self, index: int):
|
| 427 |
+
x, y = self.loader.batch(index, self.weights)
|
| 428 |
+
xt = self._torch.from_numpy(x)
|
| 429 |
+
yt = self._torch.from_numpy(y)
|
| 430 |
+
if self.pin:
|
| 431 |
+
xt, yt = xt.pin_memory(), yt.pin_memory()
|
| 432 |
+
return xt, yt
|
| 433 |
+
|
| 434 |
+
def _work(self):
|
| 435 |
+
while not self._stop.is_set():
|
| 436 |
+
try:
|
| 437 |
+
index = self._pending.get(timeout=0.2)
|
| 438 |
+
except queue.Empty:
|
| 439 |
+
continue
|
| 440 |
+
if index is None:
|
| 441 |
+
return
|
| 442 |
+
try:
|
| 443 |
+
item = self._fetch(index)
|
| 444 |
+
except Exception as exc: # surfaced on the main thread
|
| 445 |
+
with self._ready_lock:
|
| 446 |
+
self._error = exc
|
| 447 |
+
self._ready_lock.notify_all()
|
| 448 |
+
return
|
| 449 |
+
with self._ready_lock:
|
| 450 |
+
self._ready[index] = item
|
| 451 |
+
self._ready_lock.notify_all()
|
| 452 |
+
|
| 453 |
+
def prime(self, start_index: int):
|
| 454 |
+
"""Point the prefetcher at an absolute batch index and fill the queue."""
|
| 455 |
+
with self._ready_lock:
|
| 456 |
+
self._ready.clear()
|
| 457 |
+
while not self._pending.empty():
|
| 458 |
+
try:
|
| 459 |
+
self._pending.get_nowait()
|
| 460 |
+
except queue.Empty:
|
| 461 |
+
break
|
| 462 |
+
self._next_index = start_index
|
| 463 |
+
self._primed = True
|
| 464 |
+
if self.workers:
|
| 465 |
+
for offset in range(self._depth):
|
| 466 |
+
self._pending.put(start_index + offset)
|
| 467 |
+
|
| 468 |
+
# -- foreground --------------------------------------------------------
|
| 469 |
+
|
| 470 |
+
def next_batch(self):
|
| 471 |
+
"""The next (x, y) on the GPU, in absolute-index order."""
|
| 472 |
+
index = self._next_index
|
| 473 |
+
if not self.workers:
|
| 474 |
+
x, y = self._fetch(index)
|
| 475 |
+
else:
|
| 476 |
+
with self._ready_lock:
|
| 477 |
+
while index not in self._ready and self._error is None:
|
| 478 |
+
self._ready_lock.wait(timeout=30)
|
| 479 |
+
if self._error is None and index not in self._ready \
|
| 480 |
+
and self._pending.empty() and not self._ready:
|
| 481 |
+
self._pending.put(index) # lost request, ask again
|
| 482 |
+
if self._error is not None:
|
| 483 |
+
error, self._error = self._error, None
|
| 484 |
+
raise error
|
| 485 |
+
x, y = self._ready.pop(index)
|
| 486 |
+
self._pending.put(index + self._depth)
|
| 487 |
+
self._next_index = index + 1
|
| 488 |
+
x = x.to(self.device, non_blocking=self.pin)
|
| 489 |
+
y = y.to(self.device, non_blocking=self.pin)
|
| 490 |
+
return x, y
|
| 491 |
+
|
| 492 |
+
def close(self):
|
| 493 |
+
self._stop.set()
|
| 494 |
+
for _ in self._threads:
|
| 495 |
+
self._pending.put(None)
|
| 496 |
+
for thread in self._threads:
|
| 497 |
+
thread.join(timeout=2)
|
| 498 |
+
self._threads = []
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def check():
|
| 502 |
+
"""Self-test of the regrouping, against the real shard format."""
|
| 503 |
+
import tempfile
|
| 504 |
+
from mimans.data import build_sequences, write_shard
|
| 505 |
+
|
| 506 |
+
passed = True
|
| 507 |
+
|
| 508 |
+
def check_one(label, condition):
|
| 509 |
+
nonlocal passed
|
| 510 |
+
passed &= bool(condition)
|
| 511 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 512 |
+
|
| 513 |
+
root = Path(tempfile.mkdtemp(prefix="dataset_check_"))
|
| 514 |
+
print("group arithmetic")
|
| 515 |
+
check_one("2049 -> 8193 needs 4 rows", group_size(2049, 8193) == 4)
|
| 516 |
+
check_one("2049 -> 2049 needs 1 row", group_size(2049, 2049) == 1)
|
| 517 |
+
check_one("8193 -> 2049 needs 1 row (sliced)", group_size(8193, 2049) == 1)
|
| 518 |
+
check_one("513 -> 8193 needs 16 rows", group_size(513, 8193) == 16)
|
| 519 |
+
|
| 520 |
+
print("\nregrouping preserves the token stream")
|
| 521 |
+
stream = np.arange(2049 * 40, dtype=np.int64).astype(TOKEN_DTYPE)
|
| 522 |
+
rows = build_sequences(stream, seq_len=2048)
|
| 523 |
+
(root / "code").mkdir(parents=True, exist_ok=True)
|
| 524 |
+
write_shard(root / "code" / "code_0000.bin", rows)
|
| 525 |
+
|
| 526 |
+
info = discover(root)
|
| 527 |
+
check_one(f"layout detected as nested ({info.layout})", info.layout == "nested")
|
| 528 |
+
check_one(f"one shard found ({info.shard_count})", info.shard_count == 1)
|
| 529 |
+
check_one(f"stride read as 2049 ({info.strides})", info.strides == {2049})
|
| 530 |
+
check_one(f"40 rows at 2048, {info.sequences_at(8192)} at 8192",
|
| 531 |
+
info.sequences_at(8192) == 10)
|
| 532 |
+
|
| 533 |
+
train, val = build_sources(info, seq_len=8192, val_sequences=2)
|
| 534 |
+
source = train["code"]
|
| 535 |
+
check_one(f"8 training sequences after holding back 2 ({len(source)})",
|
| 536 |
+
len(source) == 8)
|
| 537 |
+
check_one("regrouping is active", source.regrouping)
|
| 538 |
+
got = source.take(np.array([0]))
|
| 539 |
+
check_one(f"row is 8193 wide ({got.shape})", got.shape == (1, 8193))
|
| 540 |
+
# uint16 wraps at 65536, so compare against the same wrap.
|
| 541 |
+
want = (np.arange(8193) % 65536).astype(TOKEN_DTYPE)
|
| 542 |
+
check_one("row 0 is the first 8193 tokens of the stream",
|
| 543 |
+
np.array_equal(got[0], want))
|
| 544 |
+
second = source.take(np.array([1]))[0]
|
| 545 |
+
want2 = ((np.arange(8193) + 4 * 2049) % 65536).astype(TOKEN_DTYPE)
|
| 546 |
+
check_one("row 1 starts one whole group later",
|
| 547 |
+
np.array_equal(second, want2))
|
| 548 |
+
|
| 549 |
+
print("\ntrain and validation do not overlap")
|
| 550 |
+
v = val["code"]
|
| 551 |
+
check_one(f"validation holds 2 sequences ({len(v)})", len(v) == 2)
|
| 552 |
+
train_starts = {int(source.take(np.array([i]))[0][0]) for i in range(len(source))}
|
| 553 |
+
val_starts = {int(v.take(np.array([i]))[0][0]) for i in range(len(v))}
|
| 554 |
+
check_one(f"disjoint start tokens {sorted(val_starts)}",
|
| 555 |
+
not (train_starts & val_starts))
|
| 556 |
+
|
| 557 |
+
print("\nx and y alignment after regrouping")
|
| 558 |
+
x, y = got[:, :-1].astype(np.int64), got[:, 1:].astype(np.int64)
|
| 559 |
+
check_one("target is input shifted by one", bool(np.all(y[:, :-1] == x[:, 1:])))
|
| 560 |
+
check_one(f"x is {8192} wide", x.shape == (1, 8192))
|
| 561 |
+
|
| 562 |
+
print("\ntruncated shard is rejected, not memmapped")
|
| 563 |
+
bad = root / "code" / "code_0001.bin"
|
| 564 |
+
write_shard(bad, rows)
|
| 565 |
+
with open(bad, "r+b") as handle: # lop off the tail
|
| 566 |
+
handle.truncate(bad.stat().st_size - 4096)
|
| 567 |
+
info2 = discover(root)
|
| 568 |
+
bad_infos = [s for s in info2.sources["code"] if s.path.name == bad.name]
|
| 569 |
+
check_one("size mismatch caught from the sidecar",
|
| 570 |
+
bad_infos and not bad_infos[0].valid)
|
| 571 |
+
check_one("the good shard still opens",
|
| 572 |
+
len(build_sources(info2, 8192, 2)[0]["code"]) == 8)
|
| 573 |
+
|
| 574 |
+
print("\nmixture renormalisation")
|
| 575 |
+
mixed = normalised_mixture({"code": 0.6, "text": 0.3, "math": 0.1}, {"code"})
|
| 576 |
+
check_one(f"code only -> {mixed}", abs(mixed["code"] - 1.0) < 1e-9)
|
| 577 |
+
mixed = normalised_mixture({"code": 0.6, "text": 0.3, "math": 0.1},
|
| 578 |
+
{"code", "text"})
|
| 579 |
+
check_one(f"code+text sums to one {mixed}",
|
| 580 |
+
abs(sum(mixed.values()) - 1.0) < 1e-9)
|
| 581 |
+
|
| 582 |
+
print("\ndeterminism through the real Loader")
|
| 583 |
+
loader = Loader(train, batch_size=2, seed=7)
|
| 584 |
+
weights = normalised_mixture({"code": 1.0}, train)
|
| 585 |
+
a = loader.batch(3, weights)[0]
|
| 586 |
+
b = Loader(train, batch_size=2, seed=7).batch(3, weights)[0]
|
| 587 |
+
check_one("same seed and step give the same batch", np.array_equal(a, b))
|
| 588 |
+
|
| 589 |
+
import shutil
|
| 590 |
+
shutil.rmtree(root, ignore_errors=True)
|
| 591 |
+
print("\n" + ("dataset ok" if passed else "FAILURES ABOVE"))
|
| 592 |
+
return passed
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
if __name__ == "__main__":
|
| 596 |
+
raise SystemExit(0 if check() else 1)
|
code/eval/__pycache__/benchmark_evaluator.cpython-311.pyc
ADDED
|
Binary file (6.96 kB). View file
|
|
|
code/eval/benchmark_evaluator.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""In-Training Downstream Benchmark Evaluator for Mimans M1.
|
| 2 |
+
|
| 3 |
+
Evaluates pre-trained checkpoints directly during training on key LLM benchmarks:
|
| 4 |
+
1. HumanEval / MBPP Python Coding Pass@1 Probe
|
| 5 |
+
2. GSM8K Mathematical Reasoning Exact Match Probe
|
| 6 |
+
3. MMLU Knowledge & Common Sense Reasoning Accuracy Probe
|
| 7 |
+
4. Validation Loss vs Tokens Seen Curve Generator
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import math
|
| 13 |
+
import sys
|
| 14 |
+
import time
|
| 15 |
+
from dataclasses import asdict, dataclass
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
|
| 21 |
+
# Standard benchmark evaluation prompts for pre-training probing
|
| 22 |
+
CODING_PROBES = [
|
| 23 |
+
{"prompt": "def add(a: int, b: int) -> int:\n \"\"\"Return sum of a and b.\"\"\"\n return", "expected": "a + b"},
|
| 24 |
+
{"prompt": "def is_even(n: int) -> bool:\n \"\"\"Check if number is even.\"\"\"\n return", "expected": "n % 2 == 0"},
|
| 25 |
+
{"prompt": "def factorial(n: int) -> int:\n \"\"\"Calculate factorial of n.\"\"\"\n if n <= 1:\n return 1\n return", "expected": "n * factorial(n - 1)"},
|
| 26 |
+
{"prompt": "def reverse_string(s: str) -> str:\n \"\"\"Reverse string s.\"\"\"\n return", "expected": "s[::-1]"},
|
| 27 |
+
{"prompt": "def max_element(lst: list) -> int:\n \"\"\"Find maximum in list.\"\"\"\n return", "expected": "max(lst)"},
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
MATH_PROBES = [
|
| 31 |
+
{"prompt": "Q: What is 15 + 27?\nA:", "expected": "42"},
|
| 32 |
+
{"prompt": "Q: If a square has side length 6, what is its area?\nA:", "expected": "36"},
|
| 33 |
+
{"prompt": "Q: What is 12 * 8?\nA:", "expected": "96"},
|
| 34 |
+
{"prompt": "Q: Solve for x: 2*x + 4 = 10.\nA:", "expected": "3"},
|
| 35 |
+
{"prompt": "Q: What is 100 / 4?\nA:", "expected": "25"},
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
REASONING_PROBES = [
|
| 39 |
+
{"prompt": "Question: Is python a programming language?\nAnswer:", "expected": "Yes"},
|
| 40 |
+
{"prompt": "Question: Which planet is closest to the Sun?\nAnswer:", "expected": "Mercury"},
|
| 41 |
+
{"prompt": "Question: What is the capital of France?\nAnswer:", "expected": "Paris"},
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class BenchmarkScores:
|
| 47 |
+
step: int
|
| 48 |
+
tokens_seen: int
|
| 49 |
+
timestamp: str
|
| 50 |
+
humaneval_pass1: float
|
| 51 |
+
gsm8k_accuracy: float
|
| 52 |
+
mmlu_accuracy: float
|
| 53 |
+
overall_benchmark_score: float
|
| 54 |
+
|
| 55 |
+
def to_dict(self) -> dict:
|
| 56 |
+
return asdict(self)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def evaluate_probes(model, tokenizer, device: str = "cuda") -> BenchmarkScores:
|
| 60 |
+
"""Run fast in-training downstream benchmark probes on model."""
|
| 61 |
+
model.eval()
|
| 62 |
+
|
| 63 |
+
# 1. Evaluate Coding Probes (HumanEval / MBPP)
|
| 64 |
+
code_correct = 0
|
| 65 |
+
with torch.no_grad():
|
| 66 |
+
for probe in CODING_PROBES:
|
| 67 |
+
ids = tokenizer.encode(probe["prompt"]).ids
|
| 68 |
+
x = torch.tensor([ids], dtype=torch.long, device=device)
|
| 69 |
+
logits, _ = model(x)
|
| 70 |
+
next_id = torch.argmax(logits[0, -1, :]).item()
|
| 71 |
+
next_str = tokenizer.decode([next_id]).strip()
|
| 72 |
+
if probe["expected"].strip().startswith(next_str) or next_str in probe["expected"]:
|
| 73 |
+
code_correct += 1
|
| 74 |
+
humaneval_score = round((code_correct / len(CODING_PROBES)) * 100.0, 1)
|
| 75 |
+
|
| 76 |
+
# 2. Evaluate Math Probes (GSM8K)
|
| 77 |
+
math_correct = 0
|
| 78 |
+
with torch.no_grad():
|
| 79 |
+
for probe in MATH_PROBES:
|
| 80 |
+
ids = tokenizer.encode(probe["prompt"]).ids
|
| 81 |
+
x = torch.tensor([ids], dtype=torch.long, device=device)
|
| 82 |
+
logits, _ = model(x)
|
| 83 |
+
next_id = torch.argmax(logits[0, -1, :]).item()
|
| 84 |
+
next_str = tokenizer.decode([next_id]).strip()
|
| 85 |
+
if probe["expected"].strip() in next_str or next_str in probe["expected"]:
|
| 86 |
+
math_correct += 1
|
| 87 |
+
gsm8k_score = round((math_correct / len(MATH_PROBES)) * 100.0, 1)
|
| 88 |
+
|
| 89 |
+
# 3. Evaluate Reasoning Probes (MMLU)
|
| 90 |
+
reasoning_correct = 0
|
| 91 |
+
with torch.no_grad():
|
| 92 |
+
for probe in REASONING_PROBES:
|
| 93 |
+
ids = tokenizer.encode(probe["prompt"]).ids
|
| 94 |
+
x = torch.tensor([ids], dtype=torch.long, device=device)
|
| 95 |
+
logits, _ = model(x)
|
| 96 |
+
next_id = torch.argmax(logits[0, -1, :]).item()
|
| 97 |
+
next_str = tokenizer.decode([next_id]).strip()
|
| 98 |
+
if probe["expected"].strip().lower() in next_str.lower():
|
| 99 |
+
reasoning_correct += 1
|
| 100 |
+
mmlu_score = round((reasoning_correct / len(REASONING_PROBES)) * 100.0, 1)
|
| 101 |
+
|
| 102 |
+
overall = round((humaneval_score + gsm8k_score + mmlu_score) / 3.0, 1)
|
| 103 |
+
|
| 104 |
+
model.train()
|
| 105 |
+
return BenchmarkScores(
|
| 106 |
+
step=0,
|
| 107 |
+
tokens_seen=0,
|
| 108 |
+
timestamp=time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 109 |
+
humaneval_pass1=humaneval_score,
|
| 110 |
+
gsm8k_accuracy=gsm8k_score,
|
| 111 |
+
mmlu_accuracy=mmlu_score,
|
| 112 |
+
overall_benchmark_score=overall,
|
| 113 |
+
)
|
code/export_hf.py
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Export Mimans checkpoints to HuggingFace format and upload to HF Hub.
|
| 2 |
+
|
| 3 |
+
Converts Mimans model weights to standard safetensors, outputs config.json,
|
| 4 |
+
modeling_mimans.py, configuration_mimans.py, generate.py, tokenizer files,
|
| 5 |
+
source code, and README.md, and uploads to HuggingFace Hub.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from safetensors.torch import save_file
|
| 18 |
+
|
| 19 |
+
# Add CODE directory to sys.path
|
| 20 |
+
repo_root = Path(__file__).resolve().parents[2]
|
| 21 |
+
code_dir = repo_root / "CODE"
|
| 22 |
+
if str(code_dir) not in sys.path:
|
| 23 |
+
sys.path.insert(0, str(code_dir))
|
| 24 |
+
|
| 25 |
+
from mimans import paths
|
| 26 |
+
from mimans.training import checkpoint as ckpt
|
| 27 |
+
|
| 28 |
+
DEFAULT_HF_REPO = "ankushthakurr09/MimansM1_v1"
|
| 29 |
+
DEFAULT_HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def convert_state_dict_to_hf(state_dict: dict) -> dict:
|
| 33 |
+
"""Map Mimans CodeModel weights to HuggingFace tensor keys."""
|
| 34 |
+
hf_dict = {}
|
| 35 |
+
|
| 36 |
+
if "model" in state_dict:
|
| 37 |
+
model_weights = state_dict["model"]
|
| 38 |
+
elif "state" in state_dict and "model" in state_dict["state"]:
|
| 39 |
+
model_weights = state_dict["state"]["model"]
|
| 40 |
+
else:
|
| 41 |
+
model_weights = state_dict
|
| 42 |
+
|
| 43 |
+
for key, val in model_weights.items():
|
| 44 |
+
tensor = val.detach().cpu().contiguous()
|
| 45 |
+
if tensor.dtype == torch.float32:
|
| 46 |
+
tensor = tensor.to(torch.bfloat16)
|
| 47 |
+
|
| 48 |
+
if key == "embedding.weight":
|
| 49 |
+
hf_dict["model.embed_tokens.weight"] = tensor
|
| 50 |
+
hf_dict["lm_head.weight"] = tensor.clone()
|
| 51 |
+
elif key in ("final_norm.scale", "final_norm.weight"):
|
| 52 |
+
hf_dict["model.norm.weight"] = tensor
|
| 53 |
+
elif key.startswith("blocks."):
|
| 54 |
+
parts = key.split(".")
|
| 55 |
+
layer_idx = parts[1]
|
| 56 |
+
sub = ".".join(parts[2:])
|
| 57 |
+
|
| 58 |
+
prefix = f"model.layers.{layer_idx}"
|
| 59 |
+
if sub in ("attn_norm.scale", "attn_norm.weight"):
|
| 60 |
+
hf_dict[f"{prefix}.input_layernorm.weight"] = tensor
|
| 61 |
+
elif sub in ("ffn_norm.scale", "ffn_norm.weight"):
|
| 62 |
+
hf_dict[f"{prefix}.post_attention_layernorm.weight"] = tensor
|
| 63 |
+
elif sub in ("attn.wq.weight", "attn.wq.scale"):
|
| 64 |
+
hf_dict[f"{prefix}.self_attn.q_proj.weight"] = tensor
|
| 65 |
+
elif sub in ("attn.wk.weight", "attn.wk.scale"):
|
| 66 |
+
hf_dict[f"{prefix}.self_attn.k_proj.weight"] = tensor
|
| 67 |
+
elif sub in ("attn.wv.weight", "attn.wv.scale"):
|
| 68 |
+
hf_dict[f"{prefix}.self_attn.v_proj.weight"] = tensor
|
| 69 |
+
elif sub in ("attn.wo.weight", "attn.wo.scale"):
|
| 70 |
+
hf_dict[f"{prefix}.self_attn.o_proj.weight"] = tensor
|
| 71 |
+
elif sub in ("attn.q_norm.scale", "attn.q_norm.weight"):
|
| 72 |
+
hf_dict[f"{prefix}.self_attn.q_norm.weight"] = tensor
|
| 73 |
+
elif sub in ("attn.k_norm.scale", "attn.k_norm.weight"):
|
| 74 |
+
hf_dict[f"{prefix}.self_attn.k_norm.weight"] = tensor
|
| 75 |
+
elif sub == "ffn.w_gate.weight":
|
| 76 |
+
hf_dict[f"{prefix}.mlp.gate_proj.weight"] = tensor
|
| 77 |
+
elif sub == "ffn.w_up.weight":
|
| 78 |
+
hf_dict[f"{prefix}.mlp.up_proj.weight"] = tensor
|
| 79 |
+
elif sub == "ffn.w_down.weight":
|
| 80 |
+
hf_dict[f"{prefix}.mlp.down_proj.weight"] = tensor
|
| 81 |
+
else:
|
| 82 |
+
hf_dict[f"{prefix}.{sub}"] = tensor
|
| 83 |
+
else:
|
| 84 |
+
hf_dict[key] = tensor
|
| 85 |
+
|
| 86 |
+
return hf_dict
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def build_hf_config(tokens_seen: int = 0) -> dict:
|
| 90 |
+
"""Build standard HuggingFace config.json with custom auto_map."""
|
| 91 |
+
return {
|
| 92 |
+
"architectures": ["MimansForCausalLM"],
|
| 93 |
+
"auto_map": {
|
| 94 |
+
"AutoConfig": "configuration_mimans.MimansConfig",
|
| 95 |
+
"AutoModelForCausalLM": "modeling_mimans.MimansForCausalLM"
|
| 96 |
+
},
|
| 97 |
+
"bos_token_id": 0,
|
| 98 |
+
"eos_token_id": 0,
|
| 99 |
+
"pad_token_id": 6,
|
| 100 |
+
"hidden_act": "silu",
|
| 101 |
+
"hidden_size": 1280,
|
| 102 |
+
"initializer_range": 0.02,
|
| 103 |
+
"intermediate_size": 3328,
|
| 104 |
+
"max_position_embeddings": 8192,
|
| 105 |
+
"model_type": "mimans",
|
| 106 |
+
"num_attention_heads": 10,
|
| 107 |
+
"num_hidden_layers": 27,
|
| 108 |
+
"num_key_value_heads": 2,
|
| 109 |
+
"rms_norm_eps": 1e-05,
|
| 110 |
+
"rope_theta": 500000.0,
|
| 111 |
+
"torch_dtype": "bfloat16",
|
| 112 |
+
"vocab_size": 49152,
|
| 113 |
+
"tie_word_embeddings": True,
|
| 114 |
+
"use_attnres": True,
|
| 115 |
+
"tokens_trained": tokens_seen,
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def build_model_card(repo_id: str, tokens_seen: int, step: int, loss: float = None) -> str:
|
| 120 |
+
"""Generate HuggingFace Model Card README.md."""
|
| 121 |
+
tokens_str = f"{tokens_seen / 1e9:.2f}B" if tokens_seen >= 1e9 else f"{tokens_seen / 1e6:.1f}M"
|
| 122 |
+
loss_str = f"{loss:.4f}" if loss is not None else "N/A"
|
| 123 |
+
|
| 124 |
+
return f"""---
|
| 125 |
+
language:
|
| 126 |
+
- en
|
| 127 |
+
- code
|
| 128 |
+
license: apache-2.0
|
| 129 |
+
tags:
|
| 130 |
+
- code
|
| 131 |
+
- mimans
|
| 132 |
+
- causal-lm
|
| 133 |
+
- blackwell
|
| 134 |
+
- tilelang
|
| 135 |
+
pipeline_tag: text-generation
|
| 136 |
+
---
|
| 137 |
+
|
| 138 |
+
# Mimans M1 (514M) - {tokens_str} Tokens Milestone
|
| 139 |
+
|
| 140 |
+
**Mimans M1** is a 514M parameter decoder-only transformer pretrained from scratch on code, math, and technical text using **TileLang** custom Blackwell GPU kernels and **Muon + AdamW** hybrid optimization.
|
| 141 |
+
|
| 142 |
+
## Model Summary
|
| 143 |
+
- **Parameters:** 514,345,526 (~514M)
|
| 144 |
+
- **Architecture:** GQA (10:2), SwiGLU FFN, AttnRes Skip Gating, RMSNorm
|
| 145 |
+
- **Context Length:** 4,096 tokens (max 8,192, RoPE $\\theta = 500,000$)
|
| 146 |
+
- **Vocabulary:** 49,152 (Byte-level BPE with FIM support)
|
| 147 |
+
- **Training Tokens:** **{tokens_str} Tokens** (Global Step {step:,})
|
| 148 |
+
- **Latest Loss:** **{loss_str}**
|
| 149 |
+
- **Hardware:** NVIDIA GeForce RTX 5090 (Blackwell `sm_120`)
|
| 150 |
+
|
| 151 |
+
## Usage & Generation
|
| 152 |
+
|
| 153 |
+
### Loading with Hugging Face Transformers:
|
| 154 |
+
|
| 155 |
+
```python
|
| 156 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 157 |
+
|
| 158 |
+
repo_id = "{repo_id}"
|
| 159 |
+
model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True)
|
| 160 |
+
tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
|
| 161 |
+
|
| 162 |
+
prompt = "def fibonacci(n):"
|
| 163 |
+
inputs = tokenizer(prompt, return_tensors="pt")
|
| 164 |
+
outputs = model.generate(**inputs, max_new_tokens=50)
|
| 165 |
+
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
### Running Standalone Generation Script:
|
| 169 |
+
|
| 170 |
+
```bash
|
| 171 |
+
python generate.py --prompt "def hello_world():" --max-tokens 50
|
| 172 |
+
```
|
| 173 |
+
"""
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def export_checkpoint(checkpoint_dir: Path, output_dir: Path, repo_id: str = DEFAULT_HF_REPO, export_weights: bool = True) -> Path:
|
| 177 |
+
"""Export checkpoint and Python code files to HuggingFace format."""
|
| 178 |
+
checkpoint_dir = Path(checkpoint_dir)
|
| 179 |
+
output_dir = Path(output_dir)
|
| 180 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 181 |
+
|
| 182 |
+
meta = ckpt.read_metadata(checkpoint_dir) or {}
|
| 183 |
+
step = meta.get("global_step", 0)
|
| 184 |
+
tokens_seen = meta.get("tokens_seen", 0)
|
| 185 |
+
loss = meta.get("loss", None)
|
| 186 |
+
|
| 187 |
+
print(f"[EXPORT-HF] Exporting Mimans M1 (step {step}, {tokens_seen/1e6:.1f}M tokens) to {output_dir}...")
|
| 188 |
+
|
| 189 |
+
# 1. Weights export
|
| 190 |
+
if export_weights:
|
| 191 |
+
state_file = checkpoint_dir / ckpt.STATE_FILE
|
| 192 |
+
if state_file.exists():
|
| 193 |
+
print(" [weights] Loading checkpoint state...")
|
| 194 |
+
raw_state = torch.load(state_file, map_location="cpu", weights_only=False)
|
| 195 |
+
hf_state = convert_state_dict_to_hf(raw_state)
|
| 196 |
+
save_file(hf_state, str(output_dir / "model.safetensors"))
|
| 197 |
+
print(" [ok] model.safetensors written")
|
| 198 |
+
|
| 199 |
+
# 2. Config & AutoMap Python code files
|
| 200 |
+
hf_config = build_hf_config(tokens_seen)
|
| 201 |
+
(output_dir / "config.json").write_text(json.dumps(hf_config, indent=2))
|
| 202 |
+
print(" [ok] config.json written")
|
| 203 |
+
|
| 204 |
+
mimans_code_dir = code_dir / "mimans"
|
| 205 |
+
for filename in ("configuration_mimans.py", "modeling_mimans.py", "generate.py"):
|
| 206 |
+
src = mimans_code_dir / filename
|
| 207 |
+
if src.exists():
|
| 208 |
+
shutil.copy(src, output_dir / filename)
|
| 209 |
+
print(f" [ok] {filename} copied to root")
|
| 210 |
+
|
| 211 |
+
# Copy full source code package into code/ folder
|
| 212 |
+
code_pkg_dir = output_dir / "code"
|
| 213 |
+
if code_pkg_dir.exists():
|
| 214 |
+
shutil.rmtree(code_pkg_dir)
|
| 215 |
+
shutil.copytree(mimans_code_dir, code_pkg_dir)
|
| 216 |
+
print(" [ok] full source code package copied to code/")
|
| 217 |
+
|
| 218 |
+
# 3. Tokenizer files
|
| 219 |
+
tok_file = paths.TOKENIZER / "tokenizer.json"
|
| 220 |
+
if tok_file.exists():
|
| 221 |
+
shutil.copy(tok_file, output_dir / "tokenizer.json")
|
| 222 |
+
tok_cfg = {
|
| 223 |
+
"model_type": "mimans",
|
| 224 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 225 |
+
"bos_token": "<|endoftext|>",
|
| 226 |
+
"eos_token": "<|endoftext|>",
|
| 227 |
+
"pad_token": "<|pad|>",
|
| 228 |
+
"unk_token": "<|endoftext|>",
|
| 229 |
+
"clean_up_tokenization_spaces": False,
|
| 230 |
+
}
|
| 231 |
+
(output_dir / "tokenizer_config.json").write_text(json.dumps(tok_cfg, indent=2))
|
| 232 |
+
|
| 233 |
+
spec_tokens = {
|
| 234 |
+
"bos_token": "<|endoftext|>",
|
| 235 |
+
"eos_token": "<|endoftext|>",
|
| 236 |
+
"pad_token": "<|pad|>",
|
| 237 |
+
"unk_token": "<|endoftext|>",
|
| 238 |
+
}
|
| 239 |
+
(output_dir / "special_tokens_map.json").write_text(json.dumps(spec_tokens, indent=2))
|
| 240 |
+
print(" [ok] tokenizer.json, tokenizer_config.json & special_tokens_map.json written")
|
| 241 |
+
|
| 242 |
+
# 4. Write README.md Model Card
|
| 243 |
+
card_text = build_model_card(repo_id, tokens_seen, step, loss)
|
| 244 |
+
(output_dir / "README.md").write_text(card_text)
|
| 245 |
+
print(" [ok] README.md model card written")
|
| 246 |
+
|
| 247 |
+
print(f"[EXPORT-HF] Export complete -> {output_dir}")
|
| 248 |
+
return output_dir
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def upload_to_hf(export_dir: Path, repo_id: str = DEFAULT_HF_REPO, token: str = DEFAULT_HF_TOKEN) -> bool:
|
| 252 |
+
"""Upload exported model folder to HuggingFace Hub."""
|
| 253 |
+
token = token or DEFAULT_HF_TOKEN or os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
| 254 |
+
if not token:
|
| 255 |
+
print("[UPLOAD-HF] Skipping upload: No HF_TOKEN found.")
|
| 256 |
+
return False
|
| 257 |
+
|
| 258 |
+
try:
|
| 259 |
+
from huggingface_hub import HfApi
|
| 260 |
+
api = HfApi(token=token)
|
| 261 |
+
print(f"[UPLOAD-HF] Connecting to repository {repo_id} on HuggingFace Hub...")
|
| 262 |
+
api.create_repo(repo_id=repo_id, exist_ok=True, private=False)
|
| 263 |
+
|
| 264 |
+
print(f"[UPLOAD-HF] Uploading folder {export_dir} to {repo_id}...")
|
| 265 |
+
api.upload_folder(
|
| 266 |
+
folder_path=str(export_dir),
|
| 267 |
+
repo_id=repo_id,
|
| 268 |
+
commit_message=f"Upload Mimans M1 architecture code & model checkpoint ({export_dir.name})",
|
| 269 |
+
)
|
| 270 |
+
print(f"\n[UPLOAD-HF] SUCCESS! Model & code live at: https://huggingface.co/{repo_id}")
|
| 271 |
+
return True
|
| 272 |
+
except Exception as exc:
|
| 273 |
+
print(f"[UPLOAD-HF] Upload failed: {exc}")
|
| 274 |
+
return False
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def main():
|
| 278 |
+
parser = argparse.ArgumentParser(description="Export and Upload Mimans Model & Code to HuggingFace")
|
| 279 |
+
parser.add_argument("--checkpoint", type=Path, default=None, help="Path to checkpoint directory (default: latest valid)")
|
| 280 |
+
parser.add_argument("--out", type=Path, default=Path("D:/mimanus-training/huggingface/mimans-m1"), help="Output HF export directory")
|
| 281 |
+
parser.add_argument("--repo-id", default=DEFAULT_HF_REPO, help="HuggingFace Hub Repo ID")
|
| 282 |
+
parser.add_argument("--upload", action="store_true", default=True, help="Upload to HuggingFace Hub")
|
| 283 |
+
parser.add_argument("--token", default=DEFAULT_HF_TOKEN, help="HuggingFace API Token")
|
| 284 |
+
args = parser.parse_args()
|
| 285 |
+
|
| 286 |
+
from mimans.config import RunConfig
|
| 287 |
+
ckpt_dir = args.checkpoint
|
| 288 |
+
if ckpt_dir is None:
|
| 289 |
+
ckpt_dir = ckpt.latest_valid(RunConfig().checkpoint_dir)
|
| 290 |
+
if not ckpt_dir:
|
| 291 |
+
raise SystemExit("No valid checkpoints found on disk.")
|
| 292 |
+
|
| 293 |
+
exported_path = export_checkpoint(ckpt_dir, args.out, repo_id=args.repo_id)
|
| 294 |
+
|
| 295 |
+
if args.upload or args.token or os.environ.get("HF_TOKEN"):
|
| 296 |
+
upload_to_hf(exported_path, args.repo_id, token=args.token)
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
if __name__ == "__main__":
|
| 300 |
+
main()
|
code/generate.py
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone inference & token generation script for Mimans M1.
|
| 2 |
+
|
| 3 |
+
This script contains the native Mimans M1 architecture with Attention Residual
|
| 4 |
+
Skip Gating (AttnRes), SwiGLU, and Rotary Position Embeddings (RoPE).
|
| 5 |
+
It does NOT require HuggingFace AutoModelForCausalLM and runs natively on CPU or CUDA.
|
| 6 |
+
|
| 7 |
+
Usage via CLI:
|
| 8 |
+
python generate.py --prompt "def add(a, b):\n return" --temp 0
|
| 9 |
+
python generate.py --prompt "def quicksort(arr):" --max-tokens 150 --temp 0.7
|
| 10 |
+
|
| 11 |
+
Usage via Python:
|
| 12 |
+
from generate import load_model, generate
|
| 13 |
+
model, tokenizer = load_model(".")
|
| 14 |
+
text = generate(model, tokenizer, "def fibonacci(n):", temperature=0.0)
|
| 15 |
+
print(text)
|
| 16 |
+
"""
|
| 17 |
+
import argparse
|
| 18 |
+
import math
|
| 19 |
+
import os
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import List, Optional, Tuple
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
import torch.nn as nn
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
from safetensors.torch import load_file
|
| 28 |
+
from tokenizers import Tokenizer
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# =====================================================================
|
| 32 |
+
# 1. Architecture Components
|
| 33 |
+
# =====================================================================
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class ModelConfig:
|
| 37 |
+
vocab: int = 49152
|
| 38 |
+
d_model: int = 1280
|
| 39 |
+
n_layers: int = 27
|
| 40 |
+
n_heads: int = 10
|
| 41 |
+
n_kv_heads: int = 2
|
| 42 |
+
head_dim: int = 128
|
| 43 |
+
ffn_hidden: int = 3328
|
| 44 |
+
layers_per_block: int = 3
|
| 45 |
+
norm_eps: float = 1e-5
|
| 46 |
+
rope_theta: float = 500_000.0
|
| 47 |
+
max_seq_len: int = 8192
|
| 48 |
+
use_attnres: bool = True
|
| 49 |
+
tied: bool = True
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class RMSNorm(nn.Module):
|
| 53 |
+
def __init__(self, dim: int, eps: float = 1e-5):
|
| 54 |
+
super().__init__()
|
| 55 |
+
self.eps = eps
|
| 56 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 57 |
+
|
| 58 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 59 |
+
orig_dtype = x.dtype
|
| 60 |
+
x_f32 = x.float()
|
| 61 |
+
var = x_f32.pow(2).mean(-1, keepdim=True)
|
| 62 |
+
normed = x_f32 * torch.rsqrt(var + self.eps)
|
| 63 |
+
return (self.weight * normed).to(orig_dtype)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class Rotary(nn.Module):
|
| 67 |
+
def __init__(self, head_dim: int, max_seq_len: int = 8192, theta: float = 500_000.0):
|
| 68 |
+
super().__init__()
|
| 69 |
+
self.head_dim = head_dim
|
| 70 |
+
self.max_seq_len = max_seq_len
|
| 71 |
+
self.theta = theta
|
| 72 |
+
freqs = 1.0 / (theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim))
|
| 73 |
+
angles = torch.outer(torch.arange(self.max_seq_len).float(), freqs)
|
| 74 |
+
table = torch.cat((angles, angles), dim=-1)
|
| 75 |
+
self.register_buffer("cos", table.cos(), persistent=False)
|
| 76 |
+
self.register_buffer("sin", table.sin(), persistent=False)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def rotate_half(x: torch.Tensor) -> torch.Tensor:
|
| 80 |
+
left, right = x.chunk(2, dim=-1)
|
| 81 |
+
return torch.cat((-right, left), dim=-1)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 85 |
+
seq_len = x.shape[1]
|
| 86 |
+
c = cos[:seq_len].to(x.dtype).unsqueeze(1)
|
| 87 |
+
s = sin[:seq_len].to(x.dtype).unsqueeze(1)
|
| 88 |
+
return x * c + rotate_half(x) * s
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class Attention(nn.Module):
|
| 92 |
+
def __init__(self, cfg: ModelConfig, rope: Rotary):
|
| 93 |
+
super().__init__()
|
| 94 |
+
self.n_heads = cfg.n_heads
|
| 95 |
+
self.n_kv_heads = cfg.n_kv_heads
|
| 96 |
+
self.head_dim = cfg.head_dim
|
| 97 |
+
self.d_model = cfg.d_model
|
| 98 |
+
self.repeats = self.n_heads // self.n_kv_heads
|
| 99 |
+
self.rope = rope
|
| 100 |
+
self.scale = 1.0 / math.sqrt(self.head_dim)
|
| 101 |
+
|
| 102 |
+
self.wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False)
|
| 103 |
+
self.wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
|
| 104 |
+
self.wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
|
| 105 |
+
self.wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False)
|
| 106 |
+
|
| 107 |
+
self.q_norm = RMSNorm(cfg.head_dim, cfg.norm_eps)
|
| 108 |
+
self.k_norm = RMSNorm(cfg.head_dim, cfg.norm_eps)
|
| 109 |
+
|
| 110 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 111 |
+
batch, seq_len, _ = x.shape
|
| 112 |
+
q = self.q_norm(self.wq(x).view(batch, seq_len, self.n_heads, self.head_dim))
|
| 113 |
+
k = self.k_norm(self.wk(x).view(batch, seq_len, self.n_kv_heads, self.head_dim))
|
| 114 |
+
v = self.wv(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)
|
| 115 |
+
|
| 116 |
+
q = apply_rope(q, self.rope.cos, self.rope.sin)
|
| 117 |
+
k = apply_rope(k, self.rope.cos, self.rope.sin)
|
| 118 |
+
|
| 119 |
+
qh = q.transpose(1, 2)
|
| 120 |
+
kh = k.transpose(1, 2)
|
| 121 |
+
vh = v.transpose(1, 2)
|
| 122 |
+
|
| 123 |
+
out = F.scaled_dot_product_attention(qh, kh, vh, is_causal=True, scale=self.scale, enable_gqa=True)
|
| 124 |
+
out = out.transpose(1, 2).contiguous().view(batch, seq_len, self.d_model)
|
| 125 |
+
return self.wo(out)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class SwiGLU(nn.Module):
|
| 130 |
+
def __init__(self, cfg: ModelConfig):
|
| 131 |
+
super().__init__()
|
| 132 |
+
self.w_gate = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
|
| 133 |
+
self.w_up = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
|
| 134 |
+
self.w_down = nn.Parameter(torch.empty(cfg.ffn_hidden, cfg.d_model))
|
| 135 |
+
|
| 136 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 137 |
+
gate = F.silu(x @ self.w_gate)
|
| 138 |
+
up = x @ self.w_up
|
| 139 |
+
return (gate * up) @ self.w_down
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class AttnResGate(nn.Module):
|
| 143 |
+
def __init__(self, d_model: int, norm_eps: float = 1e-5):
|
| 144 |
+
super().__init__()
|
| 145 |
+
self.query = nn.Parameter(torch.zeros(d_model))
|
| 146 |
+
self.key_norm = RMSNorm(d_model, norm_eps)
|
| 147 |
+
self.recency_bias = nn.Parameter(torch.zeros(()))
|
| 148 |
+
|
| 149 |
+
def forward(self, entries: List[torch.Tensor]) -> torch.Tensor:
|
| 150 |
+
if len(entries) == 1:
|
| 151 |
+
return entries[0]
|
| 152 |
+
stack = torch.stack(entries, dim=0)
|
| 153 |
+
keys = self.key_norm(stack)
|
| 154 |
+
logits = torch.einsum("d,nbtd->nbt", self.query.to(keys.dtype), keys)
|
| 155 |
+
last = torch.zeros(stack.shape[0], 1, 1, device=logits.device, dtype=logits.dtype)
|
| 156 |
+
last[-1] = 1.0
|
| 157 |
+
logits = logits + last * self.recency_bias.to(logits.dtype)
|
| 158 |
+
weights = logits.softmax(dim=0)
|
| 159 |
+
return torch.einsum("nbt,nbtd->btd", weights, stack)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class ResidualState:
|
| 163 |
+
def __init__(self, running: torch.Tensor):
|
| 164 |
+
self.running = running
|
| 165 |
+
self.snapshots: List[torch.Tensor] = []
|
| 166 |
+
self.steps = 0
|
| 167 |
+
|
| 168 |
+
def advance(self, delta: torch.Tensor, span: int):
|
| 169 |
+
self.running = self.running + delta
|
| 170 |
+
self.steps += 1
|
| 171 |
+
if self.steps % span == 0:
|
| 172 |
+
self.snapshots.append(self.running)
|
| 173 |
+
|
| 174 |
+
def entries(self) -> List[torch.Tensor]:
|
| 175 |
+
return self.snapshots + [self.running]
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
class TransformerBlock(nn.Module):
|
| 179 |
+
|
| 180 |
+
def __init__(self, cfg: ModelConfig, rope: Rotary):
|
| 181 |
+
super().__init__()
|
| 182 |
+
self.cfg = cfg
|
| 183 |
+
self.attn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 184 |
+
self.attn = Attention(cfg, rope)
|
| 185 |
+
self.ffn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 186 |
+
self.ffn = SwiGLU(cfg)
|
| 187 |
+
self.attn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None
|
| 188 |
+
self.ffn_gate = AttnResGate(cfg.d_model, cfg.norm_eps) if cfg.use_attnres else None
|
| 189 |
+
self.sublayers = cfg.layers_per_block * 2
|
| 190 |
+
|
| 191 |
+
def forward(self, state: ResidualState) -> ResidualState:
|
| 192 |
+
h = self.attn_gate(state.entries()) if self.attn_gate else state.running
|
| 193 |
+
delta = self.attn(self.attn_norm(h))
|
| 194 |
+
state.advance(delta, self.sublayers)
|
| 195 |
+
|
| 196 |
+
h = self.ffn_gate(state.entries()) if self.ffn_gate else state.running
|
| 197 |
+
delta = self.ffn(self.ffn_norm(h))
|
| 198 |
+
state.advance(delta, self.sublayers)
|
| 199 |
+
return state
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class LMHead(nn.Module):
|
| 203 |
+
def __init__(self, cfg: ModelConfig, embedding: nn.Embedding):
|
| 204 |
+
super().__init__()
|
| 205 |
+
self.cfg = cfg
|
| 206 |
+
self.embedding = [embedding]
|
| 207 |
+
self.logit_scale = cfg.d_model ** -0.5
|
| 208 |
+
|
| 209 |
+
def forward(self, hidden: torch.Tensor) -> torch.Tensor:
|
| 210 |
+
scaled = hidden * self.logit_scale
|
| 211 |
+
weight = self.embedding[0].weight.t()
|
| 212 |
+
return scaled @ weight
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class CodeModel(nn.Module):
|
| 217 |
+
def __init__(self, cfg: ModelConfig):
|
| 218 |
+
super().__init__()
|
| 219 |
+
self.cfg = cfg
|
| 220 |
+
self.embedding = nn.Embedding(cfg.vocab, cfg.d_model)
|
| 221 |
+
self.rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
|
| 222 |
+
self.blocks = nn.ModuleList([TransformerBlock(cfg, self.rope) for _ in range(cfg.n_layers)])
|
| 223 |
+
self.final_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 224 |
+
self.head = LMHead(cfg, self.embedding)
|
| 225 |
+
|
| 226 |
+
def forward(self, tokens: torch.Tensor) -> torch.Tensor:
|
| 227 |
+
h = self.embedding(tokens)
|
| 228 |
+
state = ResidualState(running=h)
|
| 229 |
+
for block in self.blocks:
|
| 230 |
+
block(state)
|
| 231 |
+
normed = self.final_norm(state.running)
|
| 232 |
+
return self.head(normed)
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
# =====================================================================
|
| 236 |
+
# 2. Weights & Model Loading
|
| 237 |
+
# =====================================================================
|
| 238 |
+
|
| 239 |
+
def remap_hf_safetensors(hf_dict: dict) -> dict:
|
| 240 |
+
"""Remap HuggingFace safetensors keys to native CodeModel state_dict."""
|
| 241 |
+
native = {}
|
| 242 |
+
for k, v in hf_dict.items():
|
| 243 |
+
if k == "model.embed_tokens.weight":
|
| 244 |
+
native["embedding.weight"] = v
|
| 245 |
+
elif k == "model.norm.weight":
|
| 246 |
+
native["final_norm.weight"] = v
|
| 247 |
+
elif k.startswith("model.layers."):
|
| 248 |
+
parts = k.split(".")
|
| 249 |
+
layer_idx = parts[2]
|
| 250 |
+
sub = ".".join(parts[3:])
|
| 251 |
+
prefix = f"blocks.{layer_idx}"
|
| 252 |
+
if sub == "input_layernorm.weight":
|
| 253 |
+
native[f"{prefix}.attn_norm.weight"] = v
|
| 254 |
+
elif sub == "post_attention_layernorm.weight":
|
| 255 |
+
native[f"{prefix}.ffn_norm.weight"] = v
|
| 256 |
+
elif sub == "self_attn.q_proj.weight":
|
| 257 |
+
native[f"{prefix}.attn.wq.weight"] = v
|
| 258 |
+
elif sub == "self_attn.k_proj.weight":
|
| 259 |
+
native[f"{prefix}.attn.wk.weight"] = v
|
| 260 |
+
elif sub == "self_attn.v_proj.weight":
|
| 261 |
+
native[f"{prefix}.attn.wv.weight"] = v
|
| 262 |
+
elif sub == "self_attn.o_proj.weight":
|
| 263 |
+
native[f"{prefix}.attn.wo.weight"] = v
|
| 264 |
+
elif sub == "self_attn.q_norm.weight":
|
| 265 |
+
native[f"{prefix}.attn.q_norm.weight"] = v
|
| 266 |
+
elif sub == "self_attn.k_norm.weight":
|
| 267 |
+
native[f"{prefix}.attn.k_norm.weight"] = v
|
| 268 |
+
elif sub == "mlp.gate_proj.weight":
|
| 269 |
+
native[f"{prefix}.ffn.w_gate"] = v
|
| 270 |
+
elif sub == "mlp.up_proj.weight":
|
| 271 |
+
native[f"{prefix}.ffn.w_up"] = v
|
| 272 |
+
elif sub == "mlp.down_proj.weight":
|
| 273 |
+
native[f"{prefix}.ffn.w_down"] = v
|
| 274 |
+
elif sub in ("w_gate", "w_up", "w_down"):
|
| 275 |
+
native[f"{prefix}.ffn.{sub}"] = v
|
| 276 |
+
else:
|
| 277 |
+
native[f"{prefix}.{sub}"] = v
|
| 278 |
+
elif k != "lm_head.weight":
|
| 279 |
+
native[k] = v
|
| 280 |
+
return native
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def load_model(model_dir_or_repo: str = ".", device: Optional[str] = None):
|
| 284 |
+
"""Load Mimans M1 CodeModel and Tokenizer from local directory or HuggingFace repo."""
|
| 285 |
+
if device is None:
|
| 286 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 287 |
+
|
| 288 |
+
model_path = Path(model_dir_or_repo)
|
| 289 |
+
safetensors_file = model_path / "model.safetensors"
|
| 290 |
+
tokenizer_file = model_path / "tokenizer.json"
|
| 291 |
+
|
| 292 |
+
# If not found locally, attempt to download from Hugging Face Hub
|
| 293 |
+
if not safetensors_file.exists() or not tokenizer_file.exists():
|
| 294 |
+
try:
|
| 295 |
+
from huggingface_hub import hf_hub_download
|
| 296 |
+
repo_id = model_dir_or_repo if "/" in model_dir_or_repo else "ankushthakurr09/MimansM1_v1"
|
| 297 |
+
print(f"Downloading model files from HuggingFace Hub: {repo_id}...")
|
| 298 |
+
safetensors_file = Path(hf_hub_download(repo_id=repo_id, filename="model.safetensors"))
|
| 299 |
+
tokenizer_file = Path(hf_hub_download(repo_id=repo_id, filename="tokenizer.json"))
|
| 300 |
+
except Exception as e:
|
| 301 |
+
raise FileNotFoundError(
|
| 302 |
+
f"Could not find model files locally in '{model_dir_or_repo}' and failed to download: {e}"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
print(f"Loading weights from: {safetensors_file} ...")
|
| 306 |
+
hf_weights = load_file(str(safetensors_file))
|
| 307 |
+
state_dict = remap_hf_safetensors(hf_weights)
|
| 308 |
+
|
| 309 |
+
cfg = ModelConfig()
|
| 310 |
+
model = CodeModel(cfg)
|
| 311 |
+
model.load_state_dict(state_dict, strict=True)
|
| 312 |
+
model = model.to(device)
|
| 313 |
+
if device == "cuda":
|
| 314 |
+
model = model.to(torch.bfloat16)
|
| 315 |
+
model.eval()
|
| 316 |
+
|
| 317 |
+
print(f"Loading tokenizer from: {tokenizer_file} ...")
|
| 318 |
+
tokenizer = Tokenizer.from_file(str(tokenizer_file))
|
| 319 |
+
|
| 320 |
+
print(f"Model loaded successfully on {device} ({sum(p.numel() for p in model.parameters())/1e6:.1f}M params)!")
|
| 321 |
+
return model, tokenizer
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
# =====================================================================
|
| 325 |
+
# 3. Text Generation Engine
|
| 326 |
+
# =====================================================================
|
| 327 |
+
|
| 328 |
+
def generate(
|
| 329 |
+
model: CodeModel,
|
| 330 |
+
tokenizer: Tokenizer,
|
| 331 |
+
prompt: str,
|
| 332 |
+
max_tokens: int = 100,
|
| 333 |
+
temperature: float = 0.0,
|
| 334 |
+
top_k: int = 50,
|
| 335 |
+
repetition_penalty: float = 1.15,
|
| 336 |
+
) -> str:
|
| 337 |
+
"""Generate tokens from prompt using greedy or sampled decoding."""
|
| 338 |
+
device = next(model.parameters()).device
|
| 339 |
+
encoded = tokenizer.encode(prompt)
|
| 340 |
+
input_ids = torch.tensor([encoded.ids], dtype=torch.long, device=device)
|
| 341 |
+
generated_ids = list(encoded.ids)
|
| 342 |
+
|
| 343 |
+
with torch.no_grad():
|
| 344 |
+
for _ in range(max_tokens):
|
| 345 |
+
curr_input = input_ids[:, -model.cfg.max_seq_len :]
|
| 346 |
+
if device.type == "cuda":
|
| 347 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 348 |
+
logits = model(curr_input)
|
| 349 |
+
else:
|
| 350 |
+
logits = model(curr_input)
|
| 351 |
+
|
| 352 |
+
next_logits = logits[0, -1, :].clone()
|
| 353 |
+
|
| 354 |
+
# Greedy decoding (argmax)
|
| 355 |
+
if temperature <= 1e-4:
|
| 356 |
+
next_token = int(torch.argmax(next_logits).item())
|
| 357 |
+
else:
|
| 358 |
+
# Repetition penalty
|
| 359 |
+
if repetition_penalty != 1.0 and len(generated_ids) > 0:
|
| 360 |
+
for token_id in set(generated_ids):
|
| 361 |
+
if next_logits[token_id] < 0:
|
| 362 |
+
next_logits[token_id] *= repetition_penalty
|
| 363 |
+
else:
|
| 364 |
+
next_logits[token_id] /= repetition_penalty
|
| 365 |
+
|
| 366 |
+
next_logits = next_logits / temperature
|
| 367 |
+
if top_k > 0:
|
| 368 |
+
v, _ = torch.topk(next_logits, min(top_k, next_logits.size(-1)))
|
| 369 |
+
next_logits[next_logits < v[-1]] = -float("Inf")
|
| 370 |
+
|
| 371 |
+
probs = F.softmax(next_logits, dim=-1)
|
| 372 |
+
next_token = torch.multinomial(probs, num_samples=1).item()
|
| 373 |
+
|
| 374 |
+
generated_ids.append(next_token)
|
| 375 |
+
input_ids = torch.cat([input_ids, torch.tensor([[next_token]], device=device)], dim=1)
|
| 376 |
+
|
| 377 |
+
if next_token == 0: # <|endoftext|>
|
| 378 |
+
break
|
| 379 |
+
|
| 380 |
+
return tokenizer.decode(generated_ids, skip_special_tokens=False)
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
# =====================================================================
|
| 384 |
+
# 4. Command Line Entry Point
|
| 385 |
+
# =====================================================================
|
| 386 |
+
|
| 387 |
+
if __name__ == "__main__":
|
| 388 |
+
parser = argparse.ArgumentParser(description="Mimans M1 Native Token Generation")
|
| 389 |
+
parser.add_argument("--model-path", default=".", help="Local path or HuggingFace repo ID")
|
| 390 |
+
parser.add_argument("--prompt", default="def add(a, b):\n return", help="Input prompt")
|
| 391 |
+
parser.add_argument("--max-tokens", type=int, default=100, help="Max tokens to generate")
|
| 392 |
+
parser.add_argument("--temp", type=float, default=0.0, help="Temperature (0.0 for greedy)")
|
| 393 |
+
parser.add_argument("--top-k", type=int, default=50, help="Top-K sampling")
|
| 394 |
+
parser.add_argument("--rep-penalty", type=float, default=1.15, help="Repetition penalty")
|
| 395 |
+
parser.add_argument("--device", default=None, help="Device ('cpu' or 'cuda')")
|
| 396 |
+
args = parser.parse_args()
|
| 397 |
+
|
| 398 |
+
model, tokenizer = load_model(args.model_path, device=args.device)
|
| 399 |
+
|
| 400 |
+
print("\n" + "=" * 60)
|
| 401 |
+
print(f"PROMPT:\n{args.prompt}")
|
| 402 |
+
print("=" * 60)
|
| 403 |
+
|
| 404 |
+
output = generate(
|
| 405 |
+
model,
|
| 406 |
+
tokenizer,
|
| 407 |
+
args.prompt,
|
| 408 |
+
max_tokens=args.max_tokens,
|
| 409 |
+
temperature=args.temp,
|
| 410 |
+
top_k=args.top_k,
|
| 411 |
+
repetition_penalty=args.rep_penalty,
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
print("GENERATED OUTPUT:")
|
| 415 |
+
print(output)
|
| 416 |
+
print("=" * 60)
|
code/model/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The architecture, in the order a token moves through it.
|
| 2 |
+
|
| 3 |
+
norm RMSNorm, used everywhere
|
| 4 |
+
attention_tilelang embedding lookup's first consumer: rope, gqa, flash
|
| 5 |
+
swiglu_tilelang the feed forward
|
| 6 |
+
attnres the residual gate that carries state between blocks
|
| 7 |
+
block one transformer block, three layers deep
|
| 8 |
+
model CodeModel: embedding -> 27 blocks -> tied lm head
|
| 9 |
+
lm_head_tilelang the tied head and its fused cross entropy
|
| 10 |
+
"""
|
code/model/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (711 Bytes). View file
|
|
|
code/model/__pycache__/attention_tilelang.cpython-311.pyc
ADDED
|
Binary file (66.3 kB). View file
|
|
|
code/model/__pycache__/attnres.cpython-311.pyc
ADDED
|
Binary file (14 kB). View file
|
|
|
code/model/__pycache__/block.cpython-311.pyc
ADDED
|
Binary file (17.8 kB). View file
|
|
|
code/model/__pycache__/lm_head_tilelang.cpython-311.pyc
ADDED
|
Binary file (42 kB). View file
|
|
|
code/model/__pycache__/model.cpython-311.pyc
ADDED
|
Binary file (25.8 kB). View file
|
|
|
code/model/__pycache__/norm.cpython-311.pyc
ADDED
|
Binary file (14.9 kB). View file
|
|
|
code/model/__pycache__/swiglu_tilelang.cpython-311.pyc
ADDED
|
Binary file (51.3 kB). View file
|
|
|
code/model/__pycache__/tuned.cpython-311.pyc
ADDED
|
Binary file (6.86 kB). View file
|
|
|
code/model/attention_tilelang.py
ADDED
|
@@ -0,0 +1,788 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# no "from __future__ import annotations" here: tilelang reads the prim_func
|
| 2 |
+
# signature with get_type_hints, and PEP 563 would leave T.Tensor([batch, ...])
|
| 3 |
+
# as a string that cannot see the enclosing function's locals.
|
| 4 |
+
import argparse
|
| 5 |
+
import itertools
|
| 6 |
+
import math
|
| 7 |
+
import os
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from mimans import paths # noqa: F401 points the kernel cache at D:, must precede tilelang
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
import tilelang
|
| 18 |
+
import tilelang.language as T
|
| 19 |
+
from tilelang.autotuner import autotune
|
| 20 |
+
HAVE_TILELANG = True
|
| 21 |
+
except ImportError:
|
| 22 |
+
HAVE_TILELANG = False
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
DTYPE = torch.bfloat16
|
| 26 |
+
SMEM_PER_SM_KB = {8: 164, 9: 228, 10: 228, 12: 128}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def shared_mem_limit(device=0, margin=0.9):
|
| 30 |
+
try:
|
| 31 |
+
if not torch.cuda.is_available():
|
| 32 |
+
return 112 * 1024
|
| 33 |
+
props = torch.cuda.get_device_properties(device)
|
| 34 |
+
for name in ("shared_memory_per_block_optin",
|
| 35 |
+
"max_shared_memory_per_block_optin",
|
| 36 |
+
"shared_memory_per_block"):
|
| 37 |
+
value = getattr(props, name, 0)
|
| 38 |
+
if value:
|
| 39 |
+
return int(value * margin)
|
| 40 |
+
return int(SMEM_PER_SM_KB.get(props.major, 100) * 1024 * margin)
|
| 41 |
+
except Exception:
|
| 42 |
+
return 112 * 1024
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
SMEM_LIMIT = shared_mem_limit()
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def fwd_smem(block_m, block_n, dim, stages, elem=2):
|
| 49 |
+
return elem * dim * (2 * block_m + 2 * stages * block_n)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def bwd_smem(block_m, block_n, dim, stages, elem=2):
|
| 53 |
+
in_loop = (2 * elem * block_m * dim
|
| 54 |
+
+ 2 * elem * stages * block_n * dim
|
| 55 |
+
+ elem * block_m * block_n
|
| 56 |
+
+ 8 * block_n)
|
| 57 |
+
tail = 8 * block_m * dim
|
| 58 |
+
return max(in_loop, tail)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def warps_fit(block, threads, granularity=16):
|
| 62 |
+
warps = threads // 32
|
| 63 |
+
return block % warps == 0 and (block // warps) % granularity == 0
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def fwd_configs(dim=128, limit=None):
|
| 67 |
+
limit = limit or SMEM_LIMIT
|
| 68 |
+
return [dict(block_M=bm, block_N=bn, threads=t, num_stages=s)
|
| 69 |
+
for bm, bn in itertools.product((32, 64, 128, 256), repeat=2)
|
| 70 |
+
for t in (128, 256, 512) if warps_fit(bm, t)
|
| 71 |
+
for s in (1, 2, 3) if fwd_smem(bm, bn, dim, s) <= limit]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def bwd_configs(dim=128, limit=None):
|
| 75 |
+
limit = limit or SMEM_LIMIT
|
| 76 |
+
return [dict(block_M=bm, block_N=bn, threads=t, num_stages=s)
|
| 77 |
+
for bm, bn in itertools.product((32, 64, 128), repeat=2)
|
| 78 |
+
for t in (128, 256) if warps_fit(bm, t)
|
| 79 |
+
for s in (1, 2, 3) if bwd_smem(bm, bn, dim, s) <= limit]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def fit_default(preferred, space):
|
| 83 |
+
"""Keep the preferred tiles when the device allows them, otherwise take the
|
| 84 |
+
widest config of the same shape that fits. sm_120 opts in to 99 KB per
|
| 85 |
+
block, not the 128 KB per SM, so 128x64 stages=1 (96 KB) is out of reach."""
|
| 86 |
+
if preferred in space:
|
| 87 |
+
return preferred
|
| 88 |
+
same = [c for c in space
|
| 89 |
+
if all(c[k] == preferred[k] for k in ("block_M", "threads", "num_stages"))]
|
| 90 |
+
assert same, f"no config near {preferred} fits in {SMEM_LIMIT / 1024:.0f} KB"
|
| 91 |
+
return max(same, key=lambda c: c["block_N"])
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
AUTOTUNE = True
|
| 95 |
+
FWD_TILES = fit_default(dict(block_M=128, block_N=64, threads=256, num_stages=1),
|
| 96 |
+
fwd_configs())
|
| 97 |
+
BWD_TILES = fit_default(dict(block_M=64, block_N=32, threads=128, num_stages=2),
|
| 98 |
+
bwd_configs())
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if HAVE_TILELANG:
|
| 102 |
+
FAST_MATH = {tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True}
|
| 103 |
+
# CUDA 13.1's host_config.h only blesses MSVC 2019-2022 and this box has
|
| 104 |
+
# VS 18, so nvcc refuses every kernel. Override it here; unset
|
| 105 |
+
# TL_NVCC_FLAGS (or install the 2022 build tools) to get the check back.
|
| 106 |
+
JIT_FLAGS = os.environ.get("TL_NVCC_FLAGS", "-allow-unsupported-compiler").split()
|
| 107 |
+
FWD_SPACE = fwd_configs()
|
| 108 |
+
BWD_SPACE = bwd_configs()
|
| 109 |
+
|
| 110 |
+
@autotune(configs=FWD_SPACE, warmup=10, rep=20)
|
| 111 |
+
@tilelang.jit(out_idx=[3, 4], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 112 |
+
def flash_fwd(batch, heads, seq_len, dim, causal, groups,
|
| 113 |
+
block_M=128, block_N=64, threads=256, num_stages=1):
|
| 114 |
+
scale = (1.0 / dim) ** 0.5 * 1.44269504
|
| 115 |
+
kv_heads = heads // groups
|
| 116 |
+
dt, acc = T.bfloat16, T.float32
|
| 117 |
+
|
| 118 |
+
@T.prim_func
|
| 119 |
+
def kernel(
|
| 120 |
+
Q: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 121 |
+
K: T.Tensor([batch, seq_len, kv_heads, dim], dt), #type: ignore
|
| 122 |
+
V: T.Tensor([batch, seq_len, kv_heads, dim], dt), #type: ignore
|
| 123 |
+
O: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 124 |
+
lse: T.Tensor([batch, heads, seq_len], acc), #type: ignore
|
| 125 |
+
):
|
| 126 |
+
with T.Kernel(T.ceildiv(seq_len, block_M), heads, batch,
|
| 127 |
+
threads=threads) as (bx, by, bz):
|
| 128 |
+
q_s = T.alloc_shared([block_M, dim], dt)
|
| 129 |
+
k_s = T.alloc_shared([block_N, dim], dt)
|
| 130 |
+
v_s = T.alloc_shared([block_N, dim], dt)
|
| 131 |
+
o_s = T.alloc_shared([block_M, dim], dt)
|
| 132 |
+
scores = T.alloc_fragment([block_M, block_N], acc)
|
| 133 |
+
probs = T.alloc_fragment([block_M, block_N], dt)
|
| 134 |
+
out = T.alloc_fragment([block_M, dim], acc)
|
| 135 |
+
row_max = T.alloc_fragment([block_M], acc)
|
| 136 |
+
prev_max = T.alloc_fragment([block_M], acc)
|
| 137 |
+
rescale = T.alloc_fragment([block_M], acc)
|
| 138 |
+
row_sum = T.alloc_fragment([block_M], acc)
|
| 139 |
+
denom = T.alloc_fragment([block_M], acc)
|
| 140 |
+
|
| 141 |
+
T.copy(Q[bz, bx * block_M:(bx + 1) * block_M, by, :], q_s)
|
| 142 |
+
T.fill(out, 0)
|
| 143 |
+
T.fill(denom, 0)
|
| 144 |
+
T.fill(row_max, -T.infinity(acc))
|
| 145 |
+
|
| 146 |
+
steps = (T.min(T.ceildiv(seq_len, block_N),
|
| 147 |
+
T.ceildiv((bx + 1) * block_M, block_N))
|
| 148 |
+
if causal else T.ceildiv(seq_len, block_N))
|
| 149 |
+
|
| 150 |
+
for k in T.Pipelined(steps, num_stages=num_stages):
|
| 151 |
+
T.copy(K[bz, k * block_N:(k + 1) * block_N, by // groups, :], k_s)
|
| 152 |
+
if causal:
|
| 153 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 154 |
+
scores[i, j] = T.if_then_else(
|
| 155 |
+
bx * block_M + i >= k * block_N + j,
|
| 156 |
+
0, -T.infinity(acc))
|
| 157 |
+
else:
|
| 158 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 159 |
+
scores[i, j] = T.if_then_else(
|
| 160 |
+
k * block_N + j >= seq_len, -T.infinity(acc), 0)
|
| 161 |
+
T.gemm(q_s, k_s, scores, transpose_B=True,
|
| 162 |
+
policy=T.GemmWarpPolicy.FullRow)
|
| 163 |
+
|
| 164 |
+
T.copy(row_max, prev_max)
|
| 165 |
+
T.fill(row_max, -T.infinity(acc))
|
| 166 |
+
T.reduce_max(scores, row_max, dim=1, clear=False)
|
| 167 |
+
for i in T.Parallel(block_M):
|
| 168 |
+
row_max[i] = T.max(row_max[i], prev_max[i])
|
| 169 |
+
for i in T.Parallel(block_M):
|
| 170 |
+
rescale[i] = T.exp2(prev_max[i] * scale - row_max[i] * scale)
|
| 171 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 172 |
+
scores[i, j] = T.exp2(scores[i, j] * scale - row_max[i] * scale)
|
| 173 |
+
T.reduce_sum(scores, row_sum, dim=1)
|
| 174 |
+
for i in T.Parallel(block_M):
|
| 175 |
+
denom[i] = denom[i] * rescale[i] + row_sum[i]
|
| 176 |
+
T.copy(scores, probs)
|
| 177 |
+
for i, j in T.Parallel(block_M, dim):
|
| 178 |
+
out[i, j] *= rescale[i]
|
| 179 |
+
T.copy(V[bz, k * block_N:(k + 1) * block_N, by // groups, :], v_s)
|
| 180 |
+
T.gemm(probs, v_s, out, policy=T.GemmWarpPolicy.FullRow)
|
| 181 |
+
|
| 182 |
+
for i, j in T.Parallel(block_M, dim):
|
| 183 |
+
out[i, j] /= denom[i]
|
| 184 |
+
T.copy(out, o_s)
|
| 185 |
+
T.copy(o_s, O[bz, bx * block_M:(bx + 1) * block_M, by, :])
|
| 186 |
+
for i in T.Parallel(block_M):
|
| 187 |
+
denom[i] = T.log2(denom[i]) + row_max[i] * scale
|
| 188 |
+
T.copy(denom, lse[bz, by, bx * block_M:(bx + 1) * block_M])
|
| 189 |
+
|
| 190 |
+
return kernel
|
| 191 |
+
|
| 192 |
+
@tilelang.jit(out_idx=[2], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 193 |
+
def flash_delta(batch, heads, seq_len, dim):
|
| 194 |
+
dt, acc = T.bfloat16, T.float32
|
| 195 |
+
blk = 32
|
| 196 |
+
|
| 197 |
+
@T.prim_func
|
| 198 |
+
def kernel(
|
| 199 |
+
O: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 200 |
+
dO: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 201 |
+
Delta: T.Tensor([batch, heads, seq_len], acc), #type: ignore
|
| 202 |
+
):
|
| 203 |
+
with T.Kernel(heads, T.ceildiv(seq_len, blk), batch) as (bx, by, bz):
|
| 204 |
+
o = T.alloc_fragment([blk, blk], dt)
|
| 205 |
+
do = T.alloc_fragment([blk, blk], dt)
|
| 206 |
+
prod = T.alloc_fragment([blk, blk], acc)
|
| 207 |
+
total = T.alloc_fragment([blk], acc)
|
| 208 |
+
T.clear(prod)
|
| 209 |
+
for k in range(T.ceildiv(dim, blk)):
|
| 210 |
+
T.copy(O[bz, by * blk:(by + 1) * blk, bx, k * blk:(k + 1) * blk], o)
|
| 211 |
+
T.copy(dO[bz, by * blk:(by + 1) * blk, bx, k * blk:(k + 1) * blk], do)
|
| 212 |
+
for i, j in T.Parallel(blk, blk):
|
| 213 |
+
prod[i, j] += o[i, j] * do[i, j]
|
| 214 |
+
T.reduce_sum(prod, total, 1)
|
| 215 |
+
T.copy(total, Delta[bz, bx, by * blk:(by + 1) * blk])
|
| 216 |
+
|
| 217 |
+
return kernel
|
| 218 |
+
|
| 219 |
+
def dq_layout(dQ):
|
| 220 |
+
return T.Layout(dQ.shape,
|
| 221 |
+
lambda b, l, h, d: [b, l // 8, h, d // 8, (d % 2),
|
| 222 |
+
4 * (l % 8) + (d % 8) // 2])
|
| 223 |
+
|
| 224 |
+
@tilelang.jit(out_idx=[1], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 225 |
+
def flash_unswizzle(batch, heads, seq_len, dim):
|
| 226 |
+
dt, acc = T.bfloat16, T.float32
|
| 227 |
+
blk = 64
|
| 228 |
+
|
| 229 |
+
@T.prim_func
|
| 230 |
+
def kernel(
|
| 231 |
+
dQ: T.Tensor([batch, seq_len, heads, dim], acc), #type: ignore
|
| 232 |
+
dQ_out: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 233 |
+
):
|
| 234 |
+
with T.Kernel(T.ceildiv(seq_len, blk), heads, batch,
|
| 235 |
+
threads=128) as (bx, by, bz):
|
| 236 |
+
T.annotate_layout({dQ: dq_layout(dQ)})
|
| 237 |
+
T.copy(dQ[bz, bx * blk:(bx + 1) * blk, by, :],
|
| 238 |
+
dQ_out[bz, bx * blk:(bx + 1) * blk, by, :])
|
| 239 |
+
|
| 240 |
+
return kernel
|
| 241 |
+
|
| 242 |
+
@autotune(configs=BWD_SPACE, warmup=10, rep=20)
|
| 243 |
+
@tilelang.jit(pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 244 |
+
def flash_bwd(batch, heads, seq_len, dim, causal, groups,
|
| 245 |
+
block_M=64, block_N=32, threads=128, num_stages=2):
|
| 246 |
+
sm_scale = (1.0 / dim) ** 0.5
|
| 247 |
+
scale = sm_scale * 1.44269504
|
| 248 |
+
kv_heads = heads // groups
|
| 249 |
+
dt, acc = T.bfloat16, T.float32
|
| 250 |
+
|
| 251 |
+
@T.prim_func
|
| 252 |
+
def kernel(
|
| 253 |
+
Q: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 254 |
+
K: T.Tensor([batch, seq_len, kv_heads, dim], dt), #type: ignore
|
| 255 |
+
V: T.Tensor([batch, seq_len, kv_heads, dim], dt), #type: ignore
|
| 256 |
+
dO: T.Tensor([batch, seq_len, heads, dim], dt), #type: ignore
|
| 257 |
+
lse: T.Tensor([batch, heads, seq_len], acc), #type: ignore
|
| 258 |
+
Delta: T.Tensor([batch, heads, seq_len], acc), #type: ignore
|
| 259 |
+
dQ: T.Tensor([batch, seq_len, heads, dim], acc), #type: ignore
|
| 260 |
+
dK: T.Tensor([batch, seq_len, kv_heads, dim], acc), #type: ignore
|
| 261 |
+
dV: T.Tensor([batch, seq_len, kv_heads, dim], acc), #type: ignore
|
| 262 |
+
):
|
| 263 |
+
with T.Kernel(heads, T.ceildiv(seq_len, block_M), batch,
|
| 264 |
+
threads=threads) as (bx, by, bz):
|
| 265 |
+
k_s = T.alloc_shared([block_M, dim], dt)
|
| 266 |
+
v_s = T.alloc_shared([block_M, dim], dt)
|
| 267 |
+
q_s = T.alloc_shared([block_N, dim], dt)
|
| 268 |
+
do_s = T.alloc_shared([block_N, dim], dt)
|
| 269 |
+
ds_s = T.alloc_shared([block_M, block_N], dt)
|
| 270 |
+
lse_s = T.alloc_shared([block_N], acc)
|
| 271 |
+
delta_s = T.alloc_shared([block_N], acc)
|
| 272 |
+
probs = T.alloc_fragment([block_M, block_N], acc)
|
| 273 |
+
dscore = T.alloc_fragment([block_M, block_N], acc)
|
| 274 |
+
probs_c = T.alloc_fragment([block_M, block_N], dt)
|
| 275 |
+
dscore_c = T.alloc_fragment([block_M, block_N], dt)
|
| 276 |
+
dv = T.alloc_fragment([block_M, dim], acc)
|
| 277 |
+
dk = T.alloc_fragment([block_M, dim], acc)
|
| 278 |
+
dq = T.alloc_fragment([block_N, dim], acc)
|
| 279 |
+
dk_s = T.alloc_shared([block_M, dim], acc)
|
| 280 |
+
dv_s = T.alloc_shared([block_M, dim], acc)
|
| 281 |
+
|
| 282 |
+
T.annotate_layout({dQ: dq_layout(dQ)})
|
| 283 |
+
T.copy(K[bz, by * block_M:(by + 1) * block_M, bx // groups, :], k_s)
|
| 284 |
+
T.copy(V[bz, by * block_M:(by + 1) * block_M, bx // groups, :], v_s)
|
| 285 |
+
T.clear(dv)
|
| 286 |
+
T.clear(dk)
|
| 287 |
+
|
| 288 |
+
start = T.floordiv(by * block_M, block_N) if causal else 0
|
| 289 |
+
stop = T.ceildiv(seq_len, block_N)
|
| 290 |
+
for k in T.Pipelined(start, stop, num_stages=num_stages):
|
| 291 |
+
T.copy(Q[bz, k * block_N:(k + 1) * block_N, bx, :], q_s)
|
| 292 |
+
T.clear(probs)
|
| 293 |
+
T.gemm(k_s, q_s, probs, transpose_B=True,
|
| 294 |
+
policy=T.GemmWarpPolicy.FullRow)
|
| 295 |
+
T.copy(lse[bz, bx, k * block_N:(k + 1) * block_N], lse_s)
|
| 296 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 297 |
+
probs[i, j] = T.exp2(probs[i, j] * scale - lse_s[j])
|
| 298 |
+
if causal:
|
| 299 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 300 |
+
probs[i, j] = T.if_then_else(
|
| 301 |
+
by * block_M + i <= k * block_N + j, probs[i, j], 0)
|
| 302 |
+
T.copy(dO[bz, k * block_N:(k + 1) * block_N, bx, :], do_s)
|
| 303 |
+
T.clear(dscore)
|
| 304 |
+
T.gemm(v_s, do_s, dscore, transpose_B=True,
|
| 305 |
+
policy=T.GemmWarpPolicy.FullRow)
|
| 306 |
+
T.copy(probs, probs_c)
|
| 307 |
+
T.gemm(probs_c, do_s, dv, policy=T.GemmWarpPolicy.FullRow)
|
| 308 |
+
T.copy(Delta[bz, bx, k * block_N:(k + 1) * block_N], delta_s)
|
| 309 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 310 |
+
dscore_c[i, j] = probs[i, j] * (dscore[i, j] - delta_s[j]) * sm_scale
|
| 311 |
+
T.gemm(dscore_c, q_s, dk, policy=T.GemmWarpPolicy.FullRow)
|
| 312 |
+
T.copy(dscore_c, ds_s)
|
| 313 |
+
T.clear(dq)
|
| 314 |
+
T.gemm(ds_s, k_s, dq, transpose_A=True)
|
| 315 |
+
for i, j in T.Parallel(block_N, dim):
|
| 316 |
+
T.atomic_add(dQ[bz, k * block_N + i, bx, j], dq[i, j])
|
| 317 |
+
|
| 318 |
+
T.copy(dv, dv_s)
|
| 319 |
+
T.atomic_add(dV[bz, by * block_M:(by + 1) * block_M, bx // groups, :], dv_s)
|
| 320 |
+
T.copy(dk, dk_s)
|
| 321 |
+
T.atomic_add(dK[bz, by * block_M:(by + 1) * block_M, bx // groups, :], dk_s)
|
| 322 |
+
|
| 323 |
+
return kernel
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
_kernels: dict = {}
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def get_kernel(kind, *shape):
|
| 330 |
+
key = (kind, shape, AUTOTUNE)
|
| 331 |
+
if key not in _kernels:
|
| 332 |
+
from mimans.model import tuned
|
| 333 |
+
|
| 334 |
+
fn = flash_fwd if kind == "fwd" else flash_bwd
|
| 335 |
+
name = f"attention_{kind}"
|
| 336 |
+
remembered = tuned.lookup(name, shape)
|
| 337 |
+
if remembered is not None:
|
| 338 |
+
# Tuned on this card already; build it straight away.
|
| 339 |
+
kernel = fn(*shape, **remembered)
|
| 340 |
+
elif AUTOTUNE:
|
| 341 |
+
kernel = fn(*shape)
|
| 342 |
+
tuned.record(name, shape, tuned.config_of(kernel))
|
| 343 |
+
else:
|
| 344 |
+
kernel = fn(*shape, **(FWD_TILES if kind == "fwd" else BWD_TILES))
|
| 345 |
+
_kernels[key] = kernel
|
| 346 |
+
return _kernels[key]
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def warmup(batch, seq_len, cfg=None, causal=True):
|
| 350 |
+
"""Run both autotune searches up front so training never stalls mid-step."""
|
| 351 |
+
cfg = cfg or AttnConfig()
|
| 352 |
+
shape = (batch, cfg.n_heads, seq_len, cfg.head_dim, causal, cfg.groups)
|
| 353 |
+
print(f"tuning forward over {len(FWD_SPACE)} configs")
|
| 354 |
+
get_kernel("fwd", *shape)
|
| 355 |
+
print(f"tuning backward over {len(BWD_SPACE)} configs")
|
| 356 |
+
get_kernel("bwd", *shape)
|
| 357 |
+
print("cached to disk")
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
class FlashAttention(torch.autograd.Function):
|
| 361 |
+
@staticmethod
|
| 362 |
+
def forward(ctx, q, k, v, causal, groups):
|
| 363 |
+
batch, seq_len, heads, dim = q.shape
|
| 364 |
+
out, lse = get_kernel("fwd", batch, heads, seq_len, dim, causal, groups)(q, k, v)
|
| 365 |
+
ctx.save_for_backward(q, k, v, out, lse)
|
| 366 |
+
ctx.causal, ctx.groups = causal, groups
|
| 367 |
+
return out
|
| 368 |
+
|
| 369 |
+
@staticmethod
|
| 370 |
+
def backward(ctx, grad_out):
|
| 371 |
+
q, k, v, out, lse = ctx.saved_tensors
|
| 372 |
+
batch, seq_len, heads, dim = q.shape
|
| 373 |
+
kv_heads = v.shape[-2]
|
| 374 |
+
groups = heads // kv_heads
|
| 375 |
+
|
| 376 |
+
tensors = [t if t.stride(-1) == 1 else t.contiguous()
|
| 377 |
+
for t in (grad_out, q, k, v, out)]
|
| 378 |
+
grad_out, q, k, v, out = tensors
|
| 379 |
+
|
| 380 |
+
delta = flash_delta(batch, heads, seq_len, dim)(out, grad_out)
|
| 381 |
+
dq = torch.zeros(batch, seq_len, heads, dim, dtype=torch.float32, device=q.device)
|
| 382 |
+
dk = torch.zeros(batch, seq_len, kv_heads, dim, dtype=torch.float32, device=q.device)
|
| 383 |
+
dv = torch.zeros(batch, seq_len, kv_heads, dim, dtype=torch.float32, device=q.device)
|
| 384 |
+
|
| 385 |
+
get_kernel("bwd", batch, heads, seq_len, dim, ctx.causal, groups)(
|
| 386 |
+
q, k, v, grad_out, lse, delta, dq, dk, dv)
|
| 387 |
+
|
| 388 |
+
dq = flash_unswizzle(batch, heads, seq_len, dim)(dq)
|
| 389 |
+
return dq, dk.to(q.dtype), dv.to(q.dtype), None, None
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def flash_attention(q, k, v, causal=True, groups=5):
|
| 393 |
+
return FlashAttention.apply(q, k, v, causal, groups)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
@dataclass
|
| 397 |
+
class AttnConfig:
|
| 398 |
+
d_model: int = 1280
|
| 399 |
+
n_heads: int = 10
|
| 400 |
+
n_kv_heads: int = 2
|
| 401 |
+
head_dim: int = 128
|
| 402 |
+
rope_theta: float = 10_000.0
|
| 403 |
+
max_seq_len: int = 8192
|
| 404 |
+
qk_norm: bool = True
|
| 405 |
+
norm_eps: float = 1e-5
|
| 406 |
+
backend: str = "tilelang"
|
| 407 |
+
|
| 408 |
+
def __post_init__(self):
|
| 409 |
+
assert self.n_heads % self.n_kv_heads == 0
|
| 410 |
+
assert self.n_heads * self.head_dim == self.d_model
|
| 411 |
+
assert self.head_dim in (64, 128)
|
| 412 |
+
|
| 413 |
+
@property
|
| 414 |
+
def groups(self):
|
| 415 |
+
return self.n_heads // self.n_kv_heads
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
class RMSNorm(nn.Module):
|
| 419 |
+
def __init__(self, dim, eps=1e-5):
|
| 420 |
+
super().__init__()
|
| 421 |
+
self.eps = eps
|
| 422 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 423 |
+
|
| 424 |
+
def forward(self, x):
|
| 425 |
+
dtype = x.dtype
|
| 426 |
+
x = x.float()
|
| 427 |
+
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 428 |
+
return (x * self.weight.float()).to(dtype)
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
class Rotary(nn.Module):
|
| 432 |
+
def __init__(self, head_dim, max_seq_len, theta):
|
| 433 |
+
super().__init__()
|
| 434 |
+
self.head_dim = head_dim
|
| 435 |
+
self.max_seq_len = max_seq_len
|
| 436 |
+
self.build(theta)
|
| 437 |
+
|
| 438 |
+
def build(self, theta):
|
| 439 |
+
self.theta = theta
|
| 440 |
+
freqs = 1.0 / (theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim))
|
| 441 |
+
angles = torch.outer(torch.arange(self.max_seq_len).float(), freqs)
|
| 442 |
+
table = torch.cat((angles, angles), dim=-1)
|
| 443 |
+
self.register_buffer("cos", table.cos(), persistent=False)
|
| 444 |
+
self.register_buffer("sin", table.sin(), persistent=False)
|
| 445 |
+
|
| 446 |
+
def set_theta(self, theta):
|
| 447 |
+
device = self.cos.device
|
| 448 |
+
self.build(theta)
|
| 449 |
+
self.cos, self.sin = self.cos.to(device), self.sin.to(device)
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def rotate_half(x):
|
| 453 |
+
left, right = x.chunk(2, dim=-1)
|
| 454 |
+
return torch.cat((-right, left), dim=-1)
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def apply_rope(x, cos, sin):
|
| 458 |
+
seq_len = x.shape[1]
|
| 459 |
+
c = cos[:seq_len].to(x.dtype).unsqueeze(1)
|
| 460 |
+
s = sin[:seq_len].to(x.dtype).unsqueeze(1)
|
| 461 |
+
return x * c + rotate_half(x) * s
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
class Attention(nn.Module):
|
| 465 |
+
def __init__(self, cfg: AttnConfig, rope: Rotary):
|
| 466 |
+
super().__init__()
|
| 467 |
+
self.cfg = cfg
|
| 468 |
+
self.rope = rope
|
| 469 |
+
heads, kv_heads, dim = cfg.n_heads, cfg.n_kv_heads, cfg.head_dim
|
| 470 |
+
self.wq = nn.Linear(cfg.d_model, heads * dim, bias=False)
|
| 471 |
+
self.wk = nn.Linear(cfg.d_model, kv_heads * dim, bias=False)
|
| 472 |
+
self.wv = nn.Linear(cfg.d_model, kv_heads * dim, bias=False)
|
| 473 |
+
self.wo = nn.Linear(heads * dim, cfg.d_model, bias=False)
|
| 474 |
+
self.q_norm = RMSNorm(dim, cfg.norm_eps) if cfg.qk_norm else nn.Identity()
|
| 475 |
+
self.k_norm = RMSNorm(dim, cfg.norm_eps) if cfg.qk_norm else nn.Identity()
|
| 476 |
+
self.scale = 1.0 / math.sqrt(dim)
|
| 477 |
+
|
| 478 |
+
def forward(self, x, backend=None):
|
| 479 |
+
batch, seq_len, _ = x.shape
|
| 480 |
+
cfg = self.cfg
|
| 481 |
+
backend = backend or cfg.backend
|
| 482 |
+
|
| 483 |
+
q = self.q_norm(self.wq(x).view(batch, seq_len, cfg.n_heads, cfg.head_dim))
|
| 484 |
+
k = self.k_norm(self.wk(x).view(batch, seq_len, cfg.n_kv_heads, cfg.head_dim))
|
| 485 |
+
v = self.wv(x).view(batch, seq_len, cfg.n_kv_heads, cfg.head_dim)
|
| 486 |
+
q = apply_rope(q, self.rope.cos, self.rope.sin)
|
| 487 |
+
k = apply_rope(k, self.rope.cos, self.rope.sin)
|
| 488 |
+
|
| 489 |
+
if backend == "tilelang":
|
| 490 |
+
# bf16 kernel. Under autocast q/k/v arrive bf16 already; this keeps
|
| 491 |
+
# the backend usable outside autocast too.
|
| 492 |
+
out = flash_attention(q.to(DTYPE).contiguous(), k.to(DTYPE).contiguous(),
|
| 493 |
+
v.to(DTYPE).contiguous(), True, cfg.groups)
|
| 494 |
+
out = out.to(x.dtype)
|
| 495 |
+
else:
|
| 496 |
+
qh, kh, vh = (t.transpose(1, 2) for t in (q, k, v))
|
| 497 |
+
if backend == "sdpa":
|
| 498 |
+
out = F.scaled_dot_product_attention(
|
| 499 |
+
qh, kh, vh, is_causal=True, scale=self.scale, enable_gqa=True)
|
| 500 |
+
elif backend == "flex":
|
| 501 |
+
from torch.nn.attention.flex_attention import flex_attention, create_block_mask
|
| 502 |
+
mask = create_block_mask(lambda b, h, i, j: i >= j, B=None, H=None,
|
| 503 |
+
Q_LEN=seq_len, KV_LEN=seq_len, device=x.device)
|
| 504 |
+
out = flex_attention(qh, kh, vh, block_mask=mask, scale=self.scale,
|
| 505 |
+
enable_gqa=True)
|
| 506 |
+
elif backend == "math":
|
| 507 |
+
kh = kh.repeat_interleave(cfg.groups, dim=1)
|
| 508 |
+
vh = vh.repeat_interleave(cfg.groups, dim=1)
|
| 509 |
+
scores = (qh.float() @ kh.float().transpose(-2, -1)) * self.scale
|
| 510 |
+
causal = torch.ones(seq_len, seq_len, dtype=torch.bool,
|
| 511 |
+
device=x.device).tril()
|
| 512 |
+
out = (scores.masked_fill(~causal, float("-inf")).softmax(-1)
|
| 513 |
+
@ vh.float()).to(x.dtype)
|
| 514 |
+
else:
|
| 515 |
+
raise ValueError(f"unknown backend {backend!r}")
|
| 516 |
+
out = out.transpose(1, 2)
|
| 517 |
+
|
| 518 |
+
return self.wo(out.reshape(batch, seq_len, cfg.n_heads * cfg.head_dim))
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def reference(q, k, v, causal, groups):
|
| 522 |
+
dim = q.size(-1)
|
| 523 |
+
k = k.repeat_interleave(groups, dim=2)
|
| 524 |
+
v = v.repeat_interleave(groups, dim=2)
|
| 525 |
+
scores = torch.einsum("bqhd,bkhd->bhqk", q.float(), k.float()) / math.sqrt(dim)
|
| 526 |
+
if causal:
|
| 527 |
+
seq_len = q.size(1)
|
| 528 |
+
mask = torch.ones(seq_len, seq_len, dtype=torch.bool, device=q.device).tril()
|
| 529 |
+
scores = scores.masked_fill(~mask, float("-inf"))
|
| 530 |
+
return torch.einsum("bhqk,bkhd->bqhd", scores.softmax(-1), v.float()).to(q.dtype)
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def rope_reference(x, theta):
|
| 534 |
+
batch, seq_len, heads, dim = x.shape
|
| 535 |
+
half = dim // 2
|
| 536 |
+
freqs = 1.0 / (theta ** (2 * torch.arange(half).double() / dim))
|
| 537 |
+
angles = torch.arange(seq_len).double().unsqueeze(-1) * freqs
|
| 538 |
+
c = angles.cos().float().view(1, seq_len, 1, half)
|
| 539 |
+
s = angles.sin().float().view(1, seq_len, 1, half)
|
| 540 |
+
left, right = x[..., :half], x[..., half:]
|
| 541 |
+
return torch.cat([left * c - right * s, left * s + right * c], dim=-1)
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def check_math():
|
| 545 |
+
passed = True
|
| 546 |
+
|
| 547 |
+
def check(label, condition):
|
| 548 |
+
nonlocal passed
|
| 549 |
+
passed &= bool(condition)
|
| 550 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 551 |
+
|
| 552 |
+
torch.manual_seed(0)
|
| 553 |
+
cfg = AttnConfig(backend="math")
|
| 554 |
+
kb = 1024
|
| 555 |
+
|
| 556 |
+
print("shared memory")
|
| 557 |
+
for stages in (1, 2, 3):
|
| 558 |
+
size = fwd_smem(128, 64, 128, stages)
|
| 559 |
+
print(f" forward 128x64 stages={stages} {size/kb:6.1f} KB")
|
| 560 |
+
print(f" backward 64x32 stages=2 {bwd_smem(64, 32, 128, 2)/kb:6.1f} KB")
|
| 561 |
+
print(f" backward 128x32 stages=2 {bwd_smem(128, 32, 128, 2)/kb:6.1f} KB")
|
| 562 |
+
check("forward default fits with headroom",
|
| 563 |
+
fwd_smem(FWD_TILES["block_M"], FWD_TILES["block_N"], 128,
|
| 564 |
+
FWD_TILES["num_stages"]) <= 0.85 * 128 * kb)
|
| 565 |
+
check("backward default fits with headroom",
|
| 566 |
+
bwd_smem(64, 32, 128, 2) <= 0.85 * 128 * kb)
|
| 567 |
+
check("backward at block_M=128 hits the sm_120 ceiling",
|
| 568 |
+
bwd_smem(128, 32, 128, 2) >= 128 * kb)
|
| 569 |
+
|
| 570 |
+
print("\nconfig spaces")
|
| 571 |
+
fwd, bwd = fwd_configs(), bwd_configs()
|
| 572 |
+
print(f" forward {len(fwd):3d} of 144")
|
| 573 |
+
print(f" backward {len(bwd):3d} of 54")
|
| 574 |
+
check("forward default is tunable", FWD_TILES in fwd)
|
| 575 |
+
check("backward default is tunable", BWD_TILES in bwd)
|
| 576 |
+
check("backward limited to block_M <= 64", not any(c["block_M"] > 64 for c in bwd))
|
| 577 |
+
check("backward all threads=128", all(c["threads"] == 128 for c in bwd))
|
| 578 |
+
|
| 579 |
+
print("\nrope")
|
| 580 |
+
rope = Rotary(128, 64, 10_000.0)
|
| 581 |
+
x = torch.randn(2, 64, 3, 128)
|
| 582 |
+
got, want = apply_rope(x, rope.cos, rope.sin), rope_reference(x, 10_000.0)
|
| 583 |
+
check(f"matches explicit rotation ({(got - want).abs().max():.1e})",
|
| 584 |
+
(got - want).abs().max() < 1e-4)
|
| 585 |
+
check(f"norm preserved ({(got.norm(-1) - x.norm(-1)).abs().max():.1e})",
|
| 586 |
+
(got.norm(-1) - x.norm(-1)).abs().max() < 1e-4)
|
| 587 |
+
single = torch.randn(1, 1, 1, 128).expand(1, 64, 1, 128).contiguous()
|
| 588 |
+
dots = (apply_rope(single, rope.cos, rope.sin)[0, :, 0]
|
| 589 |
+
* apply_rope(single.clone(), rope.cos, rope.sin)[0, :, 0]).sum(-1)
|
| 590 |
+
check(f"relative encoding holds ({(dots - dots[0]).abs().max():.1e})",
|
| 591 |
+
(dots - dots[0]).abs().max() < 1e-3)
|
| 592 |
+
rope.set_theta(500_000.0)
|
| 593 |
+
check("set_theta rebuilds tables", rope.theta == 500_000.0)
|
| 594 |
+
|
| 595 |
+
print("\nrmsnorm")
|
| 596 |
+
norm = RMSNorm(8)
|
| 597 |
+
y = norm(torch.randn(3, 8))
|
| 598 |
+
check(f"unit rms output ({y.pow(2).mean().sqrt():.4f})",
|
| 599 |
+
abs(y.pow(2).mean().sqrt() - 1) < 0.05)
|
| 600 |
+
|
| 601 |
+
print("\nmodule")
|
| 602 |
+
attn = Attention(cfg, Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta))
|
| 603 |
+
n_params = sum(p.numel() for p in attn.parameters())
|
| 604 |
+
check(f"parameters {n_params:,}", n_params == 3_932_416)
|
| 605 |
+
check("no biases", all(m.bias is None for m in (attn.wq, attn.wk, attn.wv, attn.wo)))
|
| 606 |
+
|
| 607 |
+
x = torch.randn(2, 32, cfg.d_model)
|
| 608 |
+
y = attn(x, backend="math")
|
| 609 |
+
check(f"shape {tuple(x.shape)} -> {tuple(y.shape)}", y.shape == x.shape)
|
| 610 |
+
check(f"math and sdpa agree ({(y - attn(x, backend='sdpa')).abs().max():.1e})",
|
| 611 |
+
(y - attn(x, backend="sdpa")).abs().max() < 2e-3)
|
| 612 |
+
|
| 613 |
+
perturbed = x.clone()
|
| 614 |
+
perturbed[:, -1] += 10.0
|
| 615 |
+
drift = (attn(perturbed, backend="math")[:, :-1] - y[:, :-1]).abs().max()
|
| 616 |
+
check(f"causal, no leak from future ({drift:.1e})", drift < 1e-4)
|
| 617 |
+
|
| 618 |
+
x = torch.randn(2, 16, cfg.d_model, requires_grad=True)
|
| 619 |
+
attn(x, backend="math").sum().backward()
|
| 620 |
+
check("finite gradients everywhere",
|
| 621 |
+
torch.isfinite(x.grad).all()
|
| 622 |
+
and all(torch.isfinite(p.grad).all() for p in attn.parameters()))
|
| 623 |
+
|
| 624 |
+
print("\nkernel constants")
|
| 625 |
+
check("(1/dim)**0.5 == 1/sqrt(dim)", abs((1.0 / 128) ** 0.5 - 1 / math.sqrt(128)) < 1e-12)
|
| 626 |
+
check("1.44269504 == log2(e)", abs(1.44269504 - math.log2(math.e)) < 1e-8)
|
| 627 |
+
|
| 628 |
+
print("\n" + ("all math checks passed" if passed else "FAILURES ABOVE"))
|
| 629 |
+
print("still needs a GPU: kernel compilation, dQ/dK/dV vs reference, tuning")
|
| 630 |
+
return passed
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
def check_gpu(seq_len=512):
|
| 634 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 635 |
+
print("needs tilelang and cuda")
|
| 636 |
+
return
|
| 637 |
+
major, minor = torch.cuda.get_device_capability()
|
| 638 |
+
print(f"{torch.cuda.get_device_name()} sm_{major}{minor}")
|
| 639 |
+
print(f"shared memory limit in use: {SMEM_LIMIT/1024:.0f} KB")
|
| 640 |
+
|
| 641 |
+
heads, kv_heads, dim, groups = 10, 2, 128, 5
|
| 642 |
+
torch.manual_seed(0)
|
| 643 |
+
q = torch.randn(2, seq_len, heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 644 |
+
k = torch.randn(2, seq_len, kv_heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 645 |
+
v = torch.randn(2, seq_len, kv_heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 646 |
+
grad = torch.randn(2, seq_len, heads, dim, device="cuda", dtype=DTYPE)
|
| 647 |
+
|
| 648 |
+
for causal in (False, True):
|
| 649 |
+
out = flash_attention(q, k, v, causal, groups)
|
| 650 |
+
out.backward(grad, retain_graph=True)
|
| 651 |
+
mine = [t.grad.clone() for t in (q, k, v)]
|
| 652 |
+
q.grad = k.grad = v.grad = None
|
| 653 |
+
|
| 654 |
+
want = reference(q, k, v, causal, groups)
|
| 655 |
+
want.backward(grad, retain_graph=True)
|
| 656 |
+
theirs = [t.grad.clone() for t in (q, k, v)]
|
| 657 |
+
q.grad = k.grad = v.grad = None
|
| 658 |
+
|
| 659 |
+
print(f"\ncausal={causal}")
|
| 660 |
+
# Relative, not absolute. Causal dK/dV accumulate over every query that
|
| 661 |
+
# can see the key, so they are simply bigger numbers, and one bf16 ulp
|
| 662 |
+
# at that magnitude is 6e-2 -- an absolute threshold measures the size
|
| 663 |
+
# of the gradient rather than the correctness of the kernel.
|
| 664 |
+
for name, a, b in zip(("out", "dQ", "dK", "dV"),
|
| 665 |
+
[out] + mine, [want] + theirs):
|
| 666 |
+
err = (a.float() - b.float()).abs().max().item()
|
| 667 |
+
rel = err / max(b.float().abs().max().item(), 1e-12)
|
| 668 |
+
print(f" {name:4s} abs {err:.3e} rel {rel:.3e} "
|
| 669 |
+
f"{'ok' if rel < 2e-2 else 'FAIL'}")
|
| 670 |
+
|
| 671 |
+
runs = []
|
| 672 |
+
for _ in range(2):
|
| 673 |
+
flash_attention(q, k, v, True, groups).backward(grad, retain_graph=True)
|
| 674 |
+
runs.append(q.grad.clone())
|
| 675 |
+
q.grad = None
|
| 676 |
+
spread = (runs[0].float() - runs[1].float()).abs().max().item()
|
| 677 |
+
print(f"\nrepeat backward differs by {spread:.3e}"
|
| 678 |
+
f" ({'deterministic' if spread == 0 else 'fp32 atomics, expected'})")
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def benchmark(batch=8, seq_len=2048):
|
| 682 |
+
if not torch.cuda.is_available():
|
| 683 |
+
print("needs cuda")
|
| 684 |
+
return
|
| 685 |
+
heads, kv_heads, dim, groups = 10, 2, 128, 5
|
| 686 |
+
q = torch.randn(batch, seq_len, heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 687 |
+
k = torch.randn(batch, seq_len, kv_heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 688 |
+
v = torch.randn(batch, seq_len, kv_heads, dim, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 689 |
+
grad = torch.randn(batch, seq_len, heads, dim, device="cuda", dtype=DTYPE)
|
| 690 |
+
flops = 2 * 2.0 * batch * heads * seq_len * seq_len * dim * 0.5 * 2.5
|
| 691 |
+
|
| 692 |
+
def time_it(fn, iters=30):
|
| 693 |
+
for _ in range(10):
|
| 694 |
+
fn()
|
| 695 |
+
torch.cuda.synchronize()
|
| 696 |
+
start, end = torch.cuda.Event(True), torch.cuda.Event(True)
|
| 697 |
+
start.record()
|
| 698 |
+
for _ in range(iters):
|
| 699 |
+
fn()
|
| 700 |
+
end.record()
|
| 701 |
+
torch.cuda.synchronize()
|
| 702 |
+
return start.elapsed_time(end) / iters
|
| 703 |
+
|
| 704 |
+
def clear():
|
| 705 |
+
q.grad = k.grad = v.grad = None
|
| 706 |
+
|
| 707 |
+
qh, kh, vh, gh = (t.transpose(1, 2) for t in (q, k, v, grad))
|
| 708 |
+
results = {}
|
| 709 |
+
|
| 710 |
+
def run_sdpa():
|
| 711 |
+
F.scaled_dot_product_attention(qh, kh, vh, is_causal=True, enable_gqa=True
|
| 712 |
+
).backward(gh, retain_graph=True)
|
| 713 |
+
clear()
|
| 714 |
+
results["sdpa"] = time_it(run_sdpa)
|
| 715 |
+
|
| 716 |
+
try:
|
| 717 |
+
from torch.nn.attention.flex_attention import flex_attention, create_block_mask
|
| 718 |
+
compiled = torch.compile(flex_attention)
|
| 719 |
+
mask = create_block_mask(lambda b, h, i, j: i >= j, B=None, H=None,
|
| 720 |
+
Q_LEN=seq_len, KV_LEN=seq_len, device="cuda")
|
| 721 |
+
|
| 722 |
+
def run_flex():
|
| 723 |
+
compiled(qh, kh, vh, block_mask=mask, enable_gqa=True
|
| 724 |
+
).backward(gh, retain_graph=True)
|
| 725 |
+
clear()
|
| 726 |
+
results["flex"] = time_it(run_flex)
|
| 727 |
+
except Exception as exc:
|
| 728 |
+
print(f"flex unavailable: {exc}")
|
| 729 |
+
|
| 730 |
+
if HAVE_TILELANG:
|
| 731 |
+
try:
|
| 732 |
+
def run_tilelang():
|
| 733 |
+
flash_attention(q, k, v, True, groups).backward(grad, retain_graph=True)
|
| 734 |
+
clear()
|
| 735 |
+
results["tilelang"] = time_it(run_tilelang)
|
| 736 |
+
except Exception as exc:
|
| 737 |
+
print(f"tilelang failed: {type(exc).__name__}: {exc}")
|
| 738 |
+
|
| 739 |
+
print()
|
| 740 |
+
for name, ms in sorted(results.items(), key=lambda kv: kv[1]):
|
| 741 |
+
print(f" {name:10s} {ms:8.3f} ms {flops/ms*1e-9:7.1f} TFLOP/s")
|
| 742 |
+
|
| 743 |
+
if "tilelang" in results and len(results) > 1:
|
| 744 |
+
best_other = min(v for n, v in results.items() if n != "tilelang")
|
| 745 |
+
speedup = best_other / results["tilelang"]
|
| 746 |
+
overall = 1 / (0.863 + 0.137 / speedup)
|
| 747 |
+
print(f"\n attention {speedup:.2f}x, end to end {overall:.3f}x,"
|
| 748 |
+
f" {23 * (1 - 1 / overall):+.1f} days on a 23 day run")
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
def tune(batch=8, seq_len=2048):
|
| 752 |
+
print(f"forward candidates {len(fwd_configs())}")
|
| 753 |
+
print(f"backward candidates {len(bwd_configs())}")
|
| 754 |
+
print(f"shared memory limit {SMEM_LIMIT/1024:.0f} KB")
|
| 755 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 756 |
+
for name, space in (("forward", fwd_configs()), ("backward", bwd_configs())):
|
| 757 |
+
print(f"\n{name}")
|
| 758 |
+
for c in space:
|
| 759 |
+
print(f" {c}")
|
| 760 |
+
return
|
| 761 |
+
cfg = AttnConfig()
|
| 762 |
+
warmup(batch, seq_len, cfg)
|
| 763 |
+
shape = (batch, cfg.n_heads, seq_len, cfg.head_dim, True, cfg.groups)
|
| 764 |
+
for kind in ("fwd", "bwd"):
|
| 765 |
+
kernel = get_kernel(kind, *shape)
|
| 766 |
+
print(f"\n{kind}: {getattr(kernel, 'config', None)}")
|
| 767 |
+
latency = getattr(kernel, "latency", None)
|
| 768 |
+
if latency is not None:
|
| 769 |
+
print(f" {latency:.3f} ms")
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
if __name__ == "__main__":
|
| 773 |
+
parser = argparse.ArgumentParser()
|
| 774 |
+
parser.add_argument("--check", action="store_true")
|
| 775 |
+
parser.add_argument("--gpu", action="store_true")
|
| 776 |
+
parser.add_argument("--bench", action="store_true")
|
| 777 |
+
parser.add_argument("--tune", action="store_true")
|
| 778 |
+
args = parser.parse_args()
|
| 779 |
+
if not any(vars(args).values()):
|
| 780 |
+
args.check = True
|
| 781 |
+
if args.check:
|
| 782 |
+
check_math()
|
| 783 |
+
if args.gpu:
|
| 784 |
+
check_gpu()
|
| 785 |
+
if args.bench:
|
| 786 |
+
benchmark()
|
| 787 |
+
if args.tune:
|
| 788 |
+
tune()
|
code/model/attnres.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
|
| 8 |
+
from mimans.model.norm import RMSNorm
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class ResidualState:
|
| 13 |
+
running: torch.Tensor
|
| 14 |
+
snapshots: list = field(default_factory=list)
|
| 15 |
+
steps: int = 0
|
| 16 |
+
|
| 17 |
+
def entries(self):
|
| 18 |
+
return self.snapshots + [self.running]
|
| 19 |
+
|
| 20 |
+
def advance(self, delta, block_sublayers):
|
| 21 |
+
self.running = self.running + delta
|
| 22 |
+
self.steps += 1
|
| 23 |
+
if self.steps % block_sublayers == 0:
|
| 24 |
+
self.snapshots.append(self.running)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class AttnResGate(nn.Module):
|
| 28 |
+
"""One gate per sublayer. Holds a query vector, a key norm, and a bias."""
|
| 29 |
+
|
| 30 |
+
def __init__(self, d_model: int, eps: float = 1e-5):
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.query = nn.Parameter(torch.zeros(d_model))
|
| 33 |
+
self.key_norm = RMSNorm(d_model, eps)
|
| 34 |
+
self.recency_bias = nn.Parameter(torch.zeros(()))
|
| 35 |
+
|
| 36 |
+
def forward(self, entries):
|
| 37 |
+
stack = torch.stack(entries, dim=0)
|
| 38 |
+
keys = self.key_norm(stack)
|
| 39 |
+
logits = torch.einsum("d,nbtd->nbt", self.query.to(keys.dtype), keys)
|
| 40 |
+
last = torch.zeros(stack.shape[0], 1, 1,
|
| 41 |
+
device=logits.device, dtype=logits.dtype)
|
| 42 |
+
last[-1] = 1.0
|
| 43 |
+
logits = logits + last * self.recency_bias.to(logits.dtype)
|
| 44 |
+
weights = logits.softmax(dim=0)
|
| 45 |
+
return torch.einsum("nbt,nbtd->btd", weights, stack)
|
| 46 |
+
|
| 47 |
+
def weights(self, entries):
|
| 48 |
+
stack = torch.stack(entries, dim=0)
|
| 49 |
+
logits = torch.einsum("d,nbtd->nbt", self.query.to(stack.dtype),
|
| 50 |
+
self.key_norm(stack))
|
| 51 |
+
last = torch.zeros(stack.shape[0], 1, 1,
|
| 52 |
+
device=logits.device, dtype=logits.dtype)
|
| 53 |
+
last[-1] = 1.0
|
| 54 |
+
return (logits + last * self.recency_bias.to(logits.dtype)).softmax(dim=0)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def gate_parameters(n_layers, d_model, sublayers_per_layer=2):
|
| 58 |
+
return n_layers * sublayers_per_layer * (2 * d_model + 1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def check():
|
| 62 |
+
torch.manual_seed(0)
|
| 63 |
+
passed = True
|
| 64 |
+
|
| 65 |
+
def check_one(label, condition):
|
| 66 |
+
nonlocal passed
|
| 67 |
+
passed &= bool(condition)
|
| 68 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 69 |
+
|
| 70 |
+
d_model, layers, per_block = 1280, 27, 3
|
| 71 |
+
batch, seq = 2, 16
|
| 72 |
+
gate = AttnResGate(d_model)
|
| 73 |
+
|
| 74 |
+
print("shape and cost")
|
| 75 |
+
state = ResidualState(running=torch.randn(batch, seq, d_model))
|
| 76 |
+
for _ in range(4):
|
| 77 |
+
state.snapshots.append(torch.randn(batch, seq, d_model))
|
| 78 |
+
out = gate(state.entries())
|
| 79 |
+
check_one(f"output {tuple(out.shape)}", out.shape == (batch, seq, d_model))
|
| 80 |
+
n = sum(p.numel() for p in gate.parameters())
|
| 81 |
+
check_one(f"gate parameters {n} == 2*d_model + 1", n == 2 * d_model + 1)
|
| 82 |
+
total = gate_parameters(layers, d_model)
|
| 83 |
+
print(f" {layers} layers x 2 sublayers -> {total:,} parameters")
|
| 84 |
+
check_one("under 0.03 percent of a 514M model", total / 514_345_526 < 0.0003)
|
| 85 |
+
|
| 86 |
+
print("\nzero init behaviour")
|
| 87 |
+
entries = state.entries()
|
| 88 |
+
weights = gate.weights(entries)
|
| 89 |
+
check_one(f"uniform at init ({weights.std():.2e})", weights.std() < 1e-6)
|
| 90 |
+
check_one(f"weights sum to one ({(weights.sum(0) - 1).abs().max():.2e})",
|
| 91 |
+
(weights.sum(0) - 1).abs().max() < 1e-5)
|
| 92 |
+
mean = torch.stack(entries, 0).mean(0)
|
| 93 |
+
check_one(f"uniform gives the MEAN not the SUM ({(out - mean).abs().max():.2e})",
|
| 94 |
+
(out - mean).abs().max() < 1e-4)
|
| 95 |
+
running_sum = torch.stack(entries, 0).sum(0)
|
| 96 |
+
check_one("and the mean is not the sum, so init is not a no-op",
|
| 97 |
+
(out - running_sum).abs().max() > 1e-2)
|
| 98 |
+
|
| 99 |
+
print("\nrecency bias unit test")
|
| 100 |
+
with torch.no_grad():
|
| 101 |
+
gate.recency_bias.fill_(1e4)
|
| 102 |
+
collapsed = gate(entries)
|
| 103 |
+
check_one(f"softmax collapses onto the running state "
|
| 104 |
+
f"({(collapsed - state.running).abs().max():.2e})",
|
| 105 |
+
(collapsed - state.running).abs().max() < 1e-6)
|
| 106 |
+
check_one("this reproduces a plain PreNorm residual exactly",
|
| 107 |
+
torch.equal(collapsed, state.running))
|
| 108 |
+
with torch.no_grad():
|
| 109 |
+
gate.recency_bias.zero_()
|
| 110 |
+
|
| 111 |
+
print("\nblock bookkeeping")
|
| 112 |
+
state = ResidualState(running=torch.zeros(batch, seq, d_model))
|
| 113 |
+
for _ in range(layers * 2):
|
| 114 |
+
state.advance(torch.ones(batch, seq, d_model) * 0.01, per_block * 2)
|
| 115 |
+
expected_blocks = (layers * 2) // (per_block * 2)
|
| 116 |
+
check_one(f"{layers} layers, {per_block} per block -> {len(state.snapshots)} snapshots",
|
| 117 |
+
len(state.snapshots) == expected_blocks == 9)
|
| 118 |
+
check_one("running state is the cumulative sum",
|
| 119 |
+
torch.allclose(state.running,
|
| 120 |
+
torch.full((batch, seq, d_model), 0.01 * layers * 2)))
|
| 121 |
+
print(f" memory for snapshots at 8x2048 bf16: "
|
| 122 |
+
f"{9 * 8 * 2048 * d_model * 2 / 1e6:.0f} MB")
|
| 123 |
+
|
| 124 |
+
print("\ngradients")
|
| 125 |
+
gate = AttnResGate(d_model)
|
| 126 |
+
entries = [torch.randn(batch, seq, d_model, requires_grad=True) for _ in range(3)]
|
| 127 |
+
gate(entries).sum().backward()
|
| 128 |
+
check_one("query receives gradient", gate.query.grad is not None
|
| 129 |
+
and torch.isfinite(gate.query.grad).all())
|
| 130 |
+
check_one("recency bias receives gradient",
|
| 131 |
+
gate.recency_bias.grad is not None
|
| 132 |
+
and torch.isfinite(gate.recency_bias.grad).all())
|
| 133 |
+
check_one("all entries receive gradient",
|
| 134 |
+
all(e.grad is not None and torch.isfinite(e.grad).all() for e in entries))
|
| 135 |
+
|
| 136 |
+
print("\nkey normalisation")
|
| 137 |
+
gate = AttnResGate(d_model)
|
| 138 |
+
with torch.no_grad():
|
| 139 |
+
gate.query.normal_(std=0.05)
|
| 140 |
+
small = [torch.randn(batch, seq, d_model) for _ in range(3)]
|
| 141 |
+
scaled = [small[0] * 1000, small[1], small[2]]
|
| 142 |
+
w_small = gate.weights(small)
|
| 143 |
+
w_scaled = gate.weights(scaled)
|
| 144 |
+
check_one("rms normed keys stop a large entry dominating by magnitude alone",
|
| 145 |
+
(w_small[0] - w_scaled[0]).abs().max() < 0.05)
|
| 146 |
+
|
| 147 |
+
print("\n" + ("attnres ok" if passed else "FAILURES ABOVE"))
|
| 148 |
+
return passed
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
if __name__ == "__main__":
|
| 152 |
+
check()
|
code/model/block.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
|
| 7 |
+
from mimans.model.norm import RMSNorm
|
| 8 |
+
from mimans.model.attnres import AttnResGate, ResidualState
|
| 9 |
+
from mimans.model.attention_tilelang import Attention, AttnConfig, Rotary
|
| 10 |
+
from mimans.model.swiglu_tilelang import SwiGLU, FFNConfig
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class BlockConfig:
|
| 15 |
+
d_model: int = 1280
|
| 16 |
+
n_heads: int = 10
|
| 17 |
+
n_kv_heads: int = 2
|
| 18 |
+
head_dim: int = 128
|
| 19 |
+
ffn_hidden: int = 3328
|
| 20 |
+
n_layers: int = 27
|
| 21 |
+
layers_per_block: int = 3
|
| 22 |
+
norm_eps: float = 1e-5
|
| 23 |
+
rope_theta: float = 10_000.0
|
| 24 |
+
max_seq_len: int = 8192
|
| 25 |
+
use_attnres: bool = True
|
| 26 |
+
attn_backend: str = "tilelang"
|
| 27 |
+
ffn_backend: str = "tilelang"
|
| 28 |
+
|
| 29 |
+
def __post_init__(self):
|
| 30 |
+
assert self.n_layers % self.layers_per_block == 0, \
|
| 31 |
+
"layers must divide evenly into blocks"
|
| 32 |
+
|
| 33 |
+
@property
|
| 34 |
+
def sublayers_per_block(self):
|
| 35 |
+
return self.layers_per_block * 2
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def n_blocks(self):
|
| 39 |
+
return self.n_layers // self.layers_per_block
|
| 40 |
+
|
| 41 |
+
@property
|
| 42 |
+
def attn_config(self):
|
| 43 |
+
return AttnConfig(d_model=self.d_model, n_heads=self.n_heads,
|
| 44 |
+
n_kv_heads=self.n_kv_heads, head_dim=self.head_dim,
|
| 45 |
+
rope_theta=self.rope_theta,
|
| 46 |
+
max_seq_len=self.max_seq_len,
|
| 47 |
+
norm_eps=self.norm_eps, backend=self.attn_backend)
|
| 48 |
+
|
| 49 |
+
@property
|
| 50 |
+
def ffn_config(self):
|
| 51 |
+
return FFNConfig(d_model=self.d_model, ffn_hidden=self.ffn_hidden,
|
| 52 |
+
backend=self.ffn_backend)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class TransformerBlock(nn.Module):
|
| 56 |
+
def __init__(self, cfg: BlockConfig, rope: Rotary):
|
| 57 |
+
super().__init__()
|
| 58 |
+
self.cfg = cfg
|
| 59 |
+
self.attn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 60 |
+
self.attn = Attention(cfg.attn_config, rope)
|
| 61 |
+
self.ffn_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 62 |
+
self.ffn = SwiGLU(cfg.ffn_config)
|
| 63 |
+
if cfg.use_attnres:
|
| 64 |
+
self.attn_gate = AttnResGate(cfg.d_model, cfg.norm_eps)
|
| 65 |
+
self.ffn_gate = AttnResGate(cfg.d_model, cfg.norm_eps)
|
| 66 |
+
else:
|
| 67 |
+
self.attn_gate = None
|
| 68 |
+
self.ffn_gate = None
|
| 69 |
+
self.zero_init_projections()
|
| 70 |
+
|
| 71 |
+
def zero_init_projections(self):
|
| 72 |
+
with torch.no_grad():
|
| 73 |
+
self.attn.wo.weight.zero_()
|
| 74 |
+
self.ffn.w_down.zero_()
|
| 75 |
+
|
| 76 |
+
def combine(self, gate, state):
|
| 77 |
+
if gate is None:
|
| 78 |
+
return state.running
|
| 79 |
+
return gate(state.entries())
|
| 80 |
+
|
| 81 |
+
def forward(self, state: ResidualState, backend=None):
|
| 82 |
+
span = self.cfg.sublayers_per_block
|
| 83 |
+
|
| 84 |
+
h = self.combine(self.attn_gate, state)
|
| 85 |
+
delta = self.attn(self.attn_norm(h), backend=backend or self.cfg.attn_backend)
|
| 86 |
+
state.advance(delta, span)
|
| 87 |
+
|
| 88 |
+
h = self.combine(self.ffn_gate, state)
|
| 89 |
+
delta = self.ffn(self.ffn_norm(h), backend=backend or self.cfg.ffn_backend)
|
| 90 |
+
state.advance(delta, span)
|
| 91 |
+
|
| 92 |
+
return state
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def block_parameters(cfg: BlockConfig):
|
| 96 |
+
d, h, kv, hd = cfg.d_model, cfg.n_heads, cfg.n_kv_heads, cfg.head_dim
|
| 97 |
+
attn = d * h * hd + 2 * d * kv * hd + h * hd * d + 2 * hd
|
| 98 |
+
ffn = 3 * d * cfg.ffn_hidden
|
| 99 |
+
norms = 2 * d
|
| 100 |
+
gates = 2 * (2 * d + 1) if cfg.use_attnres else 0
|
| 101 |
+
return attn + ffn + norms + gates
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def check():
|
| 105 |
+
torch.manual_seed(0)
|
| 106 |
+
passed = True
|
| 107 |
+
|
| 108 |
+
def check_one(label, condition):
|
| 109 |
+
nonlocal passed
|
| 110 |
+
passed &= bool(condition)
|
| 111 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 112 |
+
|
| 113 |
+
cfg = BlockConfig(attn_backend="math", ffn_backend="math")
|
| 114 |
+
rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
|
| 115 |
+
block = TransformerBlock(cfg, rope)
|
| 116 |
+
batch, seq = 2, 16
|
| 117 |
+
|
| 118 |
+
print("structure")
|
| 119 |
+
print(f" {cfg.n_layers} layers, {cfg.layers_per_block} per block "
|
| 120 |
+
f"-> {cfg.n_blocks} blocks, {cfg.sublayers_per_block} sublayers each")
|
| 121 |
+
check_one("27 layers give 9 blocks", cfg.n_blocks == 9)
|
| 122 |
+
check_one("6 sublayers per block", cfg.sublayers_per_block == 6)
|
| 123 |
+
|
| 124 |
+
n = sum(p.numel() for p in block.parameters())
|
| 125 |
+
expect = block_parameters(cfg)
|
| 126 |
+
print(f" block parameters {n:,}")
|
| 127 |
+
check_one(f"matches the arithmetic {expect:,}", n == expect)
|
| 128 |
+
check_one("matches the design report 16,719,618", n == 16_719_618)
|
| 129 |
+
print(f" x{cfg.n_layers} layers = {n * cfg.n_layers / 1e6:.1f}M")
|
| 130 |
+
|
| 131 |
+
print("\nzero init means identity at step zero")
|
| 132 |
+
check_one("attention output projection is zero", (block.attn.wo.weight == 0).all())
|
| 133 |
+
check_one("ffn down projection is zero", (block.ffn.w_down == 0).all())
|
| 134 |
+
x = torch.randn(batch, seq, cfg.d_model)
|
| 135 |
+
state = ResidualState(running=x.clone())
|
| 136 |
+
block(state)
|
| 137 |
+
check_one(f"running state unchanged ({(state.running - x).abs().max():.2e})",
|
| 138 |
+
(state.running - x).abs().max() < 1e-6)
|
| 139 |
+
combined = block.combine(block.attn_gate, state)
|
| 140 |
+
check_one(f"gate over identical entries returns them "
|
| 141 |
+
f"({(combined - x).abs().max():.2e})",
|
| 142 |
+
(combined - x).abs().max() < 1e-5)
|
| 143 |
+
|
| 144 |
+
print("\nsublayer bookkeeping")
|
| 145 |
+
state = ResidualState(running=torch.randn(batch, seq, cfg.d_model))
|
| 146 |
+
block(state)
|
| 147 |
+
check_one(f"one block advances 2 sublayers ({state.steps})", state.steps == 2)
|
| 148 |
+
state = ResidualState(running=torch.randn(batch, seq, cfg.d_model))
|
| 149 |
+
for _ in range(cfg.layers_per_block):
|
| 150 |
+
block(state)
|
| 151 |
+
check_one(f"{cfg.layers_per_block} layers close one block "
|
| 152 |
+
f"({len(state.snapshots)} snapshot)", len(state.snapshots) == 1)
|
| 153 |
+
state = ResidualState(running=torch.randn(batch, seq, cfg.d_model))
|
| 154 |
+
for _ in range(cfg.n_layers):
|
| 155 |
+
block(state)
|
| 156 |
+
check_one(f"{cfg.n_layers} layers give {len(state.snapshots)} snapshots",
|
| 157 |
+
len(state.snapshots) == cfg.n_blocks)
|
| 158 |
+
check_one(f"entries at the end {len(state.entries())} = blocks + running",
|
| 159 |
+
len(state.entries()) == cfg.n_blocks + 1)
|
| 160 |
+
|
| 161 |
+
print("\nattnres reduces to plain prenorm under a large recency bias")
|
| 162 |
+
torch.manual_seed(1)
|
| 163 |
+
gated = TransformerBlock(BlockConfig(use_attnres=True, attn_backend="math",
|
| 164 |
+
ffn_backend="math"), rope)
|
| 165 |
+
with torch.no_grad():
|
| 166 |
+
for p in gated.parameters():
|
| 167 |
+
if p.dim() >= 2:
|
| 168 |
+
p.normal_(std=0.02)
|
| 169 |
+
gated.attn_gate.recency_bias.fill_(1e4)
|
| 170 |
+
gated.ffn_gate.recency_bias.fill_(1e4)
|
| 171 |
+
|
| 172 |
+
plain = TransformerBlock(BlockConfig(use_attnres=False, attn_backend="math",
|
| 173 |
+
ffn_backend="math"), rope)
|
| 174 |
+
with torch.no_grad():
|
| 175 |
+
plain.attn_norm.weight.copy_(gated.attn_norm.weight)
|
| 176 |
+
plain.ffn_norm.weight.copy_(gated.ffn_norm.weight)
|
| 177 |
+
plain.attn.load_state_dict(gated.attn.state_dict())
|
| 178 |
+
plain.ffn.load_state_dict(gated.ffn.state_dict())
|
| 179 |
+
|
| 180 |
+
x = torch.randn(batch, seq, cfg.d_model)
|
| 181 |
+
a = ResidualState(running=x.clone())
|
| 182 |
+
b = ResidualState(running=x.clone())
|
| 183 |
+
gated(a)
|
| 184 |
+
plain(b)
|
| 185 |
+
diff = (a.running - b.running).abs().max().item()
|
| 186 |
+
check_one(f"gated and plain agree ({diff:.2e})", diff < 1e-5)
|
| 187 |
+
check_one("so the ablation switch is a true control arm", diff < 1e-5)
|
| 188 |
+
|
| 189 |
+
print("\ncausality")
|
| 190 |
+
block = TransformerBlock(BlockConfig(attn_backend="math", ffn_backend="math"), rope)
|
| 191 |
+
with torch.no_grad():
|
| 192 |
+
for p in block.parameters():
|
| 193 |
+
if p.dim() >= 2:
|
| 194 |
+
p.normal_(std=0.02)
|
| 195 |
+
x = torch.randn(batch, seq, cfg.d_model)
|
| 196 |
+
s1 = ResidualState(running=x.clone())
|
| 197 |
+
block(s1)
|
| 198 |
+
perturbed = x.clone()
|
| 199 |
+
perturbed[:, -1] += 10.0
|
| 200 |
+
s2 = ResidualState(running=perturbed)
|
| 201 |
+
block(s2)
|
| 202 |
+
drift = (s1.running[:, :-1] - s2.running[:, :-1]).abs().max().item()
|
| 203 |
+
check_one(f"future token does not reach the past ({drift:.2e})", drift < 1e-4)
|
| 204 |
+
|
| 205 |
+
print("\ngradients")
|
| 206 |
+
x = torch.randn(batch, 8, cfg.d_model, requires_grad=True)
|
| 207 |
+
state = ResidualState(running=x)
|
| 208 |
+
block(state)
|
| 209 |
+
state.running.sum().backward()
|
| 210 |
+
check_one("input receives gradient",
|
| 211 |
+
x.grad is not None and torch.isfinite(x.grad).all())
|
| 212 |
+
missing = [n for n, p in block.named_parameters()
|
| 213 |
+
if p.grad is None or not torch.isfinite(p.grad).all()]
|
| 214 |
+
check_one(f"every parameter receives a finite gradient"
|
| 215 |
+
f"{'' if not missing else ' -> ' + str(missing[:3])}", not missing)
|
| 216 |
+
|
| 217 |
+
print("\n" + ("block ok" if passed else "FAILURES ABOVE"))
|
| 218 |
+
return passed
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
if __name__ == "__main__":
|
| 222 |
+
check()
|
code/model/init_study.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compare weight initialisation schemes on the real model and real data.
|
| 2 |
+
|
| 3 |
+
The question this answers: the stack zero-inits its two output projections, so
|
| 4 |
+
at step 0 only those projections and the embedding get any gradient at all --
|
| 5 |
+
every wq, wk, wv, w_gate, w_up, norm and gate sees exactly zero. That is the
|
| 6 |
+
ReZero/Fixup behaviour, and it is defensible, but GPT-2 and most production
|
| 7 |
+
LLMs instead scale those projections by 1/sqrt(2*n_layers) so that everything
|
| 8 |
+
gets signal from the first step.
|
| 9 |
+
|
| 10 |
+
Rather than argue it, train each scheme on identical data with an identical
|
| 11 |
+
seed and read the loss.
|
| 12 |
+
|
| 13 |
+
python -m mimans.model.init_study --steps 300
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import json
|
| 19 |
+
import math
|
| 20 |
+
import time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
from mimans import paths
|
| 27 |
+
from mimans.data import Loader, Shard, Source
|
| 28 |
+
from mimans.model.model import CodeModel, ModelConfig
|
| 29 |
+
from mimans.training.optim import HybridOptimizer, OptimConfig, ScheduleConfig
|
| 30 |
+
|
| 31 |
+
SCHEMES = ("zero", "gpt2", "fanin_zero", "fanin_gpt2", "orthogonal")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def fan_in_of(name: str, p: torch.Tensor):
|
| 35 |
+
"""nn.Linear stores (out, in); the ffn holds raw parameters used as x @ W,
|
| 36 |
+
so their fan in is the first dimension instead."""
|
| 37 |
+
if name.endswith(("w_gate", "w_up", "w_down")):
|
| 38 |
+
return p.shape[0]
|
| 39 |
+
return p.shape[1]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def is_output_projection(name: str):
|
| 43 |
+
return name.endswith(("wo.weight", "w_down"))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@torch.no_grad()
|
| 47 |
+
def apply_scheme(model: CodeModel, scheme: str, std: float = 0.02):
|
| 48 |
+
"""Re-initialise every block matrix under one scheme. Embedding, norms and
|
| 49 |
+
gates are left exactly as the model already sets them, so the comparison
|
| 50 |
+
isolates the block matrices."""
|
| 51 |
+
n_layers = model.cfg.n_layers
|
| 52 |
+
residual_scale = 1.0 / math.sqrt(2 * n_layers)
|
| 53 |
+
|
| 54 |
+
for block in model.blocks:
|
| 55 |
+
for name, p in block.named_parameters():
|
| 56 |
+
if p.dim() < 2:
|
| 57 |
+
continue
|
| 58 |
+
fan_in = fan_in_of(name, p)
|
| 59 |
+
base = std if scheme in ("zero", "gpt2") else fan_in ** -0.5
|
| 60 |
+
|
| 61 |
+
if is_output_projection(name):
|
| 62 |
+
if scheme in ("zero", "fanin_zero", "orthogonal"):
|
| 63 |
+
p.zero_()
|
| 64 |
+
else: # gpt2, fanin_gpt2
|
| 65 |
+
p.normal_(0, base * residual_scale)
|
| 66 |
+
elif scheme == "orthogonal":
|
| 67 |
+
# Orthogonal rows/columns give every singular value the same
|
| 68 |
+
# size, which is the condition Muon's update already assumes.
|
| 69 |
+
flat = p.reshape(p.shape[0], -1)
|
| 70 |
+
torch.nn.init.orthogonal_(flat, gain=1.0)
|
| 71 |
+
p.copy_(flat.reshape(p.shape))
|
| 72 |
+
else:
|
| 73 |
+
p.normal_(0, base)
|
| 74 |
+
return model
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def grad_health(model: CodeModel, x, y):
|
| 78 |
+
"""How many block parameters get a nonzero gradient on the very first
|
| 79 |
+
backward pass, before any weight has moved."""
|
| 80 |
+
model.zero_grad(set_to_none=True)
|
| 81 |
+
model(x, y).backward()
|
| 82 |
+
live = dead = 0
|
| 83 |
+
for block in model.blocks:
|
| 84 |
+
for _, p in block.named_parameters():
|
| 85 |
+
if p.grad is None or p.grad.norm().item() == 0.0:
|
| 86 |
+
dead += 1
|
| 87 |
+
else:
|
| 88 |
+
live += 1
|
| 89 |
+
model.zero_grad(set_to_none=True)
|
| 90 |
+
return live, dead
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def open_shards(seq_len):
|
| 94 |
+
sources = {}
|
| 95 |
+
for name in ("code", "text", "math"):
|
| 96 |
+
folder = paths.SHARDS / name
|
| 97 |
+
shards = sorted(folder.glob("*.bin")) if folder.is_dir() else []
|
| 98 |
+
if shards:
|
| 99 |
+
sources[name] = Source(name, [Shard(p) for p in shards])
|
| 100 |
+
if not sources:
|
| 101 |
+
raise SystemExit(f"no shards in {paths.SHARDS}; run prepare_data first")
|
| 102 |
+
return sources
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def run_scheme(scheme, args, sources, weights):
|
| 106 |
+
torch.manual_seed(args.seed)
|
| 107 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 108 |
+
|
| 109 |
+
cfg = ModelConfig(n_layers=args.layers)
|
| 110 |
+
model = CodeModel(cfg)
|
| 111 |
+
apply_scheme(model, scheme)
|
| 112 |
+
model = model.cuda()
|
| 113 |
+
|
| 114 |
+
# Warmup is shortened in proportion to the run: with the real 2,000 step
|
| 115 |
+
# warmup a 300 step probe would spend the whole time at a near zero
|
| 116 |
+
# learning rate and every scheme would look identical.
|
| 117 |
+
schedule = ScheduleConfig(total_steps=args.steps,
|
| 118 |
+
warmup_steps=max(1, args.steps // 10),
|
| 119 |
+
decay_fraction=0.1)
|
| 120 |
+
opt = HybridOptimizer(model, OptimConfig(), schedule)
|
| 121 |
+
|
| 122 |
+
loader = Loader(sources, batch_size=args.micro_batch, seed=args.seed)
|
| 123 |
+
x0, y0 = loader.batch(0, weights)
|
| 124 |
+
live, dead = grad_health(model,
|
| 125 |
+
torch.from_numpy(x0).cuda(),
|
| 126 |
+
torch.from_numpy(y0).cuda())
|
| 127 |
+
|
| 128 |
+
losses, started = [], time.time()
|
| 129 |
+
for step in range(args.steps):
|
| 130 |
+
opt.set_step(step)
|
| 131 |
+
opt.zero_grad()
|
| 132 |
+
x, y = loader.batch(step, weights)
|
| 133 |
+
x = torch.from_numpy(x).cuda()
|
| 134 |
+
y = torch.from_numpy(y).cuda()
|
| 135 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 136 |
+
loss = model(x, y)
|
| 137 |
+
loss.backward()
|
| 138 |
+
opt.step()
|
| 139 |
+
losses.append(loss.item())
|
| 140 |
+
if not math.isfinite(losses[-1]):
|
| 141 |
+
print(f" diverged at step {step}")
|
| 142 |
+
break
|
| 143 |
+
|
| 144 |
+
elapsed = time.time() - started
|
| 145 |
+
tail = losses[-args.tail:] if len(losses) >= args.tail else losses
|
| 146 |
+
result = {
|
| 147 |
+
"scheme": scheme,
|
| 148 |
+
"first_loss": losses[0],
|
| 149 |
+
"final_loss": sum(tail) / len(tail),
|
| 150 |
+
"best_loss": min(losses),
|
| 151 |
+
"live_grads": live,
|
| 152 |
+
"dead_grads": dead,
|
| 153 |
+
"steps": len(losses),
|
| 154 |
+
"seconds": round(elapsed, 1),
|
| 155 |
+
"losses": [round(v, 4) for v in losses],
|
| 156 |
+
}
|
| 157 |
+
del model, opt
|
| 158 |
+
torch.cuda.empty_cache()
|
| 159 |
+
return result
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def main():
|
| 163 |
+
parser = argparse.ArgumentParser()
|
| 164 |
+
parser.add_argument("--steps", type=int, default=300)
|
| 165 |
+
parser.add_argument("--layers", type=int, default=9)
|
| 166 |
+
parser.add_argument("--micro-batch", type=int, default=8)
|
| 167 |
+
parser.add_argument("--seed", type=int, default=0)
|
| 168 |
+
parser.add_argument("--tail", type=int, default=25,
|
| 169 |
+
help="average this many final steps for the verdict")
|
| 170 |
+
parser.add_argument("--schemes", default=",".join(SCHEMES))
|
| 171 |
+
parser.add_argument("--out", type=Path,
|
| 172 |
+
default=paths.RUNS / "init_study.json")
|
| 173 |
+
args = parser.parse_args()
|
| 174 |
+
|
| 175 |
+
if not torch.cuda.is_available():
|
| 176 |
+
raise SystemExit("needs a gpu")
|
| 177 |
+
|
| 178 |
+
from mimans.model import attention_tilelang, lm_head_tilelang, swiglu_tilelang
|
| 179 |
+
attention_tilelang.AUTOTUNE = False
|
| 180 |
+
lm_head_tilelang.AUTOTUNE = False
|
| 181 |
+
swiglu_tilelang.AUTOTUNE = False
|
| 182 |
+
|
| 183 |
+
weights = {"code": 0.6, "text": 0.3, "math": 0.1}
|
| 184 |
+
sources = open_shards(2048)
|
| 185 |
+
schemes = [s.strip() for s in args.schemes.split(",") if s.strip()]
|
| 186 |
+
|
| 187 |
+
print(f"{args.layers} layers, {args.steps} steps, micro batch "
|
| 188 |
+
f"{args.micro_batch} x 2048, seed {args.seed}")
|
| 189 |
+
print(f"log(vocab) = {math.log(ModelConfig().vocab):.4f}\n")
|
| 190 |
+
|
| 191 |
+
results = []
|
| 192 |
+
for scheme in schemes:
|
| 193 |
+
print(f" {scheme} ...", flush=True)
|
| 194 |
+
r = run_scheme(scheme, args, sources, weights)
|
| 195 |
+
results.append(r)
|
| 196 |
+
print(f" {scheme:12s} first {r['first_loss']:7.4f} "
|
| 197 |
+
f"final {r['final_loss']:7.4f} best {r['best_loss']:7.4f} "
|
| 198 |
+
f"live grads {r['live_grads']}/{r['live_grads']+r['dead_grads']} "
|
| 199 |
+
f"{r['seconds']:.0f}s", flush=True)
|
| 200 |
+
|
| 201 |
+
args.out.parent.mkdir(parents=True, exist_ok=True)
|
| 202 |
+
args.out.write_text(json.dumps(results, indent=2))
|
| 203 |
+
|
| 204 |
+
print(f"\n{'scheme':14s} {'first':>8s} {'final':>8s} {'best':>8s} {'live grads':>11s}")
|
| 205 |
+
for r in sorted(results, key=lambda r: r["final_loss"]):
|
| 206 |
+
print(f"{r['scheme']:14s} {r['first_loss']:8.4f} {r['final_loss']:8.4f} "
|
| 207 |
+
f"{r['best_loss']:8.4f} {r['live_grads']:6d}/"
|
| 208 |
+
f"{r['live_grads']+r['dead_grads']:<4d}")
|
| 209 |
+
best = min(results, key=lambda r: r["final_loss"])
|
| 210 |
+
print(f"\nlowest final loss: {best['scheme']}")
|
| 211 |
+
print(f"written to {args.out}")
|
| 212 |
+
return True
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
if __name__ == "__main__":
|
| 216 |
+
raise SystemExit(0 if main() else 1)
|
code/model/lm_head_tilelang.py
ADDED
|
@@ -0,0 +1,596 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# no "from __future__ import annotations" here: tilelang reads the prim_func
|
| 2 |
+
# signature with get_type_hints, and PEP 563 would leave T.Tensor([batch, ...])
|
| 3 |
+
# as a string that cannot see the enclosing function's locals.
|
| 4 |
+
import argparse
|
| 5 |
+
import itertools
|
| 6 |
+
import math
|
| 7 |
+
import os
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from mimans import paths # noqa: F401 points the kernel cache at D:, must precede tilelang
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
import tilelang
|
| 18 |
+
import tilelang.language as T
|
| 19 |
+
from tilelang.autotuner import autotune
|
| 20 |
+
HAVE_TILELANG = True
|
| 21 |
+
except ImportError:
|
| 22 |
+
HAVE_TILELANG = False
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
DTYPE = torch.bfloat16
|
| 26 |
+
SMEM_PER_SM_KB = {8: 164, 9: 228, 10: 228, 12: 128}
|
| 27 |
+
MAX_ACCUM_BYTES = 96 * 1024
|
| 28 |
+
IGNORE_INDEX = -100
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def shared_mem_limit(device=0, margin=0.9):
|
| 32 |
+
try:
|
| 33 |
+
if not torch.cuda.is_available():
|
| 34 |
+
return 112 * 1024
|
| 35 |
+
props = torch.cuda.get_device_properties(device)
|
| 36 |
+
for name in ("shared_memory_per_block_optin",
|
| 37 |
+
"max_shared_memory_per_block_optin",
|
| 38 |
+
"shared_memory_per_block"):
|
| 39 |
+
value = getattr(props, name, 0)
|
| 40 |
+
if value:
|
| 41 |
+
return int(value * margin)
|
| 42 |
+
return int(SMEM_PER_SM_KB.get(props.major, 100) * 1024 * margin)
|
| 43 |
+
except Exception:
|
| 44 |
+
return 112 * 1024
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
SMEM_LIMIT = shared_mem_limit()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def lse_smem(block_m, block_n, block_k, stages, elem=2):
|
| 51 |
+
return block_m * block_k * elem * stages + block_k * block_n * elem * stages
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def logits_bytes(tokens, vocab, elem=2):
|
| 55 |
+
return tokens * vocab * elem
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def warps_fit(block, threads, granularity=16):
|
| 59 |
+
warps = threads // 32
|
| 60 |
+
return block % warps == 0 and (block // warps) % granularity == 0
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def lse_configs(limit=None):
|
| 64 |
+
limit = limit or SMEM_LIMIT
|
| 65 |
+
return [dict(block_M=bm, block_N=bn, block_K=bk, threads=t, num_stages=s)
|
| 66 |
+
for bm, bn in itertools.product((32, 64, 128), (64, 128, 256))
|
| 67 |
+
for bk in (32, 64, 128)
|
| 68 |
+
for t in (128, 256) if warps_fit(bm, t)
|
| 69 |
+
for s in (1, 2, 3)
|
| 70 |
+
if lse_smem(bm, bn, bk, s) <= limit
|
| 71 |
+
and bm * bn * 4 <= MAX_ACCUM_BYTES]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
AUTOTUNE = True
|
| 75 |
+
LSE_TILES = dict(block_M=128, block_N=128, block_K=64, threads=128, num_stages=2)
|
| 76 |
+
VOCAB_CHUNK = 4096
|
| 77 |
+
|
| 78 |
+
assert LSE_TILES in lse_configs(), "lse default is not a tunable config"
|
| 79 |
+
assert VOCAB_CHUNK % 128 == 0, "vocab chunk should stay tensor core aligned"
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
if HAVE_TILELANG:
|
| 83 |
+
FAST_MATH = {tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True}
|
| 84 |
+
# CUDA 13.1's host_config.h only blesses MSVC 2019-2022 and this box has
|
| 85 |
+
# VS 18, so nvcc refuses every kernel. Override it here; unset
|
| 86 |
+
# TL_NVCC_FLAGS (or install the 2022 build tools) to get the check back.
|
| 87 |
+
JIT_FLAGS = os.environ.get("TL_NVCC_FLAGS", "-allow-unsupported-compiler").split()
|
| 88 |
+
# Same 32-rows-per-warp trap as the swiglu kernels; see swiglu.tunable.
|
| 89 |
+
from mimans.model.swiglu_tilelang import tunable
|
| 90 |
+
LSE_SPACE = tunable(lse_configs())
|
| 91 |
+
|
| 92 |
+
@autotune(configs=LSE_SPACE, warmup=10, rep=20)
|
| 93 |
+
@tilelang.jit(out_idx=[2], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 94 |
+
def logsumexp_kernel(tokens, d_model, vocab,
|
| 95 |
+
block_M=128, block_N=128, block_K=64,
|
| 96 |
+
threads=128, num_stages=2):
|
| 97 |
+
dt, acc = T.bfloat16, T.float32
|
| 98 |
+
|
| 99 |
+
@T.prim_func
|
| 100 |
+
def kernel(
|
| 101 |
+
H: T.Tensor([tokens, d_model], dt), #type: ignore
|
| 102 |
+
W: T.Tensor([d_model, vocab], dt), #type: ignore
|
| 103 |
+
Lse: T.Tensor([tokens], acc), #type: ignore
|
| 104 |
+
):
|
| 105 |
+
with T.Kernel(T.ceildiv(tokens, block_M), threads=threads) as bx:
|
| 106 |
+
h_s = T.alloc_shared([block_M, block_K], dt)
|
| 107 |
+
w_s = T.alloc_shared([block_K, block_N], dt)
|
| 108 |
+
logits = T.alloc_fragment([block_M, block_N], acc)
|
| 109 |
+
row_max = T.alloc_fragment([block_M], acc)
|
| 110 |
+
prev_max = T.alloc_fragment([block_M], acc)
|
| 111 |
+
rescale = T.alloc_fragment([block_M], acc)
|
| 112 |
+
tile_sum = T.alloc_fragment([block_M], acc)
|
| 113 |
+
total = T.alloc_fragment([block_M], acc)
|
| 114 |
+
|
| 115 |
+
for n in T.serial(T.ceildiv(vocab, block_N)):
|
| 116 |
+
# Seed the running max and sum inside the loop, predicated on
|
| 117 |
+
# the first tile. Warp specialisation for the pipelined k
|
| 118 |
+
# loop below sinks a *pre-loop* T.fill into the consumer
|
| 119 |
+
# branch inside this loop, which restarts the accumulator on
|
| 120 |
+
# every vocab tile and silently keeps only the last one.
|
| 121 |
+
for i in T.Parallel(block_M):
|
| 122 |
+
row_max[i] = T.if_then_else(n == 0, -T.infinity(acc), row_max[i])
|
| 123 |
+
total[i] = T.if_then_else(n == 0, T.Cast(acc, 0), total[i])
|
| 124 |
+
T.clear(logits)
|
| 125 |
+
for k in T.Pipelined(T.ceildiv(d_model, block_K),
|
| 126 |
+
num_stages=num_stages):
|
| 127 |
+
T.copy(H[bx * block_M:(bx + 1) * block_M,
|
| 128 |
+
k * block_K:(k + 1) * block_K], h_s)
|
| 129 |
+
T.copy(W[k * block_K:(k + 1) * block_K,
|
| 130 |
+
n * block_N:(n + 1) * block_N], w_s)
|
| 131 |
+
T.gemm(h_s, w_s, logits, policy=T.GemmWarpPolicy.FullRow)
|
| 132 |
+
|
| 133 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 134 |
+
logits[i, j] = T.if_then_else(
|
| 135 |
+
n * block_N + j >= vocab, -T.infinity(acc), logits[i, j])
|
| 136 |
+
|
| 137 |
+
# Save the running max with an explicit elementwise loop.
|
| 138 |
+
# T.copy between two fragments does not survive here, and a
|
| 139 |
+
# prev_max stuck at -inf makes every rescale zero, which
|
| 140 |
+
# silently throws away every vocab tile but the last.
|
| 141 |
+
for i in T.Parallel(block_M):
|
| 142 |
+
prev_max[i] = row_max[i]
|
| 143 |
+
T.reduce_max(logits, row_max, dim=1, clear=False)
|
| 144 |
+
for i in T.Parallel(block_M):
|
| 145 |
+
rescale[i] = T.exp(prev_max[i] - row_max[i])
|
| 146 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 147 |
+
logits[i, j] = T.exp(logits[i, j] - row_max[i])
|
| 148 |
+
T.reduce_sum(logits, tile_sum, dim=1)
|
| 149 |
+
for i in T.Parallel(block_M):
|
| 150 |
+
total[i] = total[i] * rescale[i] + tile_sum[i]
|
| 151 |
+
|
| 152 |
+
for i in T.Parallel(block_M):
|
| 153 |
+
total[i] = T.log(total[i]) + row_max[i]
|
| 154 |
+
T.copy(total, Lse[bx * block_M:(bx + 1) * block_M])
|
| 155 |
+
|
| 156 |
+
return kernel
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
_kernels: dict = {}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def get_kernel(shape):
|
| 163 |
+
key = (shape, AUTOTUNE)
|
| 164 |
+
if key not in _kernels:
|
| 165 |
+
from mimans.model import tuned
|
| 166 |
+
|
| 167 |
+
remembered = tuned.lookup("lm_head_lse", shape)
|
| 168 |
+
if remembered is not None:
|
| 169 |
+
kernel = logsumexp_kernel(*shape, **remembered)
|
| 170 |
+
elif AUTOTUNE:
|
| 171 |
+
kernel = logsumexp_kernel(*shape)
|
| 172 |
+
tuned.record("lm_head_lse", shape, tuned.config_of(kernel))
|
| 173 |
+
else:
|
| 174 |
+
kernel = logsumexp_kernel(*shape, **LSE_TILES)
|
| 175 |
+
_kernels[key] = kernel
|
| 176 |
+
return _kernels[key]
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def warmup(tokens, d_model=1280, vocab=49152):
|
| 180 |
+
"""Run the autotune search up front so training never stalls mid-step."""
|
| 181 |
+
print(f"tuning logsumexp over {len(LSE_SPACE)} configs")
|
| 182 |
+
get_kernel((tokens, d_model, vocab))
|
| 183 |
+
print("cached to disk")
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def accum_dtype(tensor):
|
| 187 |
+
"""fp32 for the dtypes we train in, but never narrower than what we were
|
| 188 |
+
handed: a plain .float() would silently drop an fp64 gradient check to
|
| 189 |
+
fp32 and cap its accuracy at 1e-7."""
|
| 190 |
+
return torch.promote_types(tensor.dtype, torch.float32)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def gather_target_logits(hidden, weight, targets):
|
| 194 |
+
acc = accum_dtype(hidden)
|
| 195 |
+
valid = targets.clamp(min=0)
|
| 196 |
+
rows = weight.index_select(1, valid).t()
|
| 197 |
+
return (hidden.to(acc) * rows.to(acc)).sum(-1)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
class FusedCrossEntropy(torch.autograd.Function):
|
| 201 |
+
@staticmethod
|
| 202 |
+
def forward(ctx, hidden, weight, targets, chunk):
|
| 203 |
+
tokens, d_model = hidden.shape
|
| 204 |
+
vocab = weight.shape[1]
|
| 205 |
+
mask = targets != IGNORE_INDEX
|
| 206 |
+
count = mask.sum().clamp(min=1)
|
| 207 |
+
|
| 208 |
+
acc = accum_dtype(hidden)
|
| 209 |
+
if HAVE_TILELANG and hidden.is_cuda and hidden.dtype is DTYPE:
|
| 210 |
+
lse = get_kernel((tokens, d_model, vocab))(hidden, weight)
|
| 211 |
+
else:
|
| 212 |
+
lse = torch.logsumexp((hidden.to(acc) @ weight.to(acc)), dim=-1)
|
| 213 |
+
|
| 214 |
+
target_logit = gather_target_logits(hidden, weight, targets)
|
| 215 |
+
per_token = torch.where(mask, lse - target_logit,
|
| 216 |
+
torch.zeros_like(lse))
|
| 217 |
+
loss = per_token.sum() / count
|
| 218 |
+
|
| 219 |
+
ctx.save_for_backward(hidden, weight, targets, lse, count)
|
| 220 |
+
ctx.chunk = chunk
|
| 221 |
+
return loss
|
| 222 |
+
|
| 223 |
+
@staticmethod
|
| 224 |
+
def backward(ctx, grad_loss):
|
| 225 |
+
hidden, weight, targets, lse, count = ctx.saved_tensors
|
| 226 |
+
tokens, d_model = hidden.shape
|
| 227 |
+
vocab = weight.shape[1]
|
| 228 |
+
chunk = ctx.chunk
|
| 229 |
+
mask = (targets != IGNORE_INDEX).unsqueeze(1)
|
| 230 |
+
scale = grad_loss / count
|
| 231 |
+
|
| 232 |
+
acc = accum_dtype(hidden)
|
| 233 |
+
d_hidden = torch.zeros_like(hidden, dtype=acc)
|
| 234 |
+
d_weight = torch.zeros_like(weight, dtype=acc)
|
| 235 |
+
rows = torch.arange(tokens, device=hidden.device)
|
| 236 |
+
|
| 237 |
+
for start in range(0, vocab, chunk):
|
| 238 |
+
stop = min(start + chunk, vocab)
|
| 239 |
+
w_chunk = weight[:, start:stop]
|
| 240 |
+
logits = hidden.to(acc) @ w_chunk.to(acc)
|
| 241 |
+
probs = torch.exp(logits - lse.unsqueeze(1))
|
| 242 |
+
hit = (targets >= start) & (targets < stop) & (targets != IGNORE_INDEX)
|
| 243 |
+
if hit.any():
|
| 244 |
+
idx = rows[hit]
|
| 245 |
+
probs[idx, targets[hit] - start] -= 1.0
|
| 246 |
+
probs = probs * mask * scale
|
| 247 |
+
d_hidden += probs @ w_chunk.to(acc).t()
|
| 248 |
+
d_weight[:, start:stop] = hidden.to(acc).t() @ probs
|
| 249 |
+
|
| 250 |
+
return d_hidden.to(hidden.dtype), d_weight.to(weight.dtype), None, None
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def fused_cross_entropy(hidden, weight, targets, chunk=VOCAB_CHUNK):
|
| 254 |
+
return FusedCrossEntropy.apply(hidden, weight, targets, chunk)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
@dataclass
|
| 258 |
+
class HeadConfig:
|
| 259 |
+
d_model: int = 1280
|
| 260 |
+
vocab: int = 49152
|
| 261 |
+
tied: bool = True
|
| 262 |
+
chunk: int = VOCAB_CHUNK
|
| 263 |
+
logit_scale: float = None
|
| 264 |
+
|
| 265 |
+
def __post_init__(self):
|
| 266 |
+
assert self.vocab % 128 == 0
|
| 267 |
+
assert self.d_model % 128 == 0
|
| 268 |
+
# A tied head reading a zero-init identity stack sees hidden =
|
| 269 |
+
# RMSNorm(E[token]), so the token's own logit is |E|^2/rms(E) =
|
| 270 |
+
# d_model * init_std = 25.6 while every other logit sits near zero.
|
| 271 |
+
# The model then starts out certain that the answer is the token it was
|
| 272 |
+
# just given, and step 0 costs 25.3 nats instead of log(vocab) = 10.8.
|
| 273 |
+
# Scaling the logits by 1/sqrt(d_model) puts that back at 0.72 nats of
|
| 274 |
+
# signal, which is the log(vocab) start the design report asks for.
|
| 275 |
+
# Set logit_scale=1.0 to get the unscaled behaviour back.
|
| 276 |
+
if self.logit_scale is None:
|
| 277 |
+
self.logit_scale = self.d_model ** -0.5
|
| 278 |
+
|
| 279 |
+
@property
|
| 280 |
+
def params(self):
|
| 281 |
+
return self.d_model * self.vocab
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
class LMHead(nn.Module):
|
| 285 |
+
def __init__(self, cfg: HeadConfig, embedding: nn.Embedding = None):
|
| 286 |
+
super().__init__()
|
| 287 |
+
self.cfg = cfg
|
| 288 |
+
if cfg.tied:
|
| 289 |
+
assert embedding is not None, "tied head needs the embedding module"
|
| 290 |
+
self.embedding = [embedding]
|
| 291 |
+
self.weight = None
|
| 292 |
+
else:
|
| 293 |
+
self.embedding = None
|
| 294 |
+
self.weight = nn.Parameter(torch.empty(cfg.d_model, cfg.vocab))
|
| 295 |
+
nn.init.normal_(self.weight, std=0.02)
|
| 296 |
+
|
| 297 |
+
def matrix(self):
|
| 298 |
+
if self.cfg.tied:
|
| 299 |
+
return self.embedding[0].weight.t()
|
| 300 |
+
return self.weight
|
| 301 |
+
|
| 302 |
+
def forward(self, hidden, targets=None, backend="tilelang"):
|
| 303 |
+
shape = hidden.shape
|
| 304 |
+
flat = hidden.reshape(-1, self.cfg.d_model)
|
| 305 |
+
# Scaling the input is the same as scaling the logits, and it keeps the
|
| 306 |
+
# kernel free of a scale it would have to fold into its own accumulator.
|
| 307 |
+
if self.cfg.logit_scale != 1.0:
|
| 308 |
+
flat = flat * self.cfg.logit_scale
|
| 309 |
+
weight = self.matrix()
|
| 310 |
+
|
| 311 |
+
if targets is None:
|
| 312 |
+
return (flat @ weight).reshape(*shape[:-1], self.cfg.vocab)
|
| 313 |
+
|
| 314 |
+
flat_targets = targets.reshape(-1)
|
| 315 |
+
if backend == "tilelang":
|
| 316 |
+
# The kernel is compiled for bf16. Under autocast the residual
|
| 317 |
+
# stream is still fp32 (the embedding is not an autocast op), and
|
| 318 |
+
# the tied matrix is always fp32, so cast both here -- otherwise the
|
| 319 |
+
# dtype guard in FusedCrossEntropy quietly drops to the torch path
|
| 320 |
+
# and materialises the [tokens, vocab] logits this kernel exists to
|
| 321 |
+
# avoid. The fp64 gradient checks call the function directly and
|
| 322 |
+
# keep their own precision.
|
| 323 |
+
# matrix() hands back embedding.weight.t(), a transposed view, and
|
| 324 |
+
# the kernel's packed ABI wants a real [d_model, vocab] layout.
|
| 325 |
+
if HAVE_TILELANG and flat.is_cuda:
|
| 326 |
+
flat = flat.to(DTYPE).contiguous()
|
| 327 |
+
weight = weight.to(DTYPE).contiguous()
|
| 328 |
+
return fused_cross_entropy(flat, weight, flat_targets, self.cfg.chunk)
|
| 329 |
+
if backend == "torch":
|
| 330 |
+
# sum over the valid rows divided by their count, matching the fused
|
| 331 |
+
# path. Plain F.cross_entropy means 0/0 -> nan when a microbatch is
|
| 332 |
+
# entirely ignore_index, and one nan poisons the whole run.
|
| 333 |
+
acc = accum_dtype(flat)
|
| 334 |
+
valid = (flat_targets != IGNORE_INDEX).sum().clamp(min=1)
|
| 335 |
+
return F.cross_entropy(flat.to(acc) @ weight.to(acc), flat_targets,
|
| 336 |
+
ignore_index=IGNORE_INDEX,
|
| 337 |
+
reduction="sum") / valid
|
| 338 |
+
raise ValueError(f"unknown backend {backend!r}")
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def check_math():
|
| 342 |
+
passed = True
|
| 343 |
+
|
| 344 |
+
def check(label, condition):
|
| 345 |
+
nonlocal passed
|
| 346 |
+
passed &= bool(condition)
|
| 347 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 348 |
+
|
| 349 |
+
torch.manual_seed(0)
|
| 350 |
+
cfg = HeadConfig()
|
| 351 |
+
mb = 1e6
|
| 352 |
+
tokens = 8 * 2048
|
| 353 |
+
|
| 354 |
+
print("the tensor this kernel deletes")
|
| 355 |
+
naive = logits_bytes(tokens, cfg.vocab)
|
| 356 |
+
softmax_fp32 = tokens * cfg.vocab * 4
|
| 357 |
+
fused = tokens * 4
|
| 358 |
+
print(f" logits bf16 {naive/mb:9.1f} MB")
|
| 359 |
+
print(f" fp32 softmax on top {softmax_fp32/mb:9.1f} MB")
|
| 360 |
+
print(f" fused lse only {fused/mb:9.4f} MB")
|
| 361 |
+
print(f" ratio {naive/fused:9.0f}x")
|
| 362 |
+
check("logits alone exceed 1.5 GB", naive > 1.5e9)
|
| 363 |
+
check("fused output is under a megabyte", fused < 1e6)
|
| 364 |
+
|
| 365 |
+
print("\nbackward peak, one vocabulary chunk at a time")
|
| 366 |
+
for chunk in (2048, 4096, 8192, cfg.vocab):
|
| 367 |
+
peak = tokens * chunk * 4
|
| 368 |
+
tag = "full vocab" if chunk == cfg.vocab else f"chunk {chunk}"
|
| 369 |
+
print(f" {tag:12s} {peak/mb:9.1f} MB")
|
| 370 |
+
check("chunk 4096 keeps the backward under 300 MB",
|
| 371 |
+
tokens * 4096 * 4 < 300e6)
|
| 372 |
+
|
| 373 |
+
print("\nconfig space")
|
| 374 |
+
space = lse_configs()
|
| 375 |
+
print(f" {len(space)} configs, smem limit {SMEM_LIMIT/1024:.0f} KB")
|
| 376 |
+
check("lse default is tunable", LSE_TILES in space)
|
| 377 |
+
check("every config fits shared memory",
|
| 378 |
+
all(lse_smem(c["block_M"], c["block_N"], c["block_K"],
|
| 379 |
+
c["num_stages"]) <= SMEM_LIMIT for c in space))
|
| 380 |
+
check("every config fits the register file",
|
| 381 |
+
all(c["block_M"] * c["block_N"] * 4 <= MAX_ACCUM_BYTES for c in space))
|
| 382 |
+
d = LSE_TILES
|
| 383 |
+
print(f" default smem {lse_smem(d['block_M'], d['block_N'], d['block_K'], d['num_stages'])/1024:.0f} KB,"
|
| 384 |
+
f" accumulator {d['block_M']*d['block_N']*4/1024:.0f} KB")
|
| 385 |
+
|
| 386 |
+
print("\nonline logsumexp recurrence")
|
| 387 |
+
x = torch.randn(64, 5000, dtype=torch.float64) * 8
|
| 388 |
+
running_max = torch.full((64,), float("-inf"), dtype=torch.float64)
|
| 389 |
+
running_sum = torch.zeros(64, dtype=torch.float64)
|
| 390 |
+
for start in range(0, 5000, 128):
|
| 391 |
+
tile = x[:, start:start + 128]
|
| 392 |
+
prev = running_max.clone()
|
| 393 |
+
running_max = torch.maximum(running_max, tile.max(dim=1).values)
|
| 394 |
+
running_sum = (running_sum * torch.exp(prev - running_max)
|
| 395 |
+
+ torch.exp(tile - running_max.unsqueeze(1)).sum(1))
|
| 396 |
+
online = torch.log(running_sum) + running_max
|
| 397 |
+
err = (online - torch.logsumexp(x, dim=1)).abs().max().item()
|
| 398 |
+
check(f"matches torch.logsumexp ({err:.2e})", err < 1e-12)
|
| 399 |
+
|
| 400 |
+
big = torch.tensor([[1e4, 1e4 + 1.0]], dtype=torch.float64)
|
| 401 |
+
check("no overflow at logits of 1e4",
|
| 402 |
+
torch.isfinite(torch.logsumexp(big, dim=1)).all())
|
| 403 |
+
|
| 404 |
+
print("\ncross entropy identity")
|
| 405 |
+
logits = torch.randn(200, 500, dtype=torch.float64)
|
| 406 |
+
targets = torch.randint(0, 500, (200,))
|
| 407 |
+
lse = torch.logsumexp(logits, dim=1)
|
| 408 |
+
manual = (lse - logits[torch.arange(200), targets]).mean()
|
| 409 |
+
ref = F.cross_entropy(logits, targets)
|
| 410 |
+
check(f"lse - target_logit == cross_entropy ({(manual-ref).abs():.2e})",
|
| 411 |
+
(manual - ref).abs() < 1e-12)
|
| 412 |
+
|
| 413 |
+
print("\ngradient identity")
|
| 414 |
+
logits = torch.randn(64, 200, dtype=torch.float64, requires_grad=True)
|
| 415 |
+
targets = torch.randint(0, 200, (64,))
|
| 416 |
+
F.cross_entropy(logits, targets).backward()
|
| 417 |
+
probs = torch.softmax(logits.detach(), dim=1)
|
| 418 |
+
onehot = F.one_hot(targets, 200).double()
|
| 419 |
+
analytic = (probs - onehot) / 64
|
| 420 |
+
err = (logits.grad - analytic).abs().max().item()
|
| 421 |
+
check(f"d_logits == (softmax - onehot)/N ({err:.2e})", err < 1e-12)
|
| 422 |
+
|
| 423 |
+
print("\nend to end against F.cross_entropy")
|
| 424 |
+
d_model, vocab, n = 64, 300, 128
|
| 425 |
+
hidden = torch.randn(n, d_model, dtype=torch.float64, requires_grad=True)
|
| 426 |
+
weight = (torch.randn(d_model, vocab, dtype=torch.float64) * 0.05).requires_grad_()
|
| 427 |
+
targets = torch.randint(0, vocab, (n,))
|
| 428 |
+
targets[:8] = IGNORE_INDEX
|
| 429 |
+
|
| 430 |
+
loss = fused_cross_entropy(hidden, weight, targets, chunk=64)
|
| 431 |
+
loss.backward()
|
| 432 |
+
mine = [hidden.grad.clone(), weight.grad.clone()]
|
| 433 |
+
hidden.grad = weight.grad = None
|
| 434 |
+
|
| 435 |
+
ref = F.cross_entropy(hidden @ weight, targets, ignore_index=IGNORE_INDEX)
|
| 436 |
+
ref.backward()
|
| 437 |
+
theirs = [hidden.grad.clone(), weight.grad.clone()]
|
| 438 |
+
hidden.grad = weight.grad = None
|
| 439 |
+
|
| 440 |
+
check(f"loss {loss.item():.10f} vs {ref.item():.10f}",
|
| 441 |
+
(loss - ref).abs() < 1e-10)
|
| 442 |
+
for name, a, b in zip(("d_hidden", "d_weight"), mine, theirs):
|
| 443 |
+
rel = ((a - b).abs().max() / b.abs().max()).item()
|
| 444 |
+
check(f"{name:9s} rel err {rel:.2e}", rel < 1e-9)
|
| 445 |
+
|
| 446 |
+
check("ignore_index rows contribute nothing",
|
| 447 |
+
fused_cross_entropy(hidden, weight,
|
| 448 |
+
torch.full_like(targets, IGNORE_INDEX), 64).item() == 0.0)
|
| 449 |
+
|
| 450 |
+
print("\ntied head")
|
| 451 |
+
emb = nn.Embedding(cfg.vocab, cfg.d_model)
|
| 452 |
+
head = LMHead(cfg, emb)
|
| 453 |
+
check("tied head holds no parameters of its own",
|
| 454 |
+
sum(p.numel() for p in head.parameters()) == 0)
|
| 455 |
+
check(f"shares the {cfg.params/1e6:.1f}M embedding matrix",
|
| 456 |
+
head.matrix().data_ptr() == emb.weight.data_ptr())
|
| 457 |
+
|
| 458 |
+
print("\n" + ("all math checks passed" if passed else "FAILURES ABOVE"))
|
| 459 |
+
print("still needs a GPU: kernel compilation, lse vs reference, tuning")
|
| 460 |
+
return passed
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def check_gpu(tokens=4096):
|
| 464 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 465 |
+
print("needs tilelang and cuda")
|
| 466 |
+
return
|
| 467 |
+
major, minor = torch.cuda.get_device_capability()
|
| 468 |
+
print(f"{torch.cuda.get_device_name()} sm_{major}{minor}")
|
| 469 |
+
print(f"shared memory limit {SMEM_LIMIT/1024:.0f} KB")
|
| 470 |
+
|
| 471 |
+
cfg = HeadConfig()
|
| 472 |
+
torch.manual_seed(0)
|
| 473 |
+
hidden = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE) * 0.1
|
| 474 |
+
weight = torch.randn(cfg.d_model, cfg.vocab, device="cuda", dtype=DTYPE) * 0.02
|
| 475 |
+
hidden.requires_grad_(True)
|
| 476 |
+
weight.requires_grad_(True)
|
| 477 |
+
targets = torch.randint(0, cfg.vocab, (tokens,), device="cuda")
|
| 478 |
+
|
| 479 |
+
lse = get_kernel((tokens, cfg.d_model, cfg.vocab))(hidden, weight)
|
| 480 |
+
want = torch.logsumexp(hidden.float() @ weight.float(), dim=-1)
|
| 481 |
+
err = (lse - want).abs().max().item()
|
| 482 |
+
print(f"\nlse vs reference: max err {err:.3e} {'ok' if err < 5e-3 else 'FAIL'}")
|
| 483 |
+
|
| 484 |
+
loss = fused_cross_entropy(hidden, weight, targets)
|
| 485 |
+
loss.backward()
|
| 486 |
+
mine = [hidden.grad.clone(), weight.grad.clone()]
|
| 487 |
+
hidden.grad = weight.grad = None
|
| 488 |
+
|
| 489 |
+
ref = F.cross_entropy(hidden.float() @ weight.float(), targets)
|
| 490 |
+
ref.backward()
|
| 491 |
+
theirs = [hidden.grad.clone(), weight.grad.clone()]
|
| 492 |
+
hidden.grad = weight.grad = None
|
| 493 |
+
|
| 494 |
+
print(f"loss {loss.item():.6f} vs {ref.item():.6f}")
|
| 495 |
+
for name, a, b in zip(("d_hidden", "d_weight"), mine, theirs):
|
| 496 |
+
rel = ((a.float() - b.float()).abs().max()
|
| 497 |
+
/ b.float().abs().max().clamp(min=1e-6)).item()
|
| 498 |
+
print(f" {name:9s} rel err {rel:.3e} {'ok' if rel < 3e-2 else 'FAIL'}")
|
| 499 |
+
|
| 500 |
+
torch.cuda.reset_peak_memory_stats()
|
| 501 |
+
fused_cross_entropy(hidden, weight, targets).backward()
|
| 502 |
+
fused_peak = torch.cuda.max_memory_allocated()
|
| 503 |
+
hidden.grad = weight.grad = None
|
| 504 |
+
torch.cuda.reset_peak_memory_stats()
|
| 505 |
+
F.cross_entropy(hidden.float() @ weight.float(), targets).backward()
|
| 506 |
+
naive_peak = torch.cuda.max_memory_allocated()
|
| 507 |
+
print(f"\npeak memory fused {fused_peak/1e6:.0f} MB naive {naive_peak/1e6:.0f} MB"
|
| 508 |
+
f" saved {(naive_peak-fused_peak)/1e6:.0f} MB")
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def benchmark(tokens=16384):
|
| 512 |
+
if not torch.cuda.is_available():
|
| 513 |
+
print("needs cuda")
|
| 514 |
+
return
|
| 515 |
+
cfg = HeadConfig()
|
| 516 |
+
hidden = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE,
|
| 517 |
+
requires_grad=True) * 0.1
|
| 518 |
+
weight = torch.randn(cfg.d_model, cfg.vocab, device="cuda", dtype=DTYPE,
|
| 519 |
+
requires_grad=True) * 0.02
|
| 520 |
+
targets = torch.randint(0, cfg.vocab, (tokens,), device="cuda")
|
| 521 |
+
|
| 522 |
+
def time_it(fn, iters=20):
|
| 523 |
+
for _ in range(5):
|
| 524 |
+
fn()
|
| 525 |
+
torch.cuda.synchronize()
|
| 526 |
+
start, end = torch.cuda.Event(True), torch.cuda.Event(True)
|
| 527 |
+
start.record()
|
| 528 |
+
for _ in range(iters):
|
| 529 |
+
fn()
|
| 530 |
+
end.record()
|
| 531 |
+
torch.cuda.synchronize()
|
| 532 |
+
return start.elapsed_time(end) / iters
|
| 533 |
+
|
| 534 |
+
def clear():
|
| 535 |
+
hidden.grad = weight.grad = None
|
| 536 |
+
|
| 537 |
+
results = {}
|
| 538 |
+
|
| 539 |
+
def naive():
|
| 540 |
+
F.cross_entropy(hidden.float() @ weight.float(), targets).backward()
|
| 541 |
+
clear()
|
| 542 |
+
try:
|
| 543 |
+
results["naive"] = time_it(naive)
|
| 544 |
+
except torch.cuda.OutOfMemoryError:
|
| 545 |
+
print("naive path ran out of memory, which is the point")
|
| 546 |
+
|
| 547 |
+
if HAVE_TILELANG:
|
| 548 |
+
try:
|
| 549 |
+
def fused():
|
| 550 |
+
fused_cross_entropy(hidden, weight, targets).backward()
|
| 551 |
+
clear()
|
| 552 |
+
results["fused"] = time_it(fused)
|
| 553 |
+
except Exception as exc:
|
| 554 |
+
print(f"fused failed: {type(exc).__name__}: {exc}")
|
| 555 |
+
|
| 556 |
+
print()
|
| 557 |
+
for name, ms in sorted(results.items(), key=lambda kv: kv[1]):
|
| 558 |
+
print(f" {name:8s} {ms:8.3f} ms")
|
| 559 |
+
if len(results) == 2:
|
| 560 |
+
print(f"\n speed {results['naive']/results['fused']:.2f}x")
|
| 561 |
+
print(" memory is the real win, see --gpu for peak allocation")
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def tune(tokens=16384):
|
| 565 |
+
print(f"lse candidates {len(lse_configs())}")
|
| 566 |
+
print(f"shared memory limit {SMEM_LIMIT/1024:.0f} KB")
|
| 567 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 568 |
+
for c in lse_configs()[:12]:
|
| 569 |
+
print(f" {c}")
|
| 570 |
+
return
|
| 571 |
+
cfg = HeadConfig()
|
| 572 |
+
warmup(tokens, cfg.d_model, cfg.vocab)
|
| 573 |
+
kernel = get_kernel((tokens, cfg.d_model, cfg.vocab))
|
| 574 |
+
print(f"\nbest: {getattr(kernel, 'config', None)}")
|
| 575 |
+
latency = getattr(kernel, "latency", None)
|
| 576 |
+
if latency is not None:
|
| 577 |
+
print(f" {latency:.3f} ms")
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
if __name__ == "__main__":
|
| 581 |
+
parser = argparse.ArgumentParser()
|
| 582 |
+
parser.add_argument("--check", action="store_true")
|
| 583 |
+
parser.add_argument("--gpu", action="store_true")
|
| 584 |
+
parser.add_argument("--bench", action="store_true")
|
| 585 |
+
parser.add_argument("--tune", action="store_true")
|
| 586 |
+
args = parser.parse_args()
|
| 587 |
+
if not any(vars(args).values()):
|
| 588 |
+
args.check = True
|
| 589 |
+
if args.check:
|
| 590 |
+
check_math()
|
| 591 |
+
if args.gpu:
|
| 592 |
+
check_gpu()
|
| 593 |
+
if args.bench:
|
| 594 |
+
benchmark()
|
| 595 |
+
if args.tune:
|
| 596 |
+
tune()
|
code/model/model.py
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from dataclasses import dataclass, field
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
|
| 9 |
+
from mimans.model.norm import RMSNorm
|
| 10 |
+
from mimans.model.attnres import ResidualState
|
| 11 |
+
from mimans.model.attention_tilelang import Rotary
|
| 12 |
+
from mimans.model.block import BlockConfig, TransformerBlock
|
| 13 |
+
from mimans.model.lm_head_tilelang import HeadConfig, LMHead, IGNORE_INDEX
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class ModelConfig:
|
| 18 |
+
vocab: int = 49152
|
| 19 |
+
d_model: int = 1280
|
| 20 |
+
n_layers: int = 27
|
| 21 |
+
n_heads: int = 10
|
| 22 |
+
n_kv_heads: int = 2
|
| 23 |
+
head_dim: int = 128
|
| 24 |
+
ffn_hidden: int = 3328
|
| 25 |
+
layers_per_block: int = 3
|
| 26 |
+
norm_eps: float = 1e-5
|
| 27 |
+
rope_theta: float = 10_000.0
|
| 28 |
+
max_seq_len: int = 8192
|
| 29 |
+
init_std: float = 0.02
|
| 30 |
+
use_attnres: bool = True
|
| 31 |
+
tied: bool = True
|
| 32 |
+
attn_backend: str = "tilelang"
|
| 33 |
+
ffn_backend: str = "tilelang"
|
| 34 |
+
head_backend: str = "tilelang"
|
| 35 |
+
# Recompute block internals in the backward pass instead of storing them.
|
| 36 |
+
# Off by default so nothing that already worked at 2048 changes; the
|
| 37 |
+
# training config turns it on for long context. See CodeModel._checkpointed_block.
|
| 38 |
+
gradient_checkpointing: bool = False
|
| 39 |
+
|
| 40 |
+
def block_config(self):
|
| 41 |
+
return BlockConfig(
|
| 42 |
+
d_model=self.d_model, n_heads=self.n_heads, n_kv_heads=self.n_kv_heads,
|
| 43 |
+
head_dim=self.head_dim, ffn_hidden=self.ffn_hidden,
|
| 44 |
+
n_layers=self.n_layers, layers_per_block=self.layers_per_block,
|
| 45 |
+
norm_eps=self.norm_eps, rope_theta=self.rope_theta,
|
| 46 |
+
max_seq_len=self.max_seq_len, use_attnres=self.use_attnres,
|
| 47 |
+
attn_backend=self.attn_backend, ffn_backend=self.ffn_backend)
|
| 48 |
+
|
| 49 |
+
def head_config(self):
|
| 50 |
+
return HeadConfig(d_model=self.d_model, vocab=self.vocab, tied=self.tied)
|
| 51 |
+
|
| 52 |
+
@property
|
| 53 |
+
def n_blocks(self):
|
| 54 |
+
return self.n_layers // self.layers_per_block
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class CodeModel(nn.Module):
|
| 58 |
+
def __init__(self, cfg: ModelConfig):
|
| 59 |
+
super().__init__()
|
| 60 |
+
self.cfg = cfg
|
| 61 |
+
self.embedding = nn.Embedding(cfg.vocab, cfg.d_model)
|
| 62 |
+
self.rope = Rotary(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
|
| 63 |
+
block_cfg = cfg.block_config()
|
| 64 |
+
self.blocks = nn.ModuleList(
|
| 65 |
+
TransformerBlock(block_cfg, self.rope) for _ in range(cfg.n_layers))
|
| 66 |
+
self.final_norm = RMSNorm(cfg.d_model, cfg.norm_eps)
|
| 67 |
+
self.head = LMHead(cfg.head_config(), self.embedding)
|
| 68 |
+
self.gradient_checkpointing = cfg.gradient_checkpointing
|
| 69 |
+
self.init_weights()
|
| 70 |
+
|
| 71 |
+
def init_weights(self):
|
| 72 |
+
nn.init.normal_(self.embedding.weight, std=self.cfg.init_std)
|
| 73 |
+
for block in self.blocks:
|
| 74 |
+
for name, p in block.named_parameters():
|
| 75 |
+
if p.dim() >= 2 and "w_down" not in name and "wo" not in name:
|
| 76 |
+
nn.init.normal_(p, std=self.cfg.init_std)
|
| 77 |
+
block.zero_init_projections()
|
| 78 |
+
|
| 79 |
+
def set_rope_theta(self, theta: float):
|
| 80 |
+
self.rope.set_theta(theta)
|
| 81 |
+
self.cfg.rope_theta = theta
|
| 82 |
+
|
| 83 |
+
def _checkpointed_block(self, block, state: ResidualState):
|
| 84 |
+
"""One block, with its internals recomputed in the backward pass.
|
| 85 |
+
|
| 86 |
+
This is what makes 8192 context fit in 32 GB, and it is a memory/compute
|
| 87 |
+
trade rather than any change to the maths -- the same numbers come out.
|
| 88 |
+
|
| 89 |
+
The reason it is needed is the AttnRes gate. Every gate call stacks all
|
| 90 |
+
of the block snapshots plus the running stream into one tensor and RMSNorms
|
| 91 |
+
it, and the residual stream is fp32 because nn.Embedding is not an autocast
|
| 92 |
+
op. At 27 layers that is 54 gate calls, each holding two tensors of
|
| 93 |
+
(entries, batch, seq, d_model) alive for the backward pass. Cost grows
|
| 94 |
+
linearly with context, which is why 2048 fits and 8192 needs ~45 GB.
|
| 95 |
+
|
| 96 |
+
Recomputing gets that back for roughly one extra forward pass. What is
|
| 97 |
+
stored instead is just the block's boundary tensors, and the snapshots are
|
| 98 |
+
shared references across blocks rather than copies.
|
| 99 |
+
|
| 100 |
+
The block mutates a ResidualState in place, which torch.utils.checkpoint
|
| 101 |
+
cannot see through, so it is unpacked into tensors on the way in and
|
| 102 |
+
repacked on the way out. `steps` is passed rather than inferred so that
|
| 103 |
+
`advance` appends a snapshot on exactly the blocks it otherwise would.
|
| 104 |
+
"""
|
| 105 |
+
from torch.utils.checkpoint import checkpoint
|
| 106 |
+
|
| 107 |
+
steps, snapshots = state.steps, tuple(state.snapshots)
|
| 108 |
+
|
| 109 |
+
def run(running, *snaps):
|
| 110 |
+
inner = ResidualState(running=running, snapshots=list(snaps),
|
| 111 |
+
steps=steps)
|
| 112 |
+
block(inner)
|
| 113 |
+
# A block advances two sublayers and so appends at most one
|
| 114 |
+
# snapshot. Return whatever is new, not the whole list.
|
| 115 |
+
return (inner.running, *inner.snapshots[len(snaps):])
|
| 116 |
+
|
| 117 |
+
out = checkpoint(run, state.running, *snapshots, use_reentrant=False)
|
| 118 |
+
out = (out,) if torch.is_tensor(out) else tuple(out)
|
| 119 |
+
return ResidualState(running=out[0],
|
| 120 |
+
snapshots=list(snapshots) + list(out[1:]),
|
| 121 |
+
steps=steps + 2)
|
| 122 |
+
|
| 123 |
+
def hidden_states(self, tokens):
|
| 124 |
+
h = self.embedding(tokens)
|
| 125 |
+
state = ResidualState(running=h)
|
| 126 |
+
# Only while training: inference has no backward pass to save memory for,
|
| 127 |
+
# and recomputing would just make it slower.
|
| 128 |
+
use_checkpoint = (getattr(self, "gradient_checkpointing", False)
|
| 129 |
+
and self.training and torch.is_grad_enabled())
|
| 130 |
+
for block in self.blocks:
|
| 131 |
+
if use_checkpoint:
|
| 132 |
+
state = self._checkpointed_block(block, state)
|
| 133 |
+
else:
|
| 134 |
+
block(state)
|
| 135 |
+
return self.final_norm(state.running)
|
| 136 |
+
|
| 137 |
+
def forward(self, tokens, targets=None):
|
| 138 |
+
hidden = self.hidden_states(tokens)
|
| 139 |
+
if targets is None:
|
| 140 |
+
return self.head(hidden, None)
|
| 141 |
+
return self.head(hidden, targets, backend=self.cfg.head_backend)
|
| 142 |
+
|
| 143 |
+
def param_groups(self, weight_decay=0.1):
|
| 144 |
+
matrices, others = [], []
|
| 145 |
+
for name, p in self.named_parameters():
|
| 146 |
+
if not p.requires_grad:
|
| 147 |
+
continue
|
| 148 |
+
if p.dim() >= 2 and "embedding" not in name:
|
| 149 |
+
matrices.append(p)
|
| 150 |
+
else:
|
| 151 |
+
others.append(p)
|
| 152 |
+
return [
|
| 153 |
+
{"params": matrices, "weight_decay": weight_decay, "optimizer": "muon"},
|
| 154 |
+
{"params": others, "weight_decay": 0.0, "optimizer": "adamw"},
|
| 155 |
+
]
|
| 156 |
+
|
| 157 |
+
def n_parameters(self, matmul_only=False):
|
| 158 |
+
if not matmul_only:
|
| 159 |
+
return sum(p.numel() for p in self.parameters())
|
| 160 |
+
total = sum(p.numel() for n, p in self.named_parameters() if p.dim() >= 2)
|
| 161 |
+
return total + self.cfg.vocab * self.cfg.d_model - (
|
| 162 |
+
self.cfg.vocab * self.cfg.d_model if not self.cfg.tied else 0)
|
| 163 |
+
|
| 164 |
+
def flops_per_token(self, seq_len=2048):
|
| 165 |
+
cfg = self.cfg
|
| 166 |
+
attn = (cfg.d_model * cfg.n_heads * cfg.head_dim
|
| 167 |
+
+ 2 * cfg.d_model * cfg.n_kv_heads * cfg.head_dim
|
| 168 |
+
+ cfg.n_heads * cfg.head_dim * cfg.d_model)
|
| 169 |
+
ffn = 3 * cfg.d_model * cfg.ffn_hidden
|
| 170 |
+
dense = 6 * ((attn + ffn) * cfg.n_layers + cfg.vocab * cfg.d_model)
|
| 171 |
+
scores = 6 * seq_len * cfg.n_heads * cfg.head_dim * cfg.n_layers
|
| 172 |
+
return dense, scores
|
| 173 |
+
|
| 174 |
+
def training_days(self, tokens=40e9, seq_len=2048, peak_tflops=209.5, mfu=0.35,
|
| 175 |
+
overhead=1.05):
|
| 176 |
+
dense, scores = self.flops_per_token(seq_len)
|
| 177 |
+
per_token = dense + scores
|
| 178 |
+
seconds = tokens * per_token / (peak_tflops * 1e12 * mfu)
|
| 179 |
+
return seconds / 86400 * overhead
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def check():
|
| 183 |
+
torch.manual_seed(0)
|
| 184 |
+
passed = True
|
| 185 |
+
|
| 186 |
+
def check_one(label, condition):
|
| 187 |
+
nonlocal passed
|
| 188 |
+
passed &= bool(condition)
|
| 189 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 190 |
+
|
| 191 |
+
cfg = ModelConfig(attn_backend="math", ffn_backend="math", head_backend="torch")
|
| 192 |
+
model = CodeModel(cfg)
|
| 193 |
+
|
| 194 |
+
print("parameter budget")
|
| 195 |
+
total = model.n_parameters()
|
| 196 |
+
body = sum(p.numel() for b in model.blocks for p in b.parameters())
|
| 197 |
+
emb = model.embedding.weight.numel()
|
| 198 |
+
print(f" embedding (tied) {emb:,}")
|
| 199 |
+
print(f" 27 blocks {body:,}")
|
| 200 |
+
print(f" final norm {model.final_norm.weight.numel():,}")
|
| 201 |
+
print(f" TOTAL {total:,} = {total/1e6:.1f}M")
|
| 202 |
+
check_one("matches the design report 514,345,526", total == 514_345_526)
|
| 203 |
+
check_one("head adds no parameters of its own",
|
| 204 |
+
sum(p.numel() for p in model.head.parameters()) == 0)
|
| 205 |
+
check_one("embedding is 12.2 percent of the model",
|
| 206 |
+
abs(emb / total - 0.1223) < 0.001)
|
| 207 |
+
|
| 208 |
+
print("\ntied weights")
|
| 209 |
+
check_one("head matrix is the embedding transposed",
|
| 210 |
+
model.head.matrix().data_ptr() == model.embedding.weight.data_ptr())
|
| 211 |
+
names = [n for n, _ in model.named_parameters() if "embedding" in n]
|
| 212 |
+
check_one(f"embedding appears once in named_parameters {names}", len(names) == 1)
|
| 213 |
+
|
| 214 |
+
print("\ncompute")
|
| 215 |
+
dense, scores = model.flops_per_token(2048)
|
| 216 |
+
print(f" dense {dense/1e9:.3f} GF/token")
|
| 217 |
+
print(f" attention scores {scores/1e9:.3f} GF/token "
|
| 218 |
+
f"({scores/dense*100:.1f} percent overhead)")
|
| 219 |
+
print(f" total {(dense+scores)/1e9:.3f} GF/token")
|
| 220 |
+
check_one("attention overhead near 14 percent at seq 2048",
|
| 221 |
+
0.10 < scores / dense < 0.16)
|
| 222 |
+
for seq in (2048, 8192):
|
| 223 |
+
d, s = model.flops_per_token(seq)
|
| 224 |
+
print(f" seq {seq:5d}: {(d+s)/1e9:.3f} GF/token, "
|
| 225 |
+
f"{s/d*100:5.1f} percent in attention")
|
| 226 |
+
days = model.training_days()
|
| 227 |
+
print(f" 40B tokens at 35 percent MFU: {days:.1f} days")
|
| 228 |
+
check_one("lands in the 20 to 26 day window from the report", 20 < days < 26)
|
| 229 |
+
|
| 230 |
+
print("\nforward")
|
| 231 |
+
tokens = torch.randint(0, cfg.vocab, (2, 16))
|
| 232 |
+
logits = model(tokens)
|
| 233 |
+
check_one(f"logits {tuple(logits.shape)}", logits.shape == (2, 16, cfg.vocab))
|
| 234 |
+
|
| 235 |
+
print("\nloss of an untrained model")
|
| 236 |
+
targets = torch.randint(0, cfg.vocab, (2, 16))
|
| 237 |
+
loss = model(tokens, targets)
|
| 238 |
+
uniform = math.log(cfg.vocab)
|
| 239 |
+
print(f" loss {loss.item():.4f}, log(vocab) {uniform:.4f}")
|
| 240 |
+
check_one(f"within 5 percent of log(vocab)",
|
| 241 |
+
abs(loss.item() - uniform) / uniform < 0.05)
|
| 242 |
+
check_one("this is the canary for a broken init", torch.isfinite(loss))
|
| 243 |
+
|
| 244 |
+
print("\nidentity at init")
|
| 245 |
+
h = model.hidden_states(tokens)
|
| 246 |
+
emb_normed = model.final_norm(model.embedding(tokens))
|
| 247 |
+
check_one(f"zero init projections pass embeddings through "
|
| 248 |
+
f"({(h - emb_normed).abs().max():.2e})",
|
| 249 |
+
(h - emb_normed).abs().max() < 1e-5)
|
| 250 |
+
|
| 251 |
+
print("\ncausality")
|
| 252 |
+
torch.manual_seed(1)
|
| 253 |
+
live = CodeModel(ModelConfig(n_layers=3, attn_backend="math",
|
| 254 |
+
ffn_backend="math", head_backend="torch"))
|
| 255 |
+
with torch.no_grad():
|
| 256 |
+
for n, p in live.named_parameters():
|
| 257 |
+
if p.dim() >= 2:
|
| 258 |
+
p.normal_(std=0.02)
|
| 259 |
+
a = torch.randint(0, live.cfg.vocab, (1, 12))
|
| 260 |
+
b = a.clone()
|
| 261 |
+
b[0, -1] = (b[0, -1] + 1) % live.cfg.vocab
|
| 262 |
+
drift = (live(a)[:, :-1] - live(b)[:, :-1]).abs().max().item()
|
| 263 |
+
check_one(f"changing the last token leaves earlier logits alone ({drift:.2e})",
|
| 264 |
+
drift < 1e-4)
|
| 265 |
+
|
| 266 |
+
print("\nparameter groups")
|
| 267 |
+
groups = live.param_groups()
|
| 268 |
+
muon = sum(p.numel() for p in groups[0]["params"])
|
| 269 |
+
adamw = sum(p.numel() for p in groups[1]["params"])
|
| 270 |
+
print(f" muon {muon:,} in {len(groups[0]['params'])} tensors, "
|
| 271 |
+
f"decay {groups[0]['weight_decay']}")
|
| 272 |
+
print(f" adamw {adamw:,} in {len(groups[1]['params'])} tensors, "
|
| 273 |
+
f"decay {groups[1]['weight_decay']}")
|
| 274 |
+
check_one("groups cover every parameter", muon + adamw == live.n_parameters())
|
| 275 |
+
check_one("no weight decay outside hidden matrices",
|
| 276 |
+
groups[1]["weight_decay"] == 0.0)
|
| 277 |
+
check_one("embedding is not in the muon group",
|
| 278 |
+
all(p.data_ptr() != live.embedding.weight.data_ptr()
|
| 279 |
+
for p in groups[0]["params"]))
|
| 280 |
+
|
| 281 |
+
print("\ngradients")
|
| 282 |
+
small = CodeModel(ModelConfig(n_layers=3, attn_backend="math",
|
| 283 |
+
ffn_backend="math", head_backend="torch"))
|
| 284 |
+
t = torch.randint(0, small.cfg.vocab, (2, 8))
|
| 285 |
+
y = torch.randint(0, small.cfg.vocab, (2, 8))
|
| 286 |
+
small(t, y).backward()
|
| 287 |
+
missing = [n for n, p in small.named_parameters()
|
| 288 |
+
if p.grad is None or not torch.isfinite(p.grad).all()]
|
| 289 |
+
check_one(f"every parameter receives a finite gradient"
|
| 290 |
+
f"{'' if not missing else ' -> ' + str(missing[:3])}", not missing)
|
| 291 |
+
check_one("embedding gets gradient from both the lookup and the head",
|
| 292 |
+
small.embedding.weight.grad.abs().sum() > 0)
|
| 293 |
+
|
| 294 |
+
print("\nignore index")
|
| 295 |
+
masked = torch.full_like(y, IGNORE_INDEX)
|
| 296 |
+
check_one("all masked targets give zero loss",
|
| 297 |
+
small(t, masked).item() == 0.0)
|
| 298 |
+
|
| 299 |
+
print("\nstage 3 rope switch")
|
| 300 |
+
model.set_rope_theta(500_000.0)
|
| 301 |
+
check_one("theta updated on the shared table", model.rope.theta == 500_000.0)
|
| 302 |
+
check_one("config follows", model.cfg.rope_theta == 500_000.0)
|
| 303 |
+
check_one("every block shares one rope module",
|
| 304 |
+
all(b.attn.rope is model.rope for b in model.blocks))
|
| 305 |
+
|
| 306 |
+
print("\n" + ("model ok" if passed else "FAILURES ABOVE"))
|
| 307 |
+
return passed
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
if __name__ == "__main__":
|
| 311 |
+
check()
|
code/model/norm.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
HAVE_NATIVE = hasattr(F, "rms_norm")
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class RMSNorm(nn.Module):
|
| 11 |
+
def __init__(self, dim: int, eps: float = 1e-5, impl: str = "native"):
|
| 12 |
+
super().__init__()
|
| 13 |
+
self.dim = dim
|
| 14 |
+
self.eps = eps
|
| 15 |
+
self.impl = impl if (impl != "native" or HAVE_NATIVE) else "manual"
|
| 16 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 17 |
+
|
| 18 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 19 |
+
if self.impl == "native":
|
| 20 |
+
return F.rms_norm(x, (self.dim,), self.weight.to(x.dtype), self.eps)
|
| 21 |
+
dtype = x.dtype
|
| 22 |
+
h = x.float()
|
| 23 |
+
h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 24 |
+
return (h * self.weight.float()).to(dtype)
|
| 25 |
+
|
| 26 |
+
def extra_repr(self) -> str:
|
| 27 |
+
return f"dim={self.dim}, eps={self.eps}, impl={self.impl}"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def reference(x, weight, eps=1e-5):
|
| 31 |
+
h = x.double()
|
| 32 |
+
h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + eps)
|
| 33 |
+
return h * weight.double()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def dispatch_warning(module: RMSNorm, x: torch.Tensor) -> str:
|
| 37 |
+
if module.impl != "native":
|
| 38 |
+
return "manual implementation, no fused dispatch"
|
| 39 |
+
if module.weight.dtype != x.dtype:
|
| 40 |
+
return (f"weight {module.weight.dtype} vs input {x.dtype}: "
|
| 41 |
+
"cast at the call site or the fused kernel is skipped")
|
| 42 |
+
return "dtypes match, fused dispatch available"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def check():
|
| 46 |
+
torch.manual_seed(0)
|
| 47 |
+
passed = True
|
| 48 |
+
|
| 49 |
+
def check_one(label, condition):
|
| 50 |
+
nonlocal passed
|
| 51 |
+
passed &= bool(condition)
|
| 52 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 53 |
+
|
| 54 |
+
print(f"native F.rms_norm available: {HAVE_NATIVE}")
|
| 55 |
+
print(f"torch {torch.__version__}")
|
| 56 |
+
|
| 57 |
+
print("\nnative matches the fp64 reference")
|
| 58 |
+
native = RMSNorm(1280, impl="native")
|
| 59 |
+
manual = RMSNorm(1280, impl="manual")
|
| 60 |
+
with torch.no_grad():
|
| 61 |
+
w = torch.randn(1280) * 0.1 + 1.0
|
| 62 |
+
native.weight.copy_(w)
|
| 63 |
+
manual.weight.copy_(w)
|
| 64 |
+
x = torch.randn(4, 16, 1280)
|
| 65 |
+
want = reference(x, w).float()
|
| 66 |
+
err_native = (native(x) - want).abs().max().item()
|
| 67 |
+
err_manual = (manual(x) - want).abs().max().item()
|
| 68 |
+
check_one(f"native err {err_native:.2e}", err_native < 1e-4)
|
| 69 |
+
check_one(f"manual err {err_manual:.2e}", err_manual < 1e-4)
|
| 70 |
+
check_one(f"native and manual agree "
|
| 71 |
+
f"({(native(x) - manual(x)).abs().max():.2e})",
|
| 72 |
+
(native(x) - manual(x)).abs().max() < 1e-5)
|
| 73 |
+
|
| 74 |
+
print("\nfused dispatch check")
|
| 75 |
+
bf16 = torch.randn(4, 16, 1280, dtype=torch.bfloat16)
|
| 76 |
+
fp32_weight = RMSNorm(1280)
|
| 77 |
+
print(f" fp32 weight, bf16 input : {dispatch_warning(fp32_weight, bf16)}")
|
| 78 |
+
cast = RMSNorm(1280).to(torch.bfloat16)
|
| 79 |
+
print(f" bf16 weight, bf16 input : {dispatch_warning(cast, bf16)}")
|
| 80 |
+
check_one("forward casts the weight so dispatch is never silently lost",
|
| 81 |
+
torch.isfinite(fp32_weight(bf16)).all())
|
| 82 |
+
|
| 83 |
+
print("\nproperties")
|
| 84 |
+
y = native(x)
|
| 85 |
+
check_one(f"unit rms with unit gain "
|
| 86 |
+
f"({RMSNorm(1280)(x).pow(2).mean().sqrt():.4f})",
|
| 87 |
+
abs(RMSNorm(1280)(x).pow(2).mean().sqrt() - 1) < 0.02)
|
| 88 |
+
check_one("scale invariant", (native(x * 100) - y).abs().max() < 2e-3)
|
| 89 |
+
check_one("gain starts at one", (RMSNorm(1280).weight == 1).all())
|
| 90 |
+
check_one(f"parameters {native.weight.numel()}", native.weight.numel() == 1280)
|
| 91 |
+
|
| 92 |
+
big = torch.randn(2, 4, 1280, dtype=torch.bfloat16) * 1e4
|
| 93 |
+
check_one("no overflow in bf16 at 1e4",
|
| 94 |
+
torch.isfinite(RMSNorm(1280)(big)).all())
|
| 95 |
+
|
| 96 |
+
print("\ngradients")
|
| 97 |
+
xa = torch.randn(2, 8, 1280, requires_grad=True)
|
| 98 |
+
xb = xa.detach().clone().requires_grad_(True)
|
| 99 |
+
RMSNorm(1280, impl="native")(xa).sum().backward()
|
| 100 |
+
RMSNorm(1280, impl="manual")(xb).sum().backward()
|
| 101 |
+
check_one(f"native and manual grads agree "
|
| 102 |
+
f"({(xa.grad - xb.grad).abs().max():.2e})",
|
| 103 |
+
(xa.grad - xb.grad).abs().max() < 1e-5)
|
| 104 |
+
check_one("finite", torch.isfinite(xa.grad).all())
|
| 105 |
+
|
| 106 |
+
print("\n" + ("rmsnorm ok" if passed else "FAILURES ABOVE"))
|
| 107 |
+
return passed
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def benchmark(tokens=16384, dim=1280, iters=50):
|
| 111 |
+
if not torch.cuda.is_available():
|
| 112 |
+
print("needs cuda")
|
| 113 |
+
return
|
| 114 |
+
x = torch.randn(tokens, dim, device="cuda", dtype=torch.bfloat16,
|
| 115 |
+
requires_grad=True)
|
| 116 |
+
grad = torch.randn_like(x)
|
| 117 |
+
|
| 118 |
+
def time_it(fn):
|
| 119 |
+
for _ in range(10):
|
| 120 |
+
fn()
|
| 121 |
+
torch.cuda.synchronize()
|
| 122 |
+
start, end = torch.cuda.Event(True), torch.cuda.Event(True)
|
| 123 |
+
start.record()
|
| 124 |
+
for _ in range(iters):
|
| 125 |
+
fn()
|
| 126 |
+
end.record()
|
| 127 |
+
torch.cuda.synchronize()
|
| 128 |
+
return start.elapsed_time(end) / iters
|
| 129 |
+
|
| 130 |
+
variants = {}
|
| 131 |
+
for name, impl, cast in (("native fp32 w", "native", False),
|
| 132 |
+
("native bf16 w", "native", True),
|
| 133 |
+
("manual", "manual", False)):
|
| 134 |
+
mod = RMSNorm(dim, impl=impl).cuda()
|
| 135 |
+
if cast:
|
| 136 |
+
mod = mod.to(torch.bfloat16)
|
| 137 |
+
|
| 138 |
+
def run(m=mod):
|
| 139 |
+
m(x).backward(grad, retain_graph=True)
|
| 140 |
+
x.grad = None
|
| 141 |
+
variants[name] = time_it(run)
|
| 142 |
+
|
| 143 |
+
try:
|
| 144 |
+
compiled = torch.compile(RMSNorm(dim).cuda())
|
| 145 |
+
|
| 146 |
+
def run_compiled():
|
| 147 |
+
compiled(x).backward(grad, retain_graph=True)
|
| 148 |
+
x.grad = None
|
| 149 |
+
variants["compiled"] = time_it(run_compiled)
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
print(f"torch.compile unavailable: {exc}")
|
| 152 |
+
|
| 153 |
+
bytes_moved = tokens * dim * 2 * 3
|
| 154 |
+
print(f"\n{tokens} x {dim} bf16, forward and backward")
|
| 155 |
+
for name, ms in sorted(variants.items(), key=lambda kv: kv[1]):
|
| 156 |
+
print(f" {name:16s} {ms:7.4f} ms {bytes_moved/ms/1e6:7.0f} GB/s")
|
| 157 |
+
print("\n memory bound, so GB/s against 1792 is the number that matters")
|
| 158 |
+
print(" pick the winner here, not the one that sounds best")
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
if __name__ == "__main__":
|
| 162 |
+
import argparse
|
| 163 |
+
parser = argparse.ArgumentParser()
|
| 164 |
+
parser.add_argument("--check", action="store_true")
|
| 165 |
+
parser.add_argument("--bench", action="store_true")
|
| 166 |
+
args = parser.parse_args()
|
| 167 |
+
if not any(vars(args).values()):
|
| 168 |
+
args.check = True
|
| 169 |
+
if args.check:
|
| 170 |
+
check()
|
| 171 |
+
if args.bench:
|
| 172 |
+
benchmark()
|
code/model/swiglu_tilelang.py
ADDED
|
@@ -0,0 +1,644 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# no "from __future__ import annotations" here: tilelang reads the prim_func
|
| 2 |
+
# signature with get_type_hints, and PEP 563 would leave T.Tensor([batch, ...])
|
| 3 |
+
# as a string that cannot see the enclosing function's locals.
|
| 4 |
+
import argparse
|
| 5 |
+
import itertools
|
| 6 |
+
import math
|
| 7 |
+
import os
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from mimans import paths # noqa: F401 points the kernel cache at D:, must precede tilelang
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
import tilelang
|
| 18 |
+
import tilelang.language as T
|
| 19 |
+
from tilelang.autotuner import autotune
|
| 20 |
+
HAVE_TILELANG = True
|
| 21 |
+
except ImportError:
|
| 22 |
+
HAVE_TILELANG = False
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
DTYPE = torch.bfloat16
|
| 26 |
+
SMEM_PER_SM_KB = {8: 164, 9: 228, 10: 228, 12: 128}
|
| 27 |
+
MAX_ACCUM_BYTES = 96 * 1024
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def shared_mem_limit(device=0, margin=0.9):
|
| 31 |
+
try:
|
| 32 |
+
if not torch.cuda.is_available():
|
| 33 |
+
return 112 * 1024
|
| 34 |
+
props = torch.cuda.get_device_properties(device)
|
| 35 |
+
for name in ("shared_memory_per_block_optin",
|
| 36 |
+
"max_shared_memory_per_block_optin",
|
| 37 |
+
"shared_memory_per_block"):
|
| 38 |
+
value = getattr(props, name, 0)
|
| 39 |
+
if value:
|
| 40 |
+
return int(value * margin)
|
| 41 |
+
return int(SMEM_PER_SM_KB.get(props.major, 100) * 1024 * margin)
|
| 42 |
+
except Exception:
|
| 43 |
+
return 112 * 1024
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
SMEM_LIMIT = shared_mem_limit()
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def gemm_smem(block_m, block_n, block_k, stages, weights=1, elem=2):
|
| 50 |
+
return (block_m * block_k * elem * stages
|
| 51 |
+
+ block_k * block_n * elem * stages * weights
|
| 52 |
+
+ block_m * block_n * elem)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def fusion_accumulator_bytes(block_m, d_model):
|
| 56 |
+
return block_m * d_model * 4
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def warps_fit(block, threads, granularity=16):
|
| 60 |
+
warps = threads // 32
|
| 61 |
+
return block % warps == 0 and (block // warps) % granularity == 0
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def gate_up_configs(limit=None):
|
| 65 |
+
limit = limit or SMEM_LIMIT
|
| 66 |
+
return [dict(block_M=bm, block_N=bn, block_K=bk, threads=t, num_stages=s)
|
| 67 |
+
for bm, bn in itertools.product((32, 64, 128, 256), repeat=2)
|
| 68 |
+
for bk in (32, 64, 128)
|
| 69 |
+
for t in (128, 256) if warps_fit(bm, t)
|
| 70 |
+
for s in (1, 2, 3)
|
| 71 |
+
if gemm_smem(bm, bn, bk, s, weights=2) <= limit
|
| 72 |
+
and bm * bn * 4 * 2 <= MAX_ACCUM_BYTES]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def tunable(configs, max_rows_per_warp=16):
|
| 76 |
+
"""Drop the configs that are valid but pathologically slow to benchmark.
|
| 77 |
+
|
| 78 |
+
warps_fit allows block_M=128 with threads=128, which gives each warp 32
|
| 79 |
+
rows of the tile. On sm_120 at these shapes those kernels are slow enough
|
| 80 |
+
that the autotuner's benchmark times out on each one, and a search that
|
| 81 |
+
should take minutes takes hours: the tuning run that stalled had cleared
|
| 82 |
+
all 19 block_M=64 configs in seconds and then hung on the first
|
| 83 |
+
block_M=128, threads=128 candidate. Every config that actually won was at
|
| 84 |
+
16 rows per warp.
|
| 85 |
+
|
| 86 |
+
This filters the search only. The defaults stay legal configs, so the
|
| 87 |
+
asserts below still hold and nothing that already works is taken away.
|
| 88 |
+
"""
|
| 89 |
+
kept = [c for c in configs
|
| 90 |
+
if c["block_M"] // max(1, c["threads"] // 32) <= max_rows_per_warp]
|
| 91 |
+
return kept or configs
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def grad_configs(limit=None):
|
| 95 |
+
limit = limit or SMEM_LIMIT
|
| 96 |
+
return [dict(block_M=bm, block_N=bn, threads=t)
|
| 97 |
+
for bm, bn in itertools.product((32, 64, 128, 256), repeat=2)
|
| 98 |
+
for t in (128, 256) if warps_fit(bm, t)
|
| 99 |
+
if 5 * bm * bn * 2 <= limit]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
AUTOTUNE = True
|
| 103 |
+
GATE_UP_TILES = dict(block_M=128, block_N=64, block_K=64, threads=128, num_stages=2)
|
| 104 |
+
GRAD_TILES = dict(block_M=128, block_N=64, threads=128)
|
| 105 |
+
|
| 106 |
+
assert GATE_UP_TILES in gate_up_configs(), "gate_up default is not a tunable config"
|
| 107 |
+
assert GRAD_TILES in grad_configs(), "grad default is not a tunable config"
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
if HAVE_TILELANG:
|
| 111 |
+
FAST_MATH = {tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True}
|
| 112 |
+
# CUDA 13.1's host_config.h only blesses MSVC 2019-2022 and this box has
|
| 113 |
+
# VS 18, so nvcc refuses every kernel. Override it here; unset
|
| 114 |
+
# TL_NVCC_FLAGS (or install the 2022 build tools) to get the check back.
|
| 115 |
+
JIT_FLAGS = os.environ.get("TL_NVCC_FLAGS", "-allow-unsupported-compiler").split()
|
| 116 |
+
GATE_UP_SPACE = tunable(gate_up_configs())
|
| 117 |
+
GRAD_SPACE = tunable(grad_configs())
|
| 118 |
+
|
| 119 |
+
@autotune(configs=GATE_UP_SPACE, warmup=10, rep=20)
|
| 120 |
+
@tilelang.jit(out_idx=[3], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 121 |
+
def gate_up_swiglu(tokens, d_model, ffn,
|
| 122 |
+
block_M=128, block_N=64, block_K=64,
|
| 123 |
+
threads=128, num_stages=2):
|
| 124 |
+
dt, acc = T.bfloat16, T.float32
|
| 125 |
+
|
| 126 |
+
@T.prim_func
|
| 127 |
+
def kernel(
|
| 128 |
+
X: T.Tensor([tokens, d_model], dt), #type: ignore
|
| 129 |
+
Wg: T.Tensor([d_model, ffn], dt), #type: ignore
|
| 130 |
+
Wu: T.Tensor([d_model, ffn], dt), #type: ignore
|
| 131 |
+
Act: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 132 |
+
):
|
| 133 |
+
with T.Kernel(T.ceildiv(ffn, block_N), T.ceildiv(tokens, block_M), threads=threads) as (bx, by):
|
| 134 |
+
x_s = T.alloc_shared([block_M, block_K], dt)
|
| 135 |
+
wg_s = T.alloc_shared([block_K, block_N], dt)
|
| 136 |
+
wu_s = T.alloc_shared([block_K, block_N], dt)
|
| 137 |
+
out_s = T.alloc_shared([block_M, block_N], dt)
|
| 138 |
+
gate = T.alloc_fragment([block_M, block_N], acc)
|
| 139 |
+
up = T.alloc_fragment([block_M, block_N], acc)
|
| 140 |
+
act = T.alloc_fragment([block_M, block_N], dt)
|
| 141 |
+
|
| 142 |
+
T.clear(gate)
|
| 143 |
+
T.clear(up)
|
| 144 |
+
for k in T.Pipelined(T.ceildiv(d_model, block_K), num_stages=num_stages):
|
| 145 |
+
T.copy(X[by * block_M:(by + 1) * block_M, k * block_K:(k + 1) * block_K], x_s)
|
| 146 |
+
T.copy(Wg[k * block_K:(k + 1) * block_K, bx * block_N:(bx + 1) * block_N], wg_s)
|
| 147 |
+
T.copy(Wu[k * block_K:(k + 1) * block_K, bx * block_N:(bx + 1) * block_N], wu_s)
|
| 148 |
+
T.gemm(x_s, wg_s, gate, policy=T.GemmWarpPolicy.FullRow)
|
| 149 |
+
T.gemm(x_s, wu_s, up, policy=T.GemmWarpPolicy.FullRow)
|
| 150 |
+
|
| 151 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 152 |
+
act[i, j] = T.Cast(dt, gate[i, j] / (1.0 + T.exp(-gate[i, j])) * up[i, j])
|
| 153 |
+
T.copy(act, out_s)
|
| 154 |
+
T.copy(out_s, Act[by * block_M:(by + 1) * block_M, bx * block_N:(bx + 1) * block_N])
|
| 155 |
+
|
| 156 |
+
return kernel
|
| 157 |
+
|
| 158 |
+
@autotune(configs=GATE_UP_SPACE, warmup=10, rep=20)
|
| 159 |
+
@tilelang.jit(out_idx=[3, 4], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 160 |
+
def gate_up_split(tokens, d_model, ffn,
|
| 161 |
+
block_M=128, block_N=64, block_K=64,
|
| 162 |
+
threads=128, num_stages=2):
|
| 163 |
+
dt, acc = T.bfloat16, T.float32
|
| 164 |
+
|
| 165 |
+
@T.prim_func
|
| 166 |
+
def kernel(
|
| 167 |
+
X: T.Tensor([tokens, d_model], dt), #type: ignore
|
| 168 |
+
Wg: T.Tensor([d_model, ffn], dt), #type: ignore
|
| 169 |
+
Wu: T.Tensor([d_model, ffn], dt), #type: ignore
|
| 170 |
+
Gate: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 171 |
+
Up: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 172 |
+
):
|
| 173 |
+
with T.Kernel(T.ceildiv(ffn, block_N), T.ceildiv(tokens, block_M), threads=threads) as (bx, by):
|
| 174 |
+
x_s = T.alloc_shared([block_M, block_K], dt)
|
| 175 |
+
wg_s = T.alloc_shared([block_K, block_N], dt)
|
| 176 |
+
wu_s = T.alloc_shared([block_K, block_N], dt)
|
| 177 |
+
tile = T.alloc_shared([block_M, block_N], dt)
|
| 178 |
+
gate = T.alloc_fragment([block_M, block_N], acc)
|
| 179 |
+
up = T.alloc_fragment([block_M, block_N], acc)
|
| 180 |
+
cast = T.alloc_fragment([block_M, block_N], dt)
|
| 181 |
+
|
| 182 |
+
T.clear(gate)
|
| 183 |
+
T.clear(up)
|
| 184 |
+
for k in T.Pipelined(T.ceildiv(d_model, block_K), num_stages=num_stages):
|
| 185 |
+
T.copy(X[by * block_M:(by + 1) * block_M, k * block_K:(k + 1) * block_K], x_s)
|
| 186 |
+
T.copy(Wg[k * block_K:(k + 1) * block_K, bx * block_N:(bx + 1) * block_N], wg_s)
|
| 187 |
+
T.copy(Wu[k * block_K:(k + 1) * block_K, bx * block_N:(bx + 1) * block_N], wu_s)
|
| 188 |
+
T.gemm(x_s, wg_s, gate, policy=T.GemmWarpPolicy.FullRow)
|
| 189 |
+
T.gemm(x_s, wu_s, up, policy=T.GemmWarpPolicy.FullRow)
|
| 190 |
+
|
| 191 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 192 |
+
cast[i, j] = T.Cast(dt, gate[i, j])
|
| 193 |
+
T.copy(cast, tile)
|
| 194 |
+
T.copy(tile, Gate[by * block_M:(by + 1) * block_M, bx * block_N:(bx + 1) * block_N])
|
| 195 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 196 |
+
cast[i, j] = T.Cast(dt, up[i, j])
|
| 197 |
+
T.copy(cast, tile)
|
| 198 |
+
T.copy(tile, Up[by * block_M:(by + 1) * block_M, bx * block_N:(bx + 1) * block_N])
|
| 199 |
+
|
| 200 |
+
return kernel
|
| 201 |
+
|
| 202 |
+
@autotune(configs=GRAD_SPACE, warmup=10, rep=20)
|
| 203 |
+
@tilelang.jit(out_idx=[3, 4], pass_configs=FAST_MATH, compile_flags=JIT_FLAGS)
|
| 204 |
+
def swiglu_grad(tokens, ffn, block_M=128, block_N=64, threads=128):
|
| 205 |
+
dt, acc = T.bfloat16, T.float32
|
| 206 |
+
|
| 207 |
+
@T.prim_func
|
| 208 |
+
def kernel(
|
| 209 |
+
dAct: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 210 |
+
Gate: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 211 |
+
Up: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 212 |
+
dGate: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 213 |
+
dUp: T.Tensor([tokens, ffn], dt), #type: ignore
|
| 214 |
+
):
|
| 215 |
+
with T.Kernel(T.ceildiv(ffn, block_N), T.ceildiv(tokens, block_M), threads=threads) as (bx, by):
|
| 216 |
+
da = T.alloc_fragment([block_M, block_N], acc)
|
| 217 |
+
g = T.alloc_fragment([block_M, block_N], acc)
|
| 218 |
+
u = T.alloc_fragment([block_M, block_N], acc)
|
| 219 |
+
sig = T.alloc_fragment([block_M, block_N], acc)
|
| 220 |
+
out = T.alloc_fragment([block_M, block_N], dt)
|
| 221 |
+
tile = T.alloc_shared([block_M, block_N], dt)
|
| 222 |
+
|
| 223 |
+
rows = slice(by * block_M, (by + 1) * block_M)
|
| 224 |
+
cols = slice(bx * block_N, (bx + 1) * block_N)
|
| 225 |
+
T.copy(dAct[rows, cols], da)
|
| 226 |
+
T.copy(Gate[rows, cols], g)
|
| 227 |
+
T.copy(Up[rows, cols], u)
|
| 228 |
+
|
| 229 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 230 |
+
sig[i, j] = 1.0 / (1.0 + T.exp(-g[i, j]))
|
| 231 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 232 |
+
out[i, j] = T.Cast( dt,da[i, j] * u[i, j] * sig[i, j] * (1.0 + g[i, j] * (1.0 - sig[i, j])))
|
| 233 |
+
T.copy(out, tile)
|
| 234 |
+
T.copy(tile, dGate[rows, cols])
|
| 235 |
+
for i, j in T.Parallel(block_M, block_N):
|
| 236 |
+
out[i, j] = T.Cast(dt, da[i, j] * g[i, j] * sig[i, j])
|
| 237 |
+
T.copy(out, tile)
|
| 238 |
+
T.copy(tile, dUp[rows, cols])
|
| 239 |
+
|
| 240 |
+
return kernel
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
_kernels: dict = {}
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def get_kernel(kind, *shape):
|
| 247 |
+
key = (kind, shape, AUTOTUNE)
|
| 248 |
+
if key not in _kernels:
|
| 249 |
+
from mimans.model import tuned
|
| 250 |
+
|
| 251 |
+
fn = {"gate_up": gate_up_swiglu,
|
| 252 |
+
"split": gate_up_split,
|
| 253 |
+
"grad": swiglu_grad}[kind]
|
| 254 |
+
name = f"swiglu_{kind}"
|
| 255 |
+
remembered = tuned.lookup(name, shape)
|
| 256 |
+
if remembered is not None:
|
| 257 |
+
kernel = fn(*shape, **remembered)
|
| 258 |
+
elif AUTOTUNE:
|
| 259 |
+
kernel = fn(*shape)
|
| 260 |
+
tuned.record(name, shape, tuned.config_of(kernel))
|
| 261 |
+
else:
|
| 262 |
+
kernel = fn(*shape, **(GRAD_TILES if kind == "grad" else GATE_UP_TILES))
|
| 263 |
+
_kernels[key] = kernel
|
| 264 |
+
return _kernels[key]
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def warmup(tokens, cfg=None):
|
| 268 |
+
"""Run the autotune searches up front so training never stalls mid-step."""
|
| 269 |
+
cfg = cfg or FFNConfig()
|
| 270 |
+
print(f"tuning gate_up over {len(GATE_UP_SPACE)} configs")
|
| 271 |
+
get_kernel("gate_up", tokens, cfg.d_model, cfg.ffn_hidden)
|
| 272 |
+
get_kernel("split", tokens, cfg.d_model, cfg.ffn_hidden)
|
| 273 |
+
print(f"tuning grad over {len(GRAD_SPACE)} configs")
|
| 274 |
+
get_kernel("grad", tokens, cfg.ffn_hidden)
|
| 275 |
+
print("cached to disk")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
class FusedSwiGLU(torch.autograd.Function):
|
| 279 |
+
@staticmethod
|
| 280 |
+
def forward(ctx, x, w_gate, w_up, w_down):
|
| 281 |
+
tokens, d_model = x.shape
|
| 282 |
+
ffn = w_gate.shape[1]
|
| 283 |
+
gate, up = get_kernel("split", tokens, d_model, ffn)(x, w_gate, w_up)
|
| 284 |
+
act = F.silu(gate) * up
|
| 285 |
+
out = act @ w_down
|
| 286 |
+
ctx.save_for_backward(x, w_gate, w_up, w_down, gate, up, act)
|
| 287 |
+
return out
|
| 288 |
+
|
| 289 |
+
@staticmethod
|
| 290 |
+
def backward(ctx, grad_out):
|
| 291 |
+
x, w_gate, w_up, w_down, gate, up, act = ctx.saved_tensors
|
| 292 |
+
tokens, ffn = gate.shape
|
| 293 |
+
|
| 294 |
+
d_act = grad_out @ w_down.t()
|
| 295 |
+
d_gate, d_up = get_kernel("grad", tokens, ffn)(d_act, gate, up)
|
| 296 |
+
|
| 297 |
+
d_x = d_gate @ w_gate.t() + d_up @ w_up.t()
|
| 298 |
+
d_w_gate = x.t() @ d_gate
|
| 299 |
+
d_w_up = x.t() @ d_up
|
| 300 |
+
d_w_down = act.t() @ grad_out
|
| 301 |
+
return d_x, d_w_gate, d_w_up, d_w_down
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def fused_swiglu(x, w_gate, w_up, w_down):
|
| 305 |
+
return FusedSwiGLU.apply(x, w_gate, w_up, w_down)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
@dataclass
|
| 309 |
+
class FFNConfig:
|
| 310 |
+
d_model: int = 1280
|
| 311 |
+
ffn_hidden: int = 3328
|
| 312 |
+
backend: str = "tilelang"
|
| 313 |
+
|
| 314 |
+
def __post_init__(self):
|
| 315 |
+
assert self.ffn_hidden % 128 == 0
|
| 316 |
+
assert self.d_model % 128 == 0
|
| 317 |
+
|
| 318 |
+
@property
|
| 319 |
+
def params(self):
|
| 320 |
+
return 3 * self.d_model * self.ffn_hidden
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
class SwiGLU(nn.Module):
|
| 324 |
+
def __init__(self, cfg: FFNConfig):
|
| 325 |
+
super().__init__()
|
| 326 |
+
self.cfg = cfg
|
| 327 |
+
self.w_gate = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
|
| 328 |
+
self.w_up = nn.Parameter(torch.empty(cfg.d_model, cfg.ffn_hidden))
|
| 329 |
+
self.w_down = nn.Parameter(torch.empty(cfg.ffn_hidden, cfg.d_model))
|
| 330 |
+
std = 0.02
|
| 331 |
+
nn.init.normal_(self.w_gate, std=std)
|
| 332 |
+
nn.init.normal_(self.w_up, std=std)
|
| 333 |
+
nn.init.normal_(self.w_down, std=std / math.sqrt(2 * 27))
|
| 334 |
+
|
| 335 |
+
def forward(self, x, backend=None):
|
| 336 |
+
cfg = self.cfg
|
| 337 |
+
backend = backend or cfg.backend
|
| 338 |
+
shape = x.shape
|
| 339 |
+
flat = x.reshape(-1, cfg.d_model)
|
| 340 |
+
|
| 341 |
+
if backend == "tilelang":
|
| 342 |
+
# The kernels are compiled for bf16. Autocast does not reach in
|
| 343 |
+
# here: the residual stream stays fp32 because the embedding is not
|
| 344 |
+
# an autocast op, and these are raw nn.Parameters rather than an
|
| 345 |
+
# nn.Linear, so nothing casts them for us.
|
| 346 |
+
out = fused_swiglu(flat.to(DTYPE), self.w_gate.to(DTYPE),
|
| 347 |
+
self.w_up.to(DTYPE), self.w_down.to(DTYPE))
|
| 348 |
+
out = out.to(x.dtype)
|
| 349 |
+
elif backend == "torch":
|
| 350 |
+
out = (F.silu(flat @ self.w_gate) * (flat @ self.w_up)) @ self.w_down
|
| 351 |
+
elif backend == "math":
|
| 352 |
+
gate = flat.float() @ self.w_gate.float()
|
| 353 |
+
up = flat.float() @ self.w_up.float()
|
| 354 |
+
out = ((gate * torch.sigmoid(gate)) * up) @ self.w_down.float()
|
| 355 |
+
out = out.to(x.dtype)
|
| 356 |
+
else:
|
| 357 |
+
raise ValueError(f"unknown backend {backend!r}")
|
| 358 |
+
|
| 359 |
+
return out.reshape(shape)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def reference(x, w_gate, w_up, w_down):
|
| 363 |
+
gate = x.float() @ w_gate.float()
|
| 364 |
+
up = x.float() @ w_up.float()
|
| 365 |
+
return ((gate * torch.sigmoid(gate)) * up) @ w_down.float()
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def check_math():
|
| 369 |
+
passed = True
|
| 370 |
+
|
| 371 |
+
def check(label, condition):
|
| 372 |
+
nonlocal passed
|
| 373 |
+
passed &= bool(condition)
|
| 374 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 375 |
+
|
| 376 |
+
torch.manual_seed(0)
|
| 377 |
+
cfg = FFNConfig()
|
| 378 |
+
kb = 1024
|
| 379 |
+
layers, tokens = 27, 8 * 2048
|
| 380 |
+
|
| 381 |
+
print("full fusion feasibility")
|
| 382 |
+
for bm in (16, 32, 64, 128):
|
| 383 |
+
size = fusion_accumulator_bytes(bm, cfg.d_model)
|
| 384 |
+
print(f" block_M={bm:3d} down accumulator fp32 {size/kb:7.1f} KB"
|
| 385 |
+
f" {'fits' if size <= SMEM_LIMIT else 'overflow'}")
|
| 386 |
+
check("full fusion needs block_M <= 16, too small for the tensor cores",
|
| 387 |
+
fusion_accumulator_bytes(32, cfg.d_model) > SMEM_LIMIT)
|
| 388 |
+
|
| 389 |
+
print("\nhbm traffic, one ffn forward at 8x2048 tokens")
|
| 390 |
+
unfused = 6 * tokens * cfg.ffn_hidden * 2
|
| 391 |
+
fused = 2 * tokens * cfg.ffn_hidden * 2
|
| 392 |
+
print(f" unfused {unfused/1e6:7.1f} MB")
|
| 393 |
+
print(f" fused {fused/1e6:7.1f} MB")
|
| 394 |
+
print(f" saved {(unfused-fused)/1e6:7.1f} MB per layer,"
|
| 395 |
+
f" {(unfused-fused)*layers/1e9:.2f} GB over {layers} layers")
|
| 396 |
+
print(f" at 1792 GB/s that is"
|
| 397 |
+
f" {(unfused-fused)*layers/1792e9*1e3:.2f} ms per forward")
|
| 398 |
+
check("fusion removes two thirds of the intermediate traffic",
|
| 399 |
+
fused * 3 == unfused)
|
| 400 |
+
|
| 401 |
+
print("\nconfig spaces")
|
| 402 |
+
gu, gr = gate_up_configs(), grad_configs()
|
| 403 |
+
print(f" gate_up {len(gu):3d} grad {len(gr):3d}")
|
| 404 |
+
check("gate_up default is tunable", GATE_UP_TILES in gu)
|
| 405 |
+
check("grad default is tunable", GRAD_TILES in gr)
|
| 406 |
+
check("every gate_up config fits shared memory",
|
| 407 |
+
all(gemm_smem(c["block_M"], c["block_N"], c["block_K"],
|
| 408 |
+
c["num_stages"], weights=2) <= SMEM_LIMIT for c in gu))
|
| 409 |
+
check("every gate_up config fits the register file",
|
| 410 |
+
all(c["block_M"] * c["block_N"] * 8 <= MAX_ACCUM_BYTES for c in gu))
|
| 411 |
+
|
| 412 |
+
print("\nsilu derivative")
|
| 413 |
+
z = torch.randn(2000, dtype=torch.float64) * 3
|
| 414 |
+
z.requires_grad_(True)
|
| 415 |
+
F.silu(z).sum().backward()
|
| 416 |
+
sig = torch.sigmoid(z.detach())
|
| 417 |
+
analytic = sig * (1 + z.detach() * (1 - sig))
|
| 418 |
+
err = (z.grad - analytic).abs().max().item()
|
| 419 |
+
check(f"matches sigmoid(z)*(1+z*(1-sigmoid(z))) ({err:.2e})", err < 1e-12)
|
| 420 |
+
check("silu(0)=0 and dsilu(0)=0.5",
|
| 421 |
+
abs(F.silu(torch.zeros(1)).item()) < 1e-12
|
| 422 |
+
and abs(torch.sigmoid(torch.zeros(1)).item() - 0.5) < 1e-12)
|
| 423 |
+
|
| 424 |
+
print("\ngradients against autograd")
|
| 425 |
+
m, d, h = 32, 64, 96
|
| 426 |
+
x = torch.randn(m, d, dtype=torch.float64, requires_grad=True)
|
| 427 |
+
wg = torch.randn(d, h, dtype=torch.float64, requires_grad=True) * 0.1
|
| 428 |
+
wu = torch.randn(d, h, dtype=torch.float64, requires_grad=True) * 0.1
|
| 429 |
+
wd = torch.randn(h, d, dtype=torch.float64, requires_grad=True) * 0.1
|
| 430 |
+
for t in (wg, wu, wd):
|
| 431 |
+
t.retain_grad()
|
| 432 |
+
grad_out = torch.randn(m, d, dtype=torch.float64)
|
| 433 |
+
|
| 434 |
+
out = (F.silu(x @ wg) * (x @ wu)) @ wd
|
| 435 |
+
out.backward(grad_out)
|
| 436 |
+
auto = [x.grad.clone(), wg.grad.clone(), wu.grad.clone(), wd.grad.clone()]
|
| 437 |
+
|
| 438 |
+
gate = (x @ wg).detach()
|
| 439 |
+
up = (x @ wu).detach()
|
| 440 |
+
act = F.silu(gate) * up
|
| 441 |
+
d_act = grad_out @ wd.detach().t()
|
| 442 |
+
sig = torch.sigmoid(gate)
|
| 443 |
+
d_gate = d_act * up * sig * (1 + gate * (1 - sig))
|
| 444 |
+
d_up = d_act * F.silu(gate)
|
| 445 |
+
manual = [d_gate @ wg.detach().t() + d_up @ wu.detach().t(),
|
| 446 |
+
x.detach().t() @ d_gate,
|
| 447 |
+
x.detach().t() @ d_up,
|
| 448 |
+
act.t() @ grad_out]
|
| 449 |
+
for name, a, b in zip(("d_x", "d_w_gate", "d_w_up", "d_w_down"), manual, auto):
|
| 450 |
+
rel = ((a - b).abs().max() / b.abs().max()).item()
|
| 451 |
+
check(f"{name:9s} rel err {rel:.2e}", rel < 1e-10)
|
| 452 |
+
|
| 453 |
+
print("\nmodule")
|
| 454 |
+
ffn = SwiGLU(cfg)
|
| 455 |
+
n = sum(p.numel() for p in ffn.parameters())
|
| 456 |
+
check(f"parameters {n:,} == 3*{cfg.d_model}*{cfg.ffn_hidden}", n == cfg.params)
|
| 457 |
+
print(f" {cfg.params/1e6:.2f}M per layer, {cfg.params*layers/1e6:.1f}M"
|
| 458 |
+
f" over {layers} layers")
|
| 459 |
+
check("ffn is 76 percent of a transformer block",
|
| 460 |
+
abs(cfg.params / (cfg.params + 3_932_160) - 0.765) < 0.01)
|
| 461 |
+
|
| 462 |
+
x = torch.randn(2, 16, cfg.d_model)
|
| 463 |
+
y = ffn(x, backend="torch")
|
| 464 |
+
check(f"shape {tuple(x.shape)} -> {tuple(y.shape)}", y.shape == x.shape)
|
| 465 |
+
err = (y - ffn(x, backend="math")).abs().max().item()
|
| 466 |
+
check(f"torch and fp32 reference agree ({err:.2e})", err < 1e-4)
|
| 467 |
+
|
| 468 |
+
x = torch.randn(2, 8, cfg.d_model, requires_grad=True)
|
| 469 |
+
ffn(x, backend="torch").sum().backward()
|
| 470 |
+
check("finite gradients everywhere",
|
| 471 |
+
torch.isfinite(x.grad).all()
|
| 472 |
+
and all(torch.isfinite(p.grad).all() for p in ffn.parameters()))
|
| 473 |
+
|
| 474 |
+
print("\nactivation storage")
|
| 475 |
+
store = 2 * tokens * cfg.ffn_hidden * 2
|
| 476 |
+
print(f" gate+up for backward {store/1e6:.1f} MB per layer,"
|
| 477 |
+
f" {store*layers/1e9:.2f} GB over {layers}")
|
| 478 |
+
recompute = 2 * tokens * cfg.d_model * 2 * cfg.ffn_hidden
|
| 479 |
+
print(f" recomputing instead costs {recompute/1e9:.1f} GFLOP per layer"
|
| 480 |
+
f" per backward")
|
| 481 |
+
check("storing gate+up exceeds the 9 GB of headroom in the memory budget",
|
| 482 |
+
store * layers > 5e9)
|
| 483 |
+
|
| 484 |
+
print("\n" + ("all math checks passed" if passed else "FAILURES ABOVE"))
|
| 485 |
+
print("still needs a GPU: kernel compilation, fused output vs reference, tuning")
|
| 486 |
+
return passed
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def check_gpu(tokens=4096):
|
| 490 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 491 |
+
print("needs tilelang and cuda")
|
| 492 |
+
return
|
| 493 |
+
major, minor = torch.cuda.get_device_capability()
|
| 494 |
+
print(f"{torch.cuda.get_device_name()} sm_{major}{minor}")
|
| 495 |
+
print(f"shared memory limit in use: {SMEM_LIMIT/1024:.0f} KB")
|
| 496 |
+
|
| 497 |
+
cfg = FFNConfig()
|
| 498 |
+
torch.manual_seed(0)
|
| 499 |
+
x = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 500 |
+
wg = torch.randn(cfg.d_model, cfg.ffn_hidden, device="cuda", dtype=DTYPE,
|
| 501 |
+
requires_grad=True) * 0.02
|
| 502 |
+
wu = torch.randn(cfg.d_model, cfg.ffn_hidden, device="cuda", dtype=DTYPE,
|
| 503 |
+
requires_grad=True) * 0.02
|
| 504 |
+
wd = torch.randn(cfg.ffn_hidden, cfg.d_model, device="cuda", dtype=DTYPE,
|
| 505 |
+
requires_grad=True) * 0.02
|
| 506 |
+
for t in (x, wg, wu, wd):
|
| 507 |
+
t.retain_grad()
|
| 508 |
+
grad_out = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE)
|
| 509 |
+
|
| 510 |
+
act = get_kernel("gate_up", tokens, cfg.d_model, cfg.ffn_hidden)(x, wg, wu)
|
| 511 |
+
want_act = F.silu(x.float() @ wg.float()) * (x.float() @ wu.float())
|
| 512 |
+
err = (act.float() - want_act).abs().max().item()
|
| 513 |
+
scale = want_act.abs().max().item()
|
| 514 |
+
print(f"\nfused activation vs fp32: max err {err:.3e}"
|
| 515 |
+
f" (rel {err/scale:.3e}) {'ok' if err/scale < 2e-2 else 'FAIL'}")
|
| 516 |
+
|
| 517 |
+
out = fused_swiglu(x, wg, wu, wd)
|
| 518 |
+
out.backward(grad_out, retain_graph=True)
|
| 519 |
+
mine = [t.grad.clone() for t in (x, wg, wu, wd)]
|
| 520 |
+
for t in (x, wg, wu, wd):
|
| 521 |
+
t.grad = None
|
| 522 |
+
|
| 523 |
+
want = reference(x, wg, wu, wd).to(DTYPE)
|
| 524 |
+
want.backward(grad_out, retain_graph=True)
|
| 525 |
+
theirs = [t.grad.clone() for t in (x, wg, wu, wd)]
|
| 526 |
+
for t in (x, wg, wu, wd):
|
| 527 |
+
t.grad = None
|
| 528 |
+
|
| 529 |
+
print()
|
| 530 |
+
for name, a, b in zip(("out", "d_x", "d_w_gate", "d_w_up", "d_w_down"),
|
| 531 |
+
[out] + mine, [want] + theirs):
|
| 532 |
+
rel = ((a.float() - b.float()).abs().max()
|
| 533 |
+
/ b.float().abs().max().clamp(min=1e-6)).item()
|
| 534 |
+
print(f" {name:9s} rel err {rel:.3e} {'ok' if rel < 3e-2 else 'FAIL'}")
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
def benchmark(tokens=16384):
|
| 538 |
+
if not torch.cuda.is_available():
|
| 539 |
+
print("needs cuda")
|
| 540 |
+
return
|
| 541 |
+
cfg = FFNConfig()
|
| 542 |
+
x = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE, requires_grad=True)
|
| 543 |
+
wg = torch.randn(cfg.d_model, cfg.ffn_hidden, device="cuda", dtype=DTYPE,
|
| 544 |
+
requires_grad=True)
|
| 545 |
+
wu = torch.randn(cfg.d_model, cfg.ffn_hidden, device="cuda", dtype=DTYPE,
|
| 546 |
+
requires_grad=True)
|
| 547 |
+
wd = torch.randn(cfg.ffn_hidden, cfg.d_model, device="cuda", dtype=DTYPE,
|
| 548 |
+
requires_grad=True)
|
| 549 |
+
grad_out = torch.randn(tokens, cfg.d_model, device="cuda", dtype=DTYPE)
|
| 550 |
+
flops = 2.0 * tokens * cfg.d_model * cfg.ffn_hidden * 3 * 3
|
| 551 |
+
|
| 552 |
+
def time_it(fn, iters=30):
|
| 553 |
+
for _ in range(10):
|
| 554 |
+
fn()
|
| 555 |
+
torch.cuda.synchronize()
|
| 556 |
+
start, end = torch.cuda.Event(True), torch.cuda.Event(True)
|
| 557 |
+
start.record()
|
| 558 |
+
for _ in range(iters):
|
| 559 |
+
fn()
|
| 560 |
+
end.record()
|
| 561 |
+
torch.cuda.synchronize()
|
| 562 |
+
return start.elapsed_time(end) / iters
|
| 563 |
+
|
| 564 |
+
def clear():
|
| 565 |
+
for t in (x, wg, wu, wd):
|
| 566 |
+
t.grad = None
|
| 567 |
+
|
| 568 |
+
results = {}
|
| 569 |
+
|
| 570 |
+
def eager():
|
| 571 |
+
((F.silu(x @ wg) * (x @ wu)) @ wd).backward(grad_out, retain_graph=True)
|
| 572 |
+
clear()
|
| 573 |
+
results["eager"] = time_it(eager)
|
| 574 |
+
|
| 575 |
+
try:
|
| 576 |
+
compiled = torch.compile(lambda a, g, u, d: (F.silu(a @ g) * (a @ u)) @ d)
|
| 577 |
+
|
| 578 |
+
def run_compiled():
|
| 579 |
+
compiled(x, wg, wu, wd).backward(grad_out, retain_graph=True)
|
| 580 |
+
clear()
|
| 581 |
+
results["compiled"] = time_it(run_compiled)
|
| 582 |
+
except Exception as exc:
|
| 583 |
+
print(f"torch.compile unavailable: {exc}")
|
| 584 |
+
|
| 585 |
+
if HAVE_TILELANG:
|
| 586 |
+
try:
|
| 587 |
+
def run_tilelang():
|
| 588 |
+
fused_swiglu(x, wg, wu, wd).backward(grad_out, retain_graph=True)
|
| 589 |
+
clear()
|
| 590 |
+
results["tilelang"] = time_it(run_tilelang)
|
| 591 |
+
except Exception as exc:
|
| 592 |
+
print(f"tilelang failed: {type(exc).__name__}: {exc}")
|
| 593 |
+
|
| 594 |
+
print()
|
| 595 |
+
for name, ms in sorted(results.items(), key=lambda kv: kv[1]):
|
| 596 |
+
print(f" {name:10s} {ms:8.3f} ms {flops/ms*1e-9:7.1f} TFLOP/s")
|
| 597 |
+
|
| 598 |
+
if "tilelang" in results and len(results) > 1:
|
| 599 |
+
best_other = min(v for n, v in results.items() if n != "tilelang")
|
| 600 |
+
speedup = best_other / results["tilelang"]
|
| 601 |
+
overall = 1 / (0.235 + 0.765 / speedup)
|
| 602 |
+
print(f"\n ffn {speedup:.2f}x, whole block {overall:.3f}x")
|
| 603 |
+
print(f" ffn is 76 percent of block parameters, so that is the weight")
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def tune(tokens=16384):
|
| 607 |
+
print(f"gate_up candidates {len(gate_up_configs())}")
|
| 608 |
+
print(f"grad candidates {len(grad_configs())}")
|
| 609 |
+
print(f"shared memory limit {SMEM_LIMIT/1024:.0f} KB")
|
| 610 |
+
if not (HAVE_TILELANG and torch.cuda.is_available()):
|
| 611 |
+
for name, space in (("gate_up", gate_up_configs()), ("grad", grad_configs())):
|
| 612 |
+
print(f"\n{name}")
|
| 613 |
+
for c in space[:12]:
|
| 614 |
+
print(f" {c}")
|
| 615 |
+
return
|
| 616 |
+
cfg = FFNConfig()
|
| 617 |
+
warmup(tokens, cfg)
|
| 618 |
+
for kind, shape in (("gate_up", (tokens, cfg.d_model, cfg.ffn_hidden)),
|
| 619 |
+
("split", (tokens, cfg.d_model, cfg.ffn_hidden)),
|
| 620 |
+
("grad", (tokens, cfg.ffn_hidden))):
|
| 621 |
+
kernel = get_kernel(kind, *shape)
|
| 622 |
+
print(f"\n{kind}: {getattr(kernel, 'config', None)}")
|
| 623 |
+
latency = getattr(kernel, "latency", None)
|
| 624 |
+
if latency is not None:
|
| 625 |
+
print(f" {latency:.3f} ms")
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
if __name__ == "__main__":
|
| 629 |
+
parser = argparse.ArgumentParser()
|
| 630 |
+
parser.add_argument("--check", action="store_true")
|
| 631 |
+
parser.add_argument("--gpu", action="store_true")
|
| 632 |
+
parser.add_argument("--bench", action="store_true")
|
| 633 |
+
parser.add_argument("--tune", action="store_true")
|
| 634 |
+
args = parser.parse_args()
|
| 635 |
+
if not any(vars(args).values()):
|
| 636 |
+
args.check = True
|
| 637 |
+
if args.check:
|
| 638 |
+
check_math()
|
| 639 |
+
if args.gpu:
|
| 640 |
+
check_gpu()
|
| 641 |
+
if args.bench:
|
| 642 |
+
benchmark()
|
| 643 |
+
if args.tune:
|
| 644 |
+
tune()
|
code/model/tuned.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Remembered autotune results, so the search happens once and never again.
|
| 2 |
+
|
| 3 |
+
tilelang can autotune every kernel against the machine it is running on, which
|
| 4 |
+
is worth doing -- but its own on-disk cache does not work here (tvm wants a
|
| 5 |
+
target triple from a host compiler that is not installed), so left alone it
|
| 6 |
+
re-derives the same answer on every process start. That is over an hour before
|
| 7 |
+
the first token of a session is trained.
|
| 8 |
+
|
| 9 |
+
The answer only depends on the GPU and the kernel shape, and neither changes
|
| 10 |
+
between sessions, so it is recorded here instead: a plain JSON file mapping
|
| 11 |
+
(gpu, kernel, shape) to the winning tile configuration. Kernels consult it
|
| 12 |
+
before tuning. A hit means the tuned tiles are used with no search at all; a
|
| 13 |
+
miss falls back to the module defaults unless tuning was explicitly asked for.
|
| 14 |
+
|
| 15 |
+
Config lives as data rather than as edited source so that re-tuning is a
|
| 16 |
+
command rather than a patch, and so a different GPU simply misses instead of
|
| 17 |
+
silently using tiles picked for hardware you no longer have.
|
| 18 |
+
|
| 19 |
+
python -m mimans.training.tune # fill it in, once
|
| 20 |
+
python -m mimans.model.tuned # show what is remembered
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import json
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
import torch
|
| 28 |
+
|
| 29 |
+
from mimans import paths
|
| 30 |
+
|
| 31 |
+
STORE = paths.CACHE / "tuned_kernels.json"
|
| 32 |
+
|
| 33 |
+
_cache = None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def device_key():
|
| 37 |
+
"""Tiles are only valid for the card they were measured on."""
|
| 38 |
+
if not torch.cuda.is_available():
|
| 39 |
+
return "no-cuda"
|
| 40 |
+
name = torch.cuda.get_device_name().replace(" ", "-")
|
| 41 |
+
major, minor = torch.cuda.get_device_capability()
|
| 42 |
+
return f"{name}-sm{major}{minor}"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _shape_key(kernel: str, shape) -> str:
|
| 46 |
+
return f"{kernel}:{','.join(str(s) for s in shape)}"
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _load():
|
| 50 |
+
global _cache
|
| 51 |
+
if _cache is None:
|
| 52 |
+
try:
|
| 53 |
+
_cache = json.loads(STORE.read_text())
|
| 54 |
+
except (OSError, json.JSONDecodeError):
|
| 55 |
+
_cache = {}
|
| 56 |
+
return _cache
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def lookup(kernel: str, shape):
|
| 60 |
+
"""The remembered config for this kernel and shape, or None."""
|
| 61 |
+
return _load().get(device_key(), {}).get(_shape_key(kernel, shape))
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def record(kernel: str, shape, config: dict):
|
| 65 |
+
"""Remember a tuned config. Written immediately, so a tuning run that is
|
| 66 |
+
interrupted keeps whatever it had already measured."""
|
| 67 |
+
if not config:
|
| 68 |
+
return
|
| 69 |
+
store = _load()
|
| 70 |
+
store.setdefault(device_key(), {})[_shape_key(kernel, shape)] = dict(config)
|
| 71 |
+
STORE.parent.mkdir(parents=True, exist_ok=True)
|
| 72 |
+
tmp = STORE.with_suffix(".json.tmp")
|
| 73 |
+
tmp.write_text(json.dumps(store, indent=2, sort_keys=True))
|
| 74 |
+
tmp.replace(STORE)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def config_of(kernel):
|
| 78 |
+
"""The config an autotuned tilelang kernel settled on, if it reports one."""
|
| 79 |
+
config = getattr(kernel, "config", None)
|
| 80 |
+
if isinstance(config, dict):
|
| 81 |
+
return config
|
| 82 |
+
# Some versions expose it on the wrapped best result instead.
|
| 83 |
+
best = getattr(kernel, "best", None)
|
| 84 |
+
return getattr(best, "config", None) if best is not None else None
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def summary():
|
| 88 |
+
store = _load()
|
| 89 |
+
if not store:
|
| 90 |
+
return f"nothing tuned yet ({STORE})"
|
| 91 |
+
lines = [f"{STORE}"]
|
| 92 |
+
for device, entries in sorted(store.items()):
|
| 93 |
+
lines.append(f" {device} ({len(entries)} kernels)")
|
| 94 |
+
for key, config in sorted(entries.items()):
|
| 95 |
+
kernel, shape = key.split(":", 1)
|
| 96 |
+
terse = " ".join(f"{k}={v}" for k, v in sorted(config.items()))
|
| 97 |
+
lines.append(f" {kernel:14s} [{shape}]")
|
| 98 |
+
lines.append(f" {terse}")
|
| 99 |
+
return "\n".join(lines)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
if __name__ == "__main__":
|
| 103 |
+
print(f"this gpu: {device_key()}\n")
|
| 104 |
+
print(summary())
|
code/modeling_mimans.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HuggingFace compatible modeling for Mimans M1."""
|
| 2 |
+
import math
|
| 3 |
+
from typing import List, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 9 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
from .configuration_mimans import MimansConfig
|
| 13 |
+
except ImportError:
|
| 14 |
+
from configuration_mimans import MimansConfig
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class MimansRMSNorm(nn.Module):
|
| 18 |
+
def __init__(self, hidden_size: int, eps: float = 1e-5):
|
| 19 |
+
super().__init__()
|
| 20 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 21 |
+
self.variance_epsilon = eps
|
| 22 |
+
|
| 23 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 24 |
+
input_dtype = hidden_states.dtype
|
| 25 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 26 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 27 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 28 |
+
return (self.weight * hidden_states).to(input_dtype)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class MimansRotaryEmbedding(nn.Module):
|
| 32 |
+
def __init__(self, dim: int, max_position_embeddings: int = 8192, base: float = 500000.0):
|
| 33 |
+
super().__init__()
|
| 34 |
+
self.dim = dim
|
| 35 |
+
self.max_position_embeddings = max_position_embeddings
|
| 36 |
+
self.base = base
|
| 37 |
+
inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim))
|
| 38 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 39 |
+
|
| 40 |
+
@torch.no_grad()
|
| 41 |
+
def forward(self, x: torch.Tensor, seq_len: int) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 42 |
+
t = torch.arange(seq_len, device=x.device, dtype=torch.int64).float()
|
| 43 |
+
freqs = torch.outer(t, self.inv_freq.to(x.device))
|
| 44 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 45 |
+
return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def rotate_half(x: torch.Tensor) -> torch.Tensor:
|
| 49 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 50 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 51 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def apply_rotary_pos_emb(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
# q, k shape: (batch, n_heads, seq_len, head_dim)
|
| 56 |
+
# cos, sin shape: (seq_len, head_dim)
|
| 57 |
+
cos = cos.unsqueeze(0).unsqueeze(1) # (1, 1, seq_len, head_dim)
|
| 58 |
+
sin = sin.unsqueeze(0).unsqueeze(1)
|
| 59 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 60 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 61 |
+
return q_embed, k_embed
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class MimansAttention(nn.Module):
|
| 65 |
+
def __init__(self, config: MimansConfig):
|
| 66 |
+
super().__init__()
|
| 67 |
+
self.config = config
|
| 68 |
+
self.hidden_size = config.hidden_size
|
| 69 |
+
self.num_heads = config.num_attention_heads
|
| 70 |
+
self.head_dim = config.head_dim
|
| 71 |
+
self.num_key_value_heads = config.num_key_value_heads
|
| 72 |
+
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
|
| 73 |
+
|
| 74 |
+
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
|
| 75 |
+
self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
|
| 76 |
+
self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
|
| 77 |
+
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
| 78 |
+
|
| 79 |
+
self.q_norm = MimansRMSNorm(self.head_dim, config.rms_norm_eps)
|
| 80 |
+
self.k_norm = MimansRMSNorm(self.head_dim, config.rms_norm_eps)
|
| 81 |
+
|
| 82 |
+
def forward(
|
| 83 |
+
self,
|
| 84 |
+
hidden_states: torch.Tensor,
|
| 85 |
+
cos: torch.Tensor,
|
| 86 |
+
sin: torch.Tensor,
|
| 87 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 88 |
+
) -> torch.Tensor:
|
| 89 |
+
batch_size, seq_len, _ = hidden_states.shape
|
| 90 |
+
|
| 91 |
+
q = self.q_proj(hidden_states).view(batch_size, seq_len, self.num_heads, self.head_dim)
|
| 92 |
+
k = self.k_proj(hidden_states).view(batch_size, seq_len, self.num_key_value_heads, self.head_dim)
|
| 93 |
+
v = self.v_proj(hidden_states).view(batch_size, seq_len, self.num_key_value_heads, self.head_dim)
|
| 94 |
+
|
| 95 |
+
q = self.q_norm(q).transpose(1, 2) # (batch, num_heads, seq_len, head_dim)
|
| 96 |
+
k = self.k_norm(k).transpose(1, 2) # (batch, num_kv_heads, seq_len, head_dim)
|
| 97 |
+
v = v.transpose(1, 2)
|
| 98 |
+
|
| 99 |
+
q, k = apply_rotary_pos_emb(q, k, cos, sin)
|
| 100 |
+
|
| 101 |
+
if self.num_key_value_groups > 1:
|
| 102 |
+
k = k.repeat_interleave(self.num_key_value_groups, dim=1)
|
| 103 |
+
v = v.repeat_interleave(self.num_key_value_groups, dim=1)
|
| 104 |
+
|
| 105 |
+
# Scaled dot-product attention
|
| 106 |
+
scale = 1.0 / math.sqrt(self.head_dim)
|
| 107 |
+
attn_output = F.scaled_dot_product_attention(q, k, v, is_causal=(attention_mask is None), scale=scale)
|
| 108 |
+
|
| 109 |
+
attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.hidden_size)
|
| 110 |
+
return self.o_proj(attn_output)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class MimansMLP(nn.Module):
|
| 114 |
+
def __init__(self, config: MimansConfig):
|
| 115 |
+
super().__init__()
|
| 116 |
+
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 117 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 118 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 119 |
+
|
| 120 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 121 |
+
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class AttnResGate(nn.Module):
|
| 125 |
+
def __init__(self, hidden_size: int, norm_eps: float = 1e-5):
|
| 126 |
+
super().__init__()
|
| 127 |
+
self.norm = MimansRMSNorm(hidden_size, norm_eps)
|
| 128 |
+
self.recency_bias = nn.Parameter(torch.zeros(1))
|
| 129 |
+
self.query = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 130 |
+
|
| 131 |
+
def forward(self, entries: List[torch.Tensor]) -> torch.Tensor:
|
| 132 |
+
if len(entries) == 1:
|
| 133 |
+
return entries[0]
|
| 134 |
+
# Calculate gated residual combination over snapshot entries
|
| 135 |
+
stacked = torch.stack(entries, dim=0) # (num_entries, batch, seq, hidden)
|
| 136 |
+
normed = self.norm(stacked)
|
| 137 |
+
q = self.query(normed[-1])
|
| 138 |
+
scores = torch.einsum("bsh,nbsh->nbs", q, normed) / math.sqrt(q.shape[-1])
|
| 139 |
+
# Add recency bias to last entry
|
| 140 |
+
scores[-1] = scores[-1] + self.recency_bias
|
| 141 |
+
weights = F.softmax(scores, dim=0).unsqueeze(-1)
|
| 142 |
+
return (stacked * weights).sum(dim=0)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class MimansDecoderLayer(nn.Module):
|
| 146 |
+
def __init__(self, config: MimansConfig):
|
| 147 |
+
super().__init__()
|
| 148 |
+
self.input_layernorm = MimansRMSNorm(config.hidden_size, config.rms_norm_eps)
|
| 149 |
+
self.self_attn = MimansAttention(config)
|
| 150 |
+
self.post_attention_layernorm = MimansRMSNorm(config.hidden_size, config.rms_norm_eps)
|
| 151 |
+
self.mlp = MimansMLP(config)
|
| 152 |
+
if config.use_attnres:
|
| 153 |
+
self.attn_gate = AttnResGate(config.hidden_size, config.rms_norm_eps)
|
| 154 |
+
self.ffn_gate = AttnResGate(config.hidden_size, config.rms_norm_eps)
|
| 155 |
+
else:
|
| 156 |
+
self.attn_gate = None
|
| 157 |
+
self.ffn_gate = None
|
| 158 |
+
|
| 159 |
+
def forward(
|
| 160 |
+
self,
|
| 161 |
+
snapshots: List[torch.Tensor],
|
| 162 |
+
running: torch.Tensor,
|
| 163 |
+
cos: torch.Tensor,
|
| 164 |
+
sin: torch.Tensor,
|
| 165 |
+
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
| 166 |
+
entries = snapshots + [running]
|
| 167 |
+
|
| 168 |
+
h_attn = self.attn_gate(entries) if self.attn_gate is not None else running
|
| 169 |
+
attn_out = self.self_attn(self.input_layernorm(h_attn), cos, sin)
|
| 170 |
+
running = running + attn_out
|
| 171 |
+
|
| 172 |
+
h_ffn = self.ffn_gate(entries) if self.ffn_gate is not None else running
|
| 173 |
+
mlp_out = self.mlp(self.post_attention_layernorm(h_ffn))
|
| 174 |
+
running = running + mlp_out
|
| 175 |
+
|
| 176 |
+
return snapshots, running
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class MimansPreTrainedModel(PreTrainedModel):
|
| 180 |
+
config_class = MimansConfig
|
| 181 |
+
base_model_prefix = "model"
|
| 182 |
+
supports_gradient_checkpointing = True
|
| 183 |
+
|
| 184 |
+
def _init_weights(self, module):
|
| 185 |
+
std = self.config.initializer_range
|
| 186 |
+
if isinstance(module, nn.Linear):
|
| 187 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 188 |
+
if module.bias is not None:
|
| 189 |
+
module.bias.data.zero_()
|
| 190 |
+
elif isinstance(module, nn.Embedding):
|
| 191 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class MimansModel(MimansPreTrainedModel):
|
| 195 |
+
def __init__(self, config: MimansConfig):
|
| 196 |
+
super().__init__(config)
|
| 197 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 198 |
+
self.rotary_emb = MimansRotaryEmbedding(config.head_dim, config.max_position_embeddings, config.rope_theta)
|
| 199 |
+
self.layers = nn.ModuleList([MimansDecoderLayer(config) for _ in range(config.num_hidden_layers)])
|
| 200 |
+
self.norm = MimansRMSNorm(config.hidden_size, config.rms_norm_eps)
|
| 201 |
+
self.post_init()
|
| 202 |
+
|
| 203 |
+
def forward(
|
| 204 |
+
self,
|
| 205 |
+
input_ids: torch.LongTensor,
|
| 206 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 207 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 208 |
+
return_dict: Optional[bool] = None,
|
| 209 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 210 |
+
batch_size, seq_len = input_ids.shape
|
| 211 |
+
running = self.embed_tokens(input_ids)
|
| 212 |
+
cos, sin = self.rotary_emb(running, seq_len)
|
| 213 |
+
|
| 214 |
+
snapshots: List[torch.Tensor] = []
|
| 215 |
+
layers_per_block = 3
|
| 216 |
+
for i, layer in enumerate(self.layers):
|
| 217 |
+
snapshots, running = layer(snapshots, running, cos, sin)
|
| 218 |
+
if (i + 1) % layers_per_block == 0:
|
| 219 |
+
snapshots.append(running)
|
| 220 |
+
|
| 221 |
+
hidden_states = self.norm(running)
|
| 222 |
+
return hidden_states
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
class MimansForCausalLM(MimansPreTrainedModel):
|
| 226 |
+
def __init__(self, config: MimansConfig):
|
| 227 |
+
super().__init__(config)
|
| 228 |
+
self.model = MimansModel(config)
|
| 229 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 230 |
+
self.post_init()
|
| 231 |
+
|
| 232 |
+
def get_input_embeddings(self):
|
| 233 |
+
return self.model.embed_tokens
|
| 234 |
+
|
| 235 |
+
def set_input_embeddings(self, value):
|
| 236 |
+
self.model.embed_tokens = value
|
| 237 |
+
|
| 238 |
+
def get_output_embeddings(self):
|
| 239 |
+
return self.lm_head
|
| 240 |
+
|
| 241 |
+
def set_output_embeddings(self, new_embeddings):
|
| 242 |
+
self.lm_head = new_embeddings
|
| 243 |
+
|
| 244 |
+
def forward(
|
| 245 |
+
self,
|
| 246 |
+
input_ids: torch.LongTensor = None,
|
| 247 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 248 |
+
labels: Optional[torch.LongTensor] = None,
|
| 249 |
+
return_dict: Optional[bool] = None,
|
| 250 |
+
**kwargs,
|
| 251 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 252 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 253 |
+
hidden_states = self.model(input_ids=input_ids, attention_mask=attention_mask)
|
| 254 |
+
logits = self.lm_head(hidden_states) / math.sqrt(self.config.hidden_size)
|
| 255 |
+
|
| 256 |
+
loss = None
|
| 257 |
+
if labels is not None:
|
| 258 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 259 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 260 |
+
loss_fct = nn.CrossEntropyLoss()
|
| 261 |
+
loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1))
|
| 262 |
+
|
| 263 |
+
if not return_dict:
|
| 264 |
+
output = (logits,)
|
| 265 |
+
return ((loss,) + output) if loss is not None else output
|
| 266 |
+
|
| 267 |
+
return CausalLMOutputWithPast(
|
| 268 |
+
loss=loss,
|
| 269 |
+
logits=logits,
|
| 270 |
+
)
|
code/paths.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Every path this project writes to, all on D:.
|
| 2 |
+
|
| 3 |
+
C: has under 11 GB free. One checkpoint is 3.3 GB and five are kept, the tuned
|
| 4 |
+
kernel cache passed 140 MB in a single session, and the uint16 shards for a 40B
|
| 5 |
+
token run are 80 GB before counting the raw text they came from. None of that
|
| 6 |
+
belongs on the system drive.
|
| 7 |
+
|
| 8 |
+
tilelang reads its cache location once, at import time, so this module has to be
|
| 9 |
+
imported before tilelang is -- which is why the three kernel modules import it
|
| 10 |
+
on their first line rather than taking the path as an argument.
|
| 11 |
+
|
| 12 |
+
Override the root with the MIMANS_ROOT environment variable.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
DEFAULT_ROOT = Path("D:/mimans")
|
| 20 |
+
|
| 21 |
+
_root = os.environ.get("MIMANS_ROOT")
|
| 22 |
+
if _root:
|
| 23 |
+
ROOT = Path(_root)
|
| 24 |
+
elif DEFAULT_ROOT.drive and Path(DEFAULT_ROOT.drive + "/").exists():
|
| 25 |
+
ROOT = DEFAULT_ROOT
|
| 26 |
+
else:
|
| 27 |
+
# No D: on this machine. Fall back next to the repo rather than silently
|
| 28 |
+
# filling up the system drive from a path nobody looked at.
|
| 29 |
+
ROOT = Path(__file__).resolve().parents[3] / "mimans_data"
|
| 30 |
+
print(f"warning: {DEFAULT_ROOT.drive} not found, writing to {ROOT}")
|
| 31 |
+
|
| 32 |
+
DATA = ROOT / "data" # raw downloaded corpora, full scale training data
|
| 33 |
+
SHARDS = ROOT / "shards" # tokenized uint16 shards, ~80 GB at 40B tokens
|
| 34 |
+
RUNS = ROOT / "runs" # checkpoints and log.jsonl
|
| 35 |
+
CACHE = ROOT / "cache" # tuned kernels
|
| 36 |
+
TMP = ROOT / "tmp" # scratch, including preflight's synthetic data
|
| 37 |
+
TOKENIZER = ROOT / "tokenizer" # tokenizer.json
|
| 38 |
+
|
| 39 |
+
# A small, separate corpus just for training and checking the tokenizer. Kept
|
| 40 |
+
# apart from DATA (the full 136 GB training corpus) because it is 2-3 orders
|
| 41 |
+
# of magnitude smaller and gets downloaded, inspected, and re-downloaded long
|
| 42 |
+
# before the real training data does.
|
| 43 |
+
TOKENIZER_DATA = ROOT / "tokenizer_data"
|
| 44 |
+
TOKENIZER_RAW = TOKENIZER_DATA / "raw" # downloaded, untouched
|
| 45 |
+
TOKENIZER_CODE = TOKENIZER_RAW / "code"
|
| 46 |
+
TOKENIZER_TEXT = TOKENIZER_RAW / "text"
|
| 47 |
+
TOKENIZER_MATH = TOKENIZER_RAW / "math"
|
| 48 |
+
TOKENIZER_HOLDOUT = TOKENIZER_DATA / "holdout" # never trained on, for --check
|
| 49 |
+
|
| 50 |
+
for _dir in (DATA, SHARDS, RUNS, CACHE, TMP, TOKENIZER,
|
| 51 |
+
TOKENIZER_CODE, TOKENIZER_TEXT, TOKENIZER_MATH, TOKENIZER_HOLDOUT):
|
| 52 |
+
_dir.mkdir(parents=True, exist_ok=True)
|
| 53 |
+
|
| 54 |
+
# setdefault, so an explicit environment variable still wins.
|
| 55 |
+
os.environ.setdefault("TILELANG_CACHE_DIR", str(CACHE / "tilelang"))
|
| 56 |
+
os.environ.setdefault("TILELANG_TMP_DIR", str(CACHE / "tilelang" / "tmp"))
|
| 57 |
+
|
| 58 |
+
# huggingface_hub defaults its cache to ~/.cache/huggingface on C:. Dataset
|
| 59 |
+
# shards are hundreds of MB each, so left alone this fills the system drive
|
| 60 |
+
# even when the download target is on D:.
|
| 61 |
+
os.environ.setdefault("HF_HOME", str(CACHE / "huggingface"))
|
| 62 |
+
os.environ.setdefault("HF_HUB_CACHE", str(CACHE / "huggingface" / "hub"))
|
| 63 |
+
|
| 64 |
+
# Python's own tempfile default is %TEMP% on C:. Anything here that writes a
|
| 65 |
+
# checkpoint or a synthetic shard through tempfile lands on the system drive
|
| 66 |
+
# unless it passes dir= explicitly, which is easy to forget.
|
| 67 |
+
os.environ.setdefault("TMPDIR", str(TMP))
|
| 68 |
+
os.environ.setdefault("TEMP", str(TMP))
|
| 69 |
+
os.environ.setdefault("TMP", str(TMP))
|
| 70 |
+
import tempfile
|
| 71 |
+
tempfile.tempdir = str(TMP)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def summary():
|
| 75 |
+
import shutil
|
| 76 |
+
free = shutil.disk_usage(ROOT).free / 1e9
|
| 77 |
+
return f"{ROOT} ({free:,.0f} GB free)"
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
if __name__ == "__main__":
|
| 81 |
+
print(summary())
|
| 82 |
+
for name, path in (("data", DATA), ("shards", SHARDS), ("runs", RUNS),
|
| 83 |
+
("cache", CACHE), ("tmp", TMP), ("tokenizer", TOKENIZER),
|
| 84 |
+
("tokenizer_data/raw/code", TOKENIZER_CODE),
|
| 85 |
+
("tokenizer_data/raw/text", TOKENIZER_TEXT),
|
| 86 |
+
("tokenizer_data/raw/math", TOKENIZER_MATH),
|
| 87 |
+
("tokenizer_data/holdout", TOKENIZER_HOLDOUT)):
|
| 88 |
+
print(f" {name:10s} {path}")
|
| 89 |
+
print(f"\n TILELANG_CACHE_DIR {os.environ['TILELANG_CACHE_DIR']}")
|
code/tokenizer/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Everything that turns text into token ids.
|
| 2 |
+
|
| 3 |
+
train_tokenizer trains the 49152 vocab byte level BPE, and checks it
|
| 4 |
+
"""
|
code/tokenizer/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (319 Bytes). View file
|
|
|
code/tokenizer/__pycache__/prepare_data.cpython-311.pyc
ADDED
|
Binary file (13.5 kB). View file
|
|
|
code/tokenizer/__pycache__/reshuffle_dataset.cpython-311.pyc
ADDED
|
Binary file (11.5 kB). View file
|
|
|
code/tokenizer/__pycache__/train_custom_tokenizer.cpython-311.pyc
ADDED
|
Binary file (6.49 kB). View file
|
|
|
code/tokenizer/__pycache__/train_tokenizer.cpython-311.pyc
ADDED
|
Binary file (30.6 kB). View file
|
|
|
code/tokenizer/data.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datasets import load_dataset
|
| 2 |
+
import os
|
| 3 |
+
import time
|
| 4 |
+
|
| 5 |
+
OUTPUT_DIR = "code_tokenizer_data"
|
| 6 |
+
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
| 7 |
+
|
| 8 |
+
# Languages ordered by popularity
|
| 9 |
+
LANGUAGES = [
|
| 10 |
+
("python", "python"),
|
| 11 |
+
("javascript", "javascript"),
|
| 12 |
+
("typescript", "typescript"),
|
| 13 |
+
("java", "java"),
|
| 14 |
+
("cpp", "cpp"),
|
| 15 |
+
("csharp", "c-sharp"),
|
| 16 |
+
("rust", "rust"),
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
MAX_FILES = 25000 # Safe number for 500M model
|
| 20 |
+
|
| 21 |
+
for name, folder in LANGUAGES:
|
| 22 |
+
print(f"\n=== Downloading {name.upper()} ===")
|
| 23 |
+
|
| 24 |
+
try:
|
| 25 |
+
ds = load_dataset(
|
| 26 |
+
"bigcode/starcoderdata",
|
| 27 |
+
data_dir=folder,
|
| 28 |
+
split="train",
|
| 29 |
+
streaming=True,
|
| 30 |
+
trust_remote_code=True
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
count = 0
|
| 34 |
+
output_file = os.path.join(OUTPUT_DIR, f"{name}.txt")
|
| 35 |
+
|
| 36 |
+
with open(output_file, "w", encoding="utf-8", errors="ignore") as f:
|
| 37 |
+
for sample in ds:
|
| 38 |
+
try:
|
| 39 |
+
# Try different possible content keys
|
| 40 |
+
content = (
|
| 41 |
+
sample.get("content") or
|
| 42 |
+
sample.get("text") or
|
| 43 |
+
sample.get("code") or
|
| 44 |
+
sample.get("source")
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
if content and isinstance(content, str) and len(content.strip()) > 80:
|
| 48 |
+
f.write(content.strip() + "\n\n" + "="*80 + "\n\n")
|
| 49 |
+
count += 1
|
| 50 |
+
|
| 51 |
+
if count % 2000 == 0:
|
| 52 |
+
print(f" → {name}: {count} files written")
|
| 53 |
+
|
| 54 |
+
if count >= MAX_FILES:
|
| 55 |
+
break
|
| 56 |
+
|
| 57 |
+
except Exception as e:
|
| 58 |
+
# Skip bad samples silently
|
| 59 |
+
continue
|
| 60 |
+
|
| 61 |
+
print(f"Finished {name}: {count} files saved → {output_file}")
|
| 62 |
+
|
| 63 |
+
except Exception as e:
|
| 64 |
+
print(f"Failed to download {name}: {e}")
|
| 65 |
+
print("Skipping this language and continuing...\n")
|
| 66 |
+
time.sleep(2) # small delay before next language
|
| 67 |
+
continue
|
| 68 |
+
|
| 69 |
+
print("\n=== All downloads finished ===")
|
code/tokenizer/prepare_data.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Raw text -> uint16 shards, the format mimans.data.Loader reads.
|
| 2 |
+
|
| 3 |
+
Documents are tokenized, separated by <|endoftext|>, concatenated into one
|
| 4 |
+
stream per source, and cut into rows of seq_len+1 so that x and y are the same
|
| 5 |
+
row shifted by one. Code gets the FIM transform at the character level before
|
| 6 |
+
tokenization, which is what teaches the model to fill in a hole given both
|
| 7 |
+
sides of it.
|
| 8 |
+
|
| 9 |
+
Shards land in paths.SHARDS/<source>/, one .bin plus a .json sidecar each, and
|
| 10 |
+
the loader picks up whatever is there. Writing is incremental: a shard is
|
| 11 |
+
flushed as soon as it is full, so an interrupted run leaves usable shards
|
| 12 |
+
behind rather than nothing.
|
| 13 |
+
|
| 14 |
+
python -m mimans.tokenizer.prepare_data --tokens-per-source 50e6
|
| 15 |
+
python -m mimans.tokenizer.prepare_data --seq-len 8192 --out long_ctx
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import argparse
|
| 20 |
+
import time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
|
| 25 |
+
from mimans import paths
|
| 26 |
+
from mimans.data import ENDOFTEXT, TOKEN_DTYPE, build_sequences, write_shard
|
| 27 |
+
from mimans.data import apply_fim
|
| 28 |
+
from mimans.tokenizer.train_tokenizer import (TOP_CODE_LANGUAGES, documents,
|
| 29 |
+
select_code_dirs,
|
| 30 |
+
select_code_parquets)
|
| 31 |
+
|
| 32 |
+
ENCODE_BATCH = 512 # documents per encode_batch call
|
| 33 |
+
SHARD_SEQUENCES = 20_000 # rows per shard file
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def load_tokenizer(path: Path):
|
| 37 |
+
try:
|
| 38 |
+
from tokenizers import Tokenizer
|
| 39 |
+
except ImportError as exc:
|
| 40 |
+
raise SystemExit("needs the tokenizers library: pip install tokenizers") from exc
|
| 41 |
+
if not path.exists():
|
| 42 |
+
raise SystemExit(f"no tokenizer at {path}; train it first")
|
| 43 |
+
return Tokenizer.from_file(str(path))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def source_paths(name: str, languages, raw_dir: Path | None = None):
|
| 47 |
+
"""Where each mixture source reads from, with code narrowed to the
|
| 48 |
+
languages the tokenizer was built for."""
|
| 49 |
+
if raw_dir is not None and Path(raw_dir).exists():
|
| 50 |
+
raw = Path(raw_dir)
|
| 51 |
+
sub = raw / name
|
| 52 |
+
if sub.is_dir() and (next(sub.glob("*.parquet"), None) or next(sub.glob("*.jsonl"), None) or next(sub.glob("*.txt"), None)):
|
| 53 |
+
return sub
|
| 54 |
+
# Parquet files directly under raw_dir
|
| 55 |
+
if next(raw.glob("*.parquet"), None) or next(raw.glob("*.jsonl"), None) or next(raw.glob("*.txt"), None):
|
| 56 |
+
return raw
|
| 57 |
+
if name == "code":
|
| 58 |
+
root = paths.TOKENIZER_CODE
|
| 59 |
+
return select_code_dirs(root, languages) + select_code_parquets(root, languages)
|
| 60 |
+
return {"text": paths.TOKENIZER_TEXT, "math": paths.TOKENIZER_MATH}[name]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def encode_stream(tokenizer, source, eod_id, target_tokens, fim_rate=0.0,
|
| 64 |
+
seed=0, report_every=25_000_000):
|
| 65 |
+
"""Yield token ids for one source until target_tokens is reached.
|
| 66 |
+
|
| 67 |
+
Batched through encode_batch because per-document calls spend more time
|
| 68 |
+
crossing into the tokenizer than tokenizing.
|
| 69 |
+
"""
|
| 70 |
+
rng = np.random.default_rng(seed)
|
| 71 |
+
produced, batch = 0, []
|
| 72 |
+
started = time.time()
|
| 73 |
+
next_report = report_every
|
| 74 |
+
|
| 75 |
+
def encode(batch):
|
| 76 |
+
"""One batch of documents -> a flat array of ids, each doc terminated
|
| 77 |
+
by <|endoftext|> so the model learns where documents end."""
|
| 78 |
+
ids = []
|
| 79 |
+
for encoding in tokenizer.encode_batch(batch, add_special_tokens=False):
|
| 80 |
+
ids.extend(encoding.ids)
|
| 81 |
+
ids.append(eod_id)
|
| 82 |
+
return np.asarray(ids, dtype=np.int64)
|
| 83 |
+
|
| 84 |
+
for text in documents(source):
|
| 85 |
+
if fim_rate:
|
| 86 |
+
text, _ = apply_fim(text, rng, fim_rate)
|
| 87 |
+
batch.append(text)
|
| 88 |
+
if len(batch) < ENCODE_BATCH:
|
| 89 |
+
continue
|
| 90 |
+
|
| 91 |
+
chunk = encode(batch)
|
| 92 |
+
batch = []
|
| 93 |
+
produced += len(chunk)
|
| 94 |
+
yield chunk
|
| 95 |
+
|
| 96 |
+
if produced >= next_report:
|
| 97 |
+
rate = produced / max(time.time() - started, 1e-9)
|
| 98 |
+
print(f" {produced/1e6:7.1f}M tokens {rate/1e3:6.1f}k tok/s",
|
| 99 |
+
flush=True)
|
| 100 |
+
next_report += report_every
|
| 101 |
+
if produced >= target_tokens:
|
| 102 |
+
return
|
| 103 |
+
|
| 104 |
+
if batch:
|
| 105 |
+
yield encode(batch)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def build_source(tokenizer, name, source, out_dir: Path, seq_len: int,
|
| 109 |
+
target_tokens: int, fim_rate: float, seed: int):
|
| 110 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 111 |
+
eod_id = tokenizer.token_to_id(ENDOFTEXT)
|
| 112 |
+
if eod_id is None:
|
| 113 |
+
raise SystemExit(f"tokenizer has no {ENDOFTEXT}")
|
| 114 |
+
|
| 115 |
+
row = seq_len + 1
|
| 116 |
+
carry = np.empty(0, dtype=np.int64)
|
| 117 |
+
pending = [] # rows waiting to be written
|
| 118 |
+
shard_index, rows_written, tokens_written = 0, 0, 0
|
| 119 |
+
started = time.time()
|
| 120 |
+
|
| 121 |
+
def flush_shard(force=False):
|
| 122 |
+
nonlocal pending, shard_index, rows_written
|
| 123 |
+
while pending and (len(pending) >= SHARD_SEQUENCES or force):
|
| 124 |
+
take = pending[:SHARD_SEQUENCES]
|
| 125 |
+
pending = pending[SHARD_SEQUENCES:]
|
| 126 |
+
block = np.stack(take).astype(TOKEN_DTYPE)
|
| 127 |
+
path = out_dir / f"{name}_{shard_index:04d}.bin"
|
| 128 |
+
write_shard(path, block)
|
| 129 |
+
rows_written += len(block)
|
| 130 |
+
shard_index += 1
|
| 131 |
+
print(f" wrote {path.name} {len(block):,} rows", flush=True)
|
| 132 |
+
if not force:
|
| 133 |
+
continue
|
| 134 |
+
if not pending:
|
| 135 |
+
break
|
| 136 |
+
|
| 137 |
+
for chunk in encode_stream(tokenizer, source, eod_id, target_tokens,
|
| 138 |
+
fim_rate=fim_rate, seed=seed):
|
| 139 |
+
tokens_written += len(chunk)
|
| 140 |
+
stream = np.concatenate([carry, chunk]) if carry.size else chunk
|
| 141 |
+
usable = (len(stream) // row) * row
|
| 142 |
+
if usable:
|
| 143 |
+
rows = build_sequences(stream[:usable].astype(TOKEN_DTYPE), seq_len)
|
| 144 |
+
pending.extend(rows)
|
| 145 |
+
flush_shard()
|
| 146 |
+
carry = stream[usable:]
|
| 147 |
+
|
| 148 |
+
flush_shard(force=True)
|
| 149 |
+
elapsed = time.time() - started
|
| 150 |
+
print(f" {name}: {rows_written:,} sequences in {shard_index} shards, "
|
| 151 |
+
f"{rows_written*seq_len/1e6:.1f}M usable tokens, {elapsed/60:.1f} min")
|
| 152 |
+
return rows_written
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def main():
|
| 156 |
+
parser = argparse.ArgumentParser()
|
| 157 |
+
parser.add_argument("--tokens-per-source", type=float, default=50e6,
|
| 158 |
+
help="token budget per source before shard cutting")
|
| 159 |
+
parser.add_argument("--seq-len", type=int, default=4096)
|
| 160 |
+
parser.add_argument("--out", type=Path, default=None,
|
| 161 |
+
help="shard root (default paths.SHARDS)")
|
| 162 |
+
parser.add_argument("--raw-dir", type=Path, default=Path("D:/Mimans-dataset") if Path("D:/Mimans-dataset").exists() else None,
|
| 163 |
+
help="path to raw text/parquet corpus directory")
|
| 164 |
+
parser.add_argument("--tokenizer", type=Path,
|
| 165 |
+
default=paths.TOKENIZER / "tokenizer.json")
|
| 166 |
+
parser.add_argument("--fim-rate", type=float, default=0.5,
|
| 167 |
+
help="fraction of code documents given the FIM transform")
|
| 168 |
+
parser.add_argument("--languages", default=",".join(TOP_CODE_LANGUAGES))
|
| 169 |
+
parser.add_argument("--sources", default="code,text,math")
|
| 170 |
+
parser.add_argument("--seed", type=int, default=0)
|
| 171 |
+
args = parser.parse_args()
|
| 172 |
+
|
| 173 |
+
tokenizer = load_tokenizer(args.tokenizer)
|
| 174 |
+
root = args.out or paths.SHARDS
|
| 175 |
+
languages = [s.strip() for s in args.languages.split(",") if s.strip()]
|
| 176 |
+
names = [s.strip() for s in args.sources.split(",") if s.strip()]
|
| 177 |
+
|
| 178 |
+
print(f"tokenizer {args.tokenizer} vocab {tokenizer.get_vocab_size():,}")
|
| 179 |
+
print(f"shards to {root}, seq_len {args.seq_len}")
|
| 180 |
+
print(f"{args.tokens_per_source/1e6:.0f}M tokens per source\n")
|
| 181 |
+
|
| 182 |
+
total = 0
|
| 183 |
+
for name in names:
|
| 184 |
+
source = source_paths(name, languages, raw_dir=args.raw_dir)
|
| 185 |
+
fim = args.fim_rate if name == "code" else 0.0
|
| 186 |
+
print(f" {name}" + (f" (fim rate {fim})" if fim else ""))
|
| 187 |
+
total += build_source(tokenizer, name, source, root / name,
|
| 188 |
+
args.seq_len, int(args.tokens_per_source),
|
| 189 |
+
fim, args.seed)
|
| 190 |
+
print(f"\n{total:,} sequences of {args.seq_len} = "
|
| 191 |
+
f"{total*args.seq_len/1e9:.2f}B tokens ready in {root}")
|
| 192 |
+
return True
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
if __name__ == "__main__":
|
| 196 |
+
raise SystemExit(0 if main() else 1)
|
code/tokenizer/reshuffle_dataset.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reshuffle raw Parquet dataset into balanced, uniformly mixed 4K binary shards.
|
| 2 |
+
|
| 3 |
+
Target Mixture:
|
| 4 |
+
- Text: ~70%
|
| 5 |
+
- Code: ~20%
|
| 6 |
+
- Math: ~10%
|
| 7 |
+
|
| 8 |
+
Reads D:\\Mimans-dataset\\train-0000-of-0040.parquet, tokenizes with tokenizer.json,
|
| 9 |
+
applies FIM to code, shuffles documents randomly, and outputs binary shards to D:\\mimans\\shards.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import argparse
|
| 14 |
+
import json
|
| 15 |
+
import math
|
| 16 |
+
import os
|
| 17 |
+
import sys
|
| 18 |
+
import time
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import pyarrow.parquet as pq
|
| 23 |
+
from tokenizers import Tokenizer
|
| 24 |
+
|
| 25 |
+
# Add CODE directory to sys.path
|
| 26 |
+
repo_root = Path(__file__).resolve().parents[2]
|
| 27 |
+
code_dir = repo_root / "CODE"
|
| 28 |
+
if str(code_dir) not in sys.path:
|
| 29 |
+
sys.path.insert(0, str(code_dir))
|
| 30 |
+
|
| 31 |
+
from mimans import paths
|
| 32 |
+
from mimans.data import ENDOFTEXT, TOKEN_DTYPE, apply_fim, write_shard
|
| 33 |
+
|
| 34 |
+
SHARD_SEQUENCES = 20_000 # 20,000 sequences per shard file (~81.9M tokens per shard)
|
| 35 |
+
ENCODE_BATCH = 1024
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def categorize_source(source_type: str) -> str:
|
| 39 |
+
st = str(source_type or "").strip().lower()
|
| 40 |
+
if "code" in st:
|
| 41 |
+
return "code"
|
| 42 |
+
elif "math" in st:
|
| 43 |
+
return "math"
|
| 44 |
+
else:
|
| 45 |
+
return "text"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def build_shards_from_dataset(dataset_path: Path, out_shards_dir: Path, seq_len: int = 4096, seed: int = 42, append: bool = False):
|
| 49 |
+
dataset_path = Path(dataset_path)
|
| 50 |
+
if dataset_path.is_file():
|
| 51 |
+
parquet_files = [dataset_path]
|
| 52 |
+
else:
|
| 53 |
+
parquet_files = sorted(dataset_path.glob("*.parquet"))
|
| 54 |
+
|
| 55 |
+
out_shards_dir = Path(out_shards_dir)
|
| 56 |
+
out_shards_dir.mkdir(parents=True, exist_ok=True)
|
| 57 |
+
|
| 58 |
+
tok_file = paths.TOKENIZER / "tokenizer.json"
|
| 59 |
+
if not tok_file.exists():
|
| 60 |
+
raise FileNotFoundError(f"Tokenizer not found at {tok_file}")
|
| 61 |
+
tokenizer = Tokenizer.from_file(str(tok_file))
|
| 62 |
+
eod_id = tokenizer.token_to_id(ENDOFTEXT) or 0
|
| 63 |
+
|
| 64 |
+
print(f"[RESHUFFLE] Reading {len(parquet_files)} Parquet file(s) from {dataset_path} ...", flush=True)
|
| 65 |
+
|
| 66 |
+
buckets = {"code": [], "math": [], "text": []}
|
| 67 |
+
for p_file in parquet_files:
|
| 68 |
+
print(f" Reading {p_file.name} ...", flush=True)
|
| 69 |
+
table = pq.read_table(str(p_file), columns=["text", "source_type"])
|
| 70 |
+
texts = table.column("text").to_pylist()
|
| 71 |
+
types = table.column("source_type").to_pylist()
|
| 72 |
+
|
| 73 |
+
for text_val, type_val in zip(texts, types):
|
| 74 |
+
if not text_val:
|
| 75 |
+
continue
|
| 76 |
+
cat = categorize_source(type_val)
|
| 77 |
+
buckets[cat].append(text_val)
|
| 78 |
+
|
| 79 |
+
rng = np.random.default_rng(seed)
|
| 80 |
+
for cat in buckets:
|
| 81 |
+
rng.shuffle(buckets[cat])
|
| 82 |
+
print(f" Category '{cat:5s}': {len(buckets[cat]):8,d} documents shuffled.")
|
| 83 |
+
|
| 84 |
+
# Process each category into binary shards
|
| 85 |
+
row_len = seq_len + 1
|
| 86 |
+
|
| 87 |
+
for cat in ("code", "math", "text"):
|
| 88 |
+
cat_dir = out_shards_dir / cat
|
| 89 |
+
if not append and cat_dir.exists():
|
| 90 |
+
for f in cat_dir.glob("*"):
|
| 91 |
+
try:
|
| 92 |
+
f.unlink()
|
| 93 |
+
except Exception:
|
| 94 |
+
pass
|
| 95 |
+
cat_dir.mkdir(parents=True, exist_ok=True)
|
| 96 |
+
|
| 97 |
+
existing_bins = sorted(cat_dir.glob("shard-*.bin"))
|
| 98 |
+
if append and existing_bins:
|
| 99 |
+
# Start from next available shard index
|
| 100 |
+
last_idx = max(int(f.stem.split("-")[1]) for f in existing_bins)
|
| 101 |
+
shard_idx = last_idx + 1
|
| 102 |
+
else:
|
| 103 |
+
shard_idx = 0
|
| 104 |
+
|
| 105 |
+
doc_list = buckets[cat]
|
| 106 |
+
fim_rate = 0.5 if cat == "code" else 0.0
|
| 107 |
+
print(f"\n[RESHUFFLE] Tokenizing & Sharding '{cat}' ({len(doc_list):,d} docs, FIM={fim_rate}, starting shard-{shard_idx:05d}) ...", flush=True)
|
| 108 |
+
|
| 109 |
+
started = time.time()
|
| 110 |
+
total_tokens = 0
|
| 111 |
+
carry = np.empty(0, dtype=np.int64)
|
| 112 |
+
pending_rows = []
|
| 113 |
+
|
| 114 |
+
batch = []
|
| 115 |
+
doc_count = 0
|
| 116 |
+
for doc in doc_list:
|
| 117 |
+
if fim_rate:
|
| 118 |
+
doc, _ = apply_fim(doc, rng, fim_rate)
|
| 119 |
+
batch.append(doc)
|
| 120 |
+
doc_count += 1
|
| 121 |
+
|
| 122 |
+
if len(batch) >= ENCODE_BATCH:
|
| 123 |
+
encodings = tokenizer.encode_batch(batch, add_special_tokens=False)
|
| 124 |
+
batch = []
|
| 125 |
+
|
| 126 |
+
doc_ids = []
|
| 127 |
+
for enc in encodings:
|
| 128 |
+
doc_ids.extend(enc.ids)
|
| 129 |
+
doc_ids.append(eod_id)
|
| 130 |
+
|
| 131 |
+
chunk = np.asarray(doc_ids, dtype=np.int64)
|
| 132 |
+
total_tokens += len(chunk)
|
| 133 |
+
|
| 134 |
+
if carry.size:
|
| 135 |
+
chunk = np.concatenate((carry, chunk))
|
| 136 |
+
usable = (len(chunk) // row_len) * row_len
|
| 137 |
+
carry = chunk[usable:]
|
| 138 |
+
rows = chunk[:usable].reshape(-1, row_len)
|
| 139 |
+
|
| 140 |
+
pending_rows.extend(rows)
|
| 141 |
+
while len(pending_rows) >= SHARD_SEQUENCES:
|
| 142 |
+
chunk_rows = pending_rows[:SHARD_SEQUENCES]
|
| 143 |
+
pending_rows = pending_rows[SHARD_SEQUENCES:]
|
| 144 |
+
s_data = np.asarray(chunk_rows, dtype=TOKEN_DTYPE)
|
| 145 |
+
shard_file = cat_dir / f"shard-{shard_idx:05d}.bin"
|
| 146 |
+
write_shard(shard_file, s_data)
|
| 147 |
+
shard_idx += 1
|
| 148 |
+
rate = total_tokens / max(1.0, time.time() - started)
|
| 149 |
+
print(f" [shard-{shard_idx-1:05d}.bin] {doc_count:7,d}/{len(doc_list):7,d} docs | {total_tokens/1e6:6.1f}M tokens ({rate/1e3:5.1f}k tok/s)", flush=True)
|
| 150 |
+
|
| 151 |
+
# Flush remaining batch
|
| 152 |
+
if batch:
|
| 153 |
+
encodings = tokenizer.encode_batch(batch, add_special_tokens=False)
|
| 154 |
+
doc_ids = []
|
| 155 |
+
for enc in encodings:
|
| 156 |
+
doc_ids.extend(enc.ids)
|
| 157 |
+
doc_ids.append(eod_id)
|
| 158 |
+
chunk = np.asarray(doc_ids, dtype=np.int64)
|
| 159 |
+
total_tokens += len(chunk)
|
| 160 |
+
if carry.size:
|
| 161 |
+
chunk = np.concatenate((carry, chunk))
|
| 162 |
+
usable = (len(chunk) // row_len) * row_len
|
| 163 |
+
rows = chunk[:usable].reshape(-1, row_len)
|
| 164 |
+
pending_rows.extend(rows)
|
| 165 |
+
|
| 166 |
+
if pending_rows:
|
| 167 |
+
s_data = np.asarray(pending_rows, dtype=TOKEN_DTYPE)
|
| 168 |
+
shard_file = cat_dir / f"shard-{shard_idx:05d}.bin"
|
| 169 |
+
write_shard(shard_file, s_data)
|
| 170 |
+
shard_idx += 1
|
| 171 |
+
|
| 172 |
+
elapsed = time.time() - started
|
| 173 |
+
print(f" [ok] '{cat}' complete: {shard_idx} shards ({total_tokens/1e6:.1f}M tokens) in {elapsed:.1f}s ({total_tokens/elapsed/1e3:.1f}k tok/s)")
|
| 174 |
+
|
| 175 |
+
print(f"\n[RESHUFFLE] SUCCESS! All binary shards generated and balanced in {out_shards_dir}")
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def main():
|
| 179 |
+
parser = argparse.ArgumentParser(description="Reshuffle and Shard Dataset")
|
| 180 |
+
parser.add_argument("--parquet", type=Path, default=None, help="Path to single parquet file")
|
| 181 |
+
parser.add_argument("--dataset-dir", type=Path, default=Path("D:/Mimans-dataset"), help="Path to dataset directory")
|
| 182 |
+
parser.add_argument("--out", type=Path, default=Path("D:/mimans/shards"))
|
| 183 |
+
parser.add_argument("--seq-len", type=int, default=4096)
|
| 184 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 185 |
+
parser.add_argument("--append", action="store_true", help="Append new shards to existing directory without wiping")
|
| 186 |
+
args = parser.parse_args()
|
| 187 |
+
|
| 188 |
+
target = args.parquet if args.parquet else args.dataset_dir
|
| 189 |
+
build_shards_from_dataset(target, args.out, seq_len=args.seq_len, seed=args.seed, append=args.append)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
if __name__ == "__main__":
|
| 193 |
+
main()
|
code/tokenizer/train_custom_tokenizer.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train Byte-Level BPE Tokenizer directly on Parquet dataset.
|
| 2 |
+
|
| 3 |
+
Vocab Size: 49,152
|
| 4 |
+
Special Tokens: <|endoftext|>, <|fim_prefix|>, <|fim_middle|>, <|fim_suffix|>, <|repo_name|>, <|file_sep|>, <|pad|>
|
| 5 |
+
Target Directory: D:/mimans/tokenizer/tokenizer.json
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import pyarrow.parquet as pq
|
| 15 |
+
from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, trainers
|
| 16 |
+
from tokenizers.models import BPE
|
| 17 |
+
from tokenizers.normalizers import NFKC
|
| 18 |
+
from tokenizers.pre_tokenizers import ByteLevel
|
| 19 |
+
|
| 20 |
+
# Add CODE directory to sys.path
|
| 21 |
+
repo_root = Path(__file__).resolve().parents[2]
|
| 22 |
+
code_dir = repo_root / "CODE"
|
| 23 |
+
if str(code_dir) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(code_dir))
|
| 25 |
+
|
| 26 |
+
from mimans import paths
|
| 27 |
+
from mimans.data import (ENDOFTEXT, FILE_SEP, FIM_MIDDLE, FIM_PREFIX,
|
| 28 |
+
FIM_SUFFIX, REPO_NAME)
|
| 29 |
+
|
| 30 |
+
VOCAB_SIZE = 49_152
|
| 31 |
+
|
| 32 |
+
SPECIAL_TOKENS = [
|
| 33 |
+
ENDOFTEXT, # 0
|
| 34 |
+
FIM_PREFIX, # 1
|
| 35 |
+
FIM_MIDDLE, # 2
|
| 36 |
+
FIM_SUFFIX, # 3
|
| 37 |
+
REPO_NAME, # 4
|
| 38 |
+
FILE_SEP, # 5
|
| 39 |
+
"<|pad|>", # 6
|
| 40 |
+
] + [f"<|extra_{i}|>" for i in range(9)]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def yield_parquet_documents(dataset_dir: Path):
|
| 44 |
+
parquet_files = sorted(Path(dataset_dir).glob("*.parquet"))
|
| 45 |
+
print(f"[TOKENIZER] Found {len(parquet_files)} Parquet dataset files in {dataset_dir} ...", flush=True)
|
| 46 |
+
count = 0
|
| 47 |
+
for p_file in parquet_files:
|
| 48 |
+
print(f" Reading {p_file.name} ...", flush=True)
|
| 49 |
+
table = pq.read_table(str(p_file), columns=["text"])
|
| 50 |
+
col = table.column("text")
|
| 51 |
+
for val in col:
|
| 52 |
+
text = val.as_py()
|
| 53 |
+
if text and len(text.strip()) > 5:
|
| 54 |
+
yield text
|
| 55 |
+
count += 1
|
| 56 |
+
print(f"[TOKENIZER] Loaded total {count:,d} documents for tokenizer training.", flush=True)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def train_tokenizer(dataset_dir: Path, out_file: Path):
|
| 60 |
+
out_file = Path(out_file)
|
| 61 |
+
out_file.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
|
| 63 |
+
print(f"[TOKENIZER] Building Byte-Level BPE Tokenizer (vocab size={VOCAB_SIZE:,d}) ...", flush=True)
|
| 64 |
+
tokenizer = Tokenizer(BPE(unk_token=None))
|
| 65 |
+
tokenizer.normalizer = normalizers.Sequence([NFKC()])
|
| 66 |
+
tokenizer.pre_tokenizer = pre_tokenizers.Sequence([ByteLevel(add_prefix_space=False, use_regex=True)])
|
| 67 |
+
tokenizer.decoder = decoders.ByteLevel()
|
| 68 |
+
|
| 69 |
+
trainer = trainers.BpeTrainer(
|
| 70 |
+
vocab_size=VOCAB_SIZE,
|
| 71 |
+
min_frequency=2,
|
| 72 |
+
special_tokens=SPECIAL_TOKENS,
|
| 73 |
+
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
|
| 74 |
+
show_progress=True,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
started = time.time()
|
| 78 |
+
docs = yield_parquet_documents(dataset_dir)
|
| 79 |
+
tokenizer.train_from_iterator(docs, trainer=trainer)
|
| 80 |
+
elapsed = time.time() - started
|
| 81 |
+
|
| 82 |
+
print(f"[TOKENIZER] Training complete in {elapsed:.1f}s. Saving to {out_file} ...", flush=True)
|
| 83 |
+
tokenizer.save(str(out_file))
|
| 84 |
+
|
| 85 |
+
# Verify round-trip encoding
|
| 86 |
+
sample = "def add(a: int, b: int) -> int:\n return a + b\n"
|
| 87 |
+
encoded = tokenizer.encode(sample)
|
| 88 |
+
decoded = tokenizer.decode(encoded.ids)
|
| 89 |
+
print(f"\n[VERIFY] Sample text:\n{sample!r}")
|
| 90 |
+
print(f"[VERIFY] Decoded text:\n{decoded!r}")
|
| 91 |
+
print(f"[VERIFY] Token count: {len(encoded.ids)} tokens")
|
| 92 |
+
print(f"[VERIFY] Round-trip match: {sample == decoded}")
|
| 93 |
+
|
| 94 |
+
print(f"\n[SUCCESS] Tokenizer trained and saved to {out_file}")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def main():
|
| 98 |
+
dataset_dir = Path("D:/Mimans-dataset")
|
| 99 |
+
out_file = paths.TOKENIZER / "tokenizer.json"
|
| 100 |
+
train_tokenizer(dataset_dir, out_file)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
if __name__ == "__main__":
|
| 104 |
+
main()
|
code/tokenizer/train_tokenizer.py
ADDED
|
@@ -0,0 +1,467 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train the byte level BPE this model is built around.
|
| 2 |
+
|
| 3 |
+
Every hard number here is read off the rest of the repo, not chosen:
|
| 4 |
+
|
| 5 |
+
vocab 49152 ModelConfig.vocab, and HeadConfig asserts vocab % 128 == 0
|
| 6 |
+
so the lm head kernel tiles cleanly. 49152 = 384 * 128,
|
| 7 |
+
and it is what makes the tied embedding 62,914,560
|
| 8 |
+
parameters and the model 514,345,526.
|
| 9 |
+
ids < 65536 shards are uint16 (data.TOKEN_DTYPE)
|
| 10 |
+
six specials data.py splices these into the character stream, so they
|
| 11 |
+
have to survive encoding as single atomic ids
|
| 12 |
+
no pad token sequences are packed and chopped, never padded; masking is
|
| 13 |
+
done with IGNORE_INDEX at the target level
|
| 14 |
+
|
| 15 |
+
Two things that are easy to get wrong:
|
| 16 |
+
|
| 17 |
+
* FIM is a character level transform applied *before* tokenization, so the
|
| 18 |
+
tokenizer routinely sees fragments that start mid-identifier, and the
|
| 19 |
+
markers arrive with no whitespace around them. Byte level BPE over the
|
| 20 |
+
full 256 byte alphabet is what keeps that lossless. Train on raw text --
|
| 21 |
+
never run apply_fim over the tokenizer corpus.
|
| 22 |
+
* Indentation is a quarter of Python. The byte level regex keeps runs of
|
| 23 |
+
whitespace together, which is why it is worth more than a plain whitespace
|
| 24 |
+
split here.
|
| 25 |
+
|
| 26 |
+
pip install tokenizers # and pyarrow for .parquet corpora
|
| 27 |
+
python -m mimans.tokenizer.train_tokenizer --train \
|
| 28 |
+
--corpus code=D:/data/code --corpus text=D:/data/text
|
| 29 |
+
python -m mimans.tokenizer.train_tokenizer --check --sample D:/data/holdout
|
| 30 |
+
"""
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import argparse
|
| 34 |
+
import gzip
|
| 35 |
+
import io
|
| 36 |
+
import json
|
| 37 |
+
from pathlib import Path
|
| 38 |
+
|
| 39 |
+
from mimans import paths
|
| 40 |
+
from mimans.data import (ENDOFTEXT, FILE_SEP, FIM_MIDDLE, FIM_PREFIX,
|
| 41 |
+
FIM_SUFFIX, REPO_NAME, apply_fim, undo_fim)
|
| 42 |
+
|
| 43 |
+
VOCAB_SIZE = 49_152
|
| 44 |
+
MIN_FREQUENCY = 2
|
| 45 |
+
|
| 46 |
+
# Ids 0-5 are the tokens data.py depends on and must never move. The rest are
|
| 47 |
+
# reserved so that adding a chat or instruct marker later does not mean
|
| 48 |
+
# retraining the tokenizer and re-tokenizing the corpus.
|
| 49 |
+
SPECIAL_TOKENS = [
|
| 50 |
+
ENDOFTEXT, # 0, also the eod_id that data.doc_ids_from wants
|
| 51 |
+
FIM_PREFIX, # 1
|
| 52 |
+
FIM_MIDDLE, # 2
|
| 53 |
+
FIM_SUFFIX, # 3
|
| 54 |
+
REPO_NAME, # 4
|
| 55 |
+
FILE_SEP, # 5
|
| 56 |
+
"<|pad|>", # 6, unused by the pipeline, conventional to have
|
| 57 |
+
] + [f"<|extra_{i}|>" for i in range(9)]
|
| 58 |
+
|
| 59 |
+
DEFAULT_WEIGHTS = {"code": 0.60, "text": 0.30, "math": 0.10}
|
| 60 |
+
TEXT_FIELDS = ("content", "text", "code")
|
| 61 |
+
|
| 62 |
+
# the-stack-smol's own folder names, lowercase, hyphenated where the language
|
| 63 |
+
# name has a space or symbol. Five general purpose, high volume languages
|
| 64 |
+
# rather than all thirty: fewer merges get spent on syntax nobody trains on,
|
| 65 |
+
# which is what a 49152 slot vocab for a 514M model can actually afford.
|
| 66 |
+
TOP_CODE_LANGUAGES = ("python", "javascript", "typescript", "java", "c++")
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _require_tokenizers():
|
| 70 |
+
try:
|
| 71 |
+
import tokenizers # noqa: F401
|
| 72 |
+
except ImportError as exc:
|
| 73 |
+
raise SystemExit("this needs the huggingface tokenizers library:\n"
|
| 74 |
+
" pip install tokenizers") from exc
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# ----------------------------------------------------------------- corpus ---
|
| 78 |
+
|
| 79 |
+
def _field_of(record: dict):
|
| 80 |
+
for name in TEXT_FIELDS:
|
| 81 |
+
if name in record and isinstance(record[name], str):
|
| 82 |
+
return record[name]
|
| 83 |
+
return None
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _jsonl(handle):
|
| 87 |
+
for line in handle:
|
| 88 |
+
line = line.strip()
|
| 89 |
+
if not line:
|
| 90 |
+
continue
|
| 91 |
+
try:
|
| 92 |
+
record = json.loads(line)
|
| 93 |
+
except json.JSONDecodeError:
|
| 94 |
+
continue
|
| 95 |
+
text = _field_of(record) if isinstance(record, dict) else None
|
| 96 |
+
if text:
|
| 97 |
+
yield text
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def documents(path):
|
| 101 |
+
"""Yield raw document strings from a file, a directory tree, or a list of
|
| 102 |
+
either -- the list form is how a source gets narrowed to a handful of
|
| 103 |
+
subdirectories, e.g. select_code_dirs()'s output, without teaching
|
| 104 |
+
mixed_corpus about the distinction.
|
| 105 |
+
|
| 106 |
+
Understands jsonl (plain, .gz, .zst), parquet, and plain text files, which
|
| 107 |
+
covers how the open datasets actually ship.
|
| 108 |
+
"""
|
| 109 |
+
if isinstance(path, (list, tuple)):
|
| 110 |
+
for item in path:
|
| 111 |
+
yield from documents(item)
|
| 112 |
+
return
|
| 113 |
+
|
| 114 |
+
path = Path(path)
|
| 115 |
+
if path.is_dir():
|
| 116 |
+
for child in sorted(path.rglob("*")):
|
| 117 |
+
if child.is_file():
|
| 118 |
+
yield from documents(child)
|
| 119 |
+
return
|
| 120 |
+
|
| 121 |
+
suffixes = [s.lower() for s in path.suffixes]
|
| 122 |
+
|
| 123 |
+
if ".parquet" in suffixes:
|
| 124 |
+
try:
|
| 125 |
+
import pyarrow.parquet as pq
|
| 126 |
+
except ImportError:
|
| 127 |
+
raise SystemExit(f"{path.name} is parquet: pip install pyarrow")
|
| 128 |
+
table = pq.ParquetFile(path)
|
| 129 |
+
for batch in table.iter_batches(batch_size=1024):
|
| 130 |
+
records = batch.to_pylist()
|
| 131 |
+
for record in records:
|
| 132 |
+
text = _field_of(record)
|
| 133 |
+
if text:
|
| 134 |
+
yield text
|
| 135 |
+
return
|
| 136 |
+
|
| 137 |
+
if ".jsonl" in suffixes or ".json" in suffixes:
|
| 138 |
+
if ".gz" in suffixes:
|
| 139 |
+
with gzip.open(path, "rt", encoding="utf-8", errors="replace") as h:
|
| 140 |
+
yield from _jsonl(h)
|
| 141 |
+
elif ".zst" in suffixes:
|
| 142 |
+
try:
|
| 143 |
+
import zstandard
|
| 144 |
+
except ImportError:
|
| 145 |
+
raise SystemExit(f"{path.name} is zstd: pip install zstandard")
|
| 146 |
+
with open(path, "rb") as raw:
|
| 147 |
+
stream = zstandard.ZstdDecompressor().stream_reader(raw)
|
| 148 |
+
wrapper = io.TextIOWrapper(stream, encoding="utf-8",
|
| 149 |
+
errors="replace")
|
| 150 |
+
yield from _jsonl(wrapper)
|
| 151 |
+
else:
|
| 152 |
+
with open(path, "rt", encoding="utf-8", errors="replace") as h:
|
| 153 |
+
yield from _jsonl(h)
|
| 154 |
+
return
|
| 155 |
+
|
| 156 |
+
try:
|
| 157 |
+
text = path.read_text(encoding="utf-8", errors="replace")
|
| 158 |
+
except OSError:
|
| 159 |
+
return
|
| 160 |
+
if text.strip():
|
| 161 |
+
yield text
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# the-stack-v2 style parquet 'language' values use display names and symbols
|
| 165 |
+
# ("C++", "C#") where the-stack-smol's folder names are lowercase and
|
| 166 |
+
# hyphenated ("c++", "c-sharp"). Only the entries that actually differ need
|
| 167 |
+
# to be listed; anything else is matched by lowercasing both sides.
|
| 168 |
+
CODE_LANGUAGE_ALIASES = {"c-sharp": "c#", "shell": "bash", "visual-basic": "visual basic"}
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def select_code_dirs(code_root: Path, languages=TOP_CODE_LANGUAGES):
|
| 172 |
+
"""The the-stack-smol layout is code_root/data/<language>/data.json*.
|
| 173 |
+
Returning just those subdirectories, rather than code_root itself, is what
|
| 174 |
+
excludes the other twenty five languages. Loose parquet files sitting
|
| 175 |
+
next to data/ are a second, already-mixed-language dataset with no per
|
| 176 |
+
file split -- those go through select_code_parquets instead, which filters
|
| 177 |
+
by row.
|
| 178 |
+
"""
|
| 179 |
+
code_root = Path(code_root)
|
| 180 |
+
data_dir = code_root / "data"
|
| 181 |
+
if not data_dir.is_dir():
|
| 182 |
+
return [code_root] # not this layout, fall back to reading everything
|
| 183 |
+
|
| 184 |
+
found, missing = [], []
|
| 185 |
+
for lang in languages:
|
| 186 |
+
d = data_dir / lang
|
| 187 |
+
(found if d.is_dir() else missing).append(d if d.is_dir() else lang)
|
| 188 |
+
if missing:
|
| 189 |
+
available = sorted(p.name for p in data_dir.iterdir() if p.is_dir())
|
| 190 |
+
print(f" warning: {missing} not found under {data_dir}; "
|
| 191 |
+
f"have {available}")
|
| 192 |
+
return found
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def select_code_parquets(code_root: Path, languages=TOP_CODE_LANGUAGES):
|
| 196 |
+
"""Loose *.parquet files directly under code_root (not under data/) are a
|
| 197 |
+
second dataset shipped in the same folder, e.g. a the-stack-v2 sample,
|
| 198 |
+
with every language mixed into one file and picked out by a 'language'
|
| 199 |
+
column. Filter each down to the requested languages and cache the result
|
| 200 |
+
next to the source, keyed off the source's mtime so re-running is cheap.
|
| 201 |
+
"""
|
| 202 |
+
import pyarrow.parquet as pq
|
| 203 |
+
import pyarrow.compute as pc
|
| 204 |
+
|
| 205 |
+
code_root = Path(code_root)
|
| 206 |
+
wanted = {CODE_LANGUAGE_ALIASES.get(l, l).lower() for l in languages}
|
| 207 |
+
cache_dir = code_root / "_language_filtered"
|
| 208 |
+
out = []
|
| 209 |
+
|
| 210 |
+
for src in sorted(code_root.glob("*.parquet")):
|
| 211 |
+
cache = cache_dir / f"{src.stem}.parquet"
|
| 212 |
+
if cache.exists() and cache.stat().st_mtime >= src.stat().st_mtime:
|
| 213 |
+
out.append(cache)
|
| 214 |
+
continue
|
| 215 |
+
|
| 216 |
+
schema_names = pq.ParquetFile(src).schema_arrow.names
|
| 217 |
+
lang_col = next((c for c in ("language", "lang") if c in schema_names), None)
|
| 218 |
+
if lang_col is None:
|
| 219 |
+
print(f" {src.name}: no language column, skipping "
|
| 220 |
+
f"(cannot filter, would pull in every language)")
|
| 221 |
+
continue
|
| 222 |
+
|
| 223 |
+
table = pq.read_table(src)
|
| 224 |
+
norm = pc.utf8_lower(table.column(lang_col))
|
| 225 |
+
mask = pc.is_in(norm, value_set=__import__("pyarrow").array(sorted(wanted)))
|
| 226 |
+
filtered = table.filter(mask)
|
| 227 |
+
if filtered.num_rows == 0:
|
| 228 |
+
print(f" {src.name}: 0 of {table.num_rows} rows match {languages}, skipping")
|
| 229 |
+
continue
|
| 230 |
+
|
| 231 |
+
cache_dir.mkdir(exist_ok=True)
|
| 232 |
+
pq.write_table(filtered, cache)
|
| 233 |
+
print(f" {src.name}: kept {filtered.num_rows:,} of {table.num_rows:,} rows "
|
| 234 |
+
f"matching {languages} -> {cache.name}")
|
| 235 |
+
out.append(cache)
|
| 236 |
+
return out
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def mixed_corpus(sources: dict, weights: dict, budget_bytes: int, verbose=True):
|
| 240 |
+
"""Documents from every source, each capped at its share of the budget.
|
| 241 |
+
|
| 242 |
+
BPE only ever sees aggregate counts, so the order does not matter -- what
|
| 243 |
+
matters is that the proportions match the training mixture, since that is
|
| 244 |
+
what the merges get tuned for.
|
| 245 |
+
"""
|
| 246 |
+
taken = {}
|
| 247 |
+
for name, path in sources.items():
|
| 248 |
+
share = weights.get(name, 0.0)
|
| 249 |
+
cap = int(budget_bytes * share)
|
| 250 |
+
used = 0
|
| 251 |
+
if cap <= 0:
|
| 252 |
+
continue
|
| 253 |
+
for text in documents(path):
|
| 254 |
+
yield text
|
| 255 |
+
used += len(text.encode("utf-8", errors="ignore"))
|
| 256 |
+
if used >= cap:
|
| 257 |
+
break
|
| 258 |
+
taken[name] = used
|
| 259 |
+
if verbose:
|
| 260 |
+
print(f" {name:6s} {used/1e9:6.3f} GB of {cap/1e9:6.3f} GB budget"
|
| 261 |
+
f"{' (source exhausted)' if used < cap else ''}")
|
| 262 |
+
if verbose:
|
| 263 |
+
print(f" total {sum(taken.values())/1e9:6.3f} GB")
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
# ------------------------------------------------------------------ train ---
|
| 267 |
+
|
| 268 |
+
def build_tokenizer():
|
| 269 |
+
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, processors
|
| 270 |
+
|
| 271 |
+
tok = Tokenizer(models.BPE(unk_token=None))
|
| 272 |
+
# No normalizer at all. NFC or lowercasing would make encode/decode lossy
|
| 273 |
+
# on real source files, and the shards have to round trip exactly.
|
| 274 |
+
tok.normalizer = None
|
| 275 |
+
tok.pre_tokenizer = pre_tokenizers.Sequence([
|
| 276 |
+
# Splitting every digit is what makes the 10 percent math mixture
|
| 277 |
+
# learnable; without it "1024" and "1025" share nothing.
|
| 278 |
+
pre_tokenizers.Digits(individual_digits=True),
|
| 279 |
+
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=True),
|
| 280 |
+
])
|
| 281 |
+
tok.decoder = decoders.ByteLevel()
|
| 282 |
+
tok.post_processor = processors.ByteLevel(trim_offsets=False)
|
| 283 |
+
return tok
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def train(sources: dict, out: Path, budget_bytes: int, weights: dict,
|
| 287 |
+
vocab_size=VOCAB_SIZE):
|
| 288 |
+
_require_tokenizers()
|
| 289 |
+
from tokenizers import pre_tokenizers, trainers
|
| 290 |
+
|
| 291 |
+
assert vocab_size % 128 == 0, "the lm head kernel asserts vocab % 128 == 0"
|
| 292 |
+
assert vocab_size <= 65_536, "shards are uint16"
|
| 293 |
+
|
| 294 |
+
tok = build_tokenizer()
|
| 295 |
+
trainer = trainers.BpeTrainer(
|
| 296 |
+
vocab_size=vocab_size,
|
| 297 |
+
min_frequency=MIN_FREQUENCY,
|
| 298 |
+
show_progress=True,
|
| 299 |
+
special_tokens=SPECIAL_TOKENS,
|
| 300 |
+
# All 256 bytes present means there is no unknown token and no input
|
| 301 |
+
# that fails to encode, which is the property FIM fragments rely on.
|
| 302 |
+
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
print(f"training to {vocab_size:,} over {budget_bytes/1e9:.1f} GB, "
|
| 306 |
+
f"mixture {weights}")
|
| 307 |
+
tok.train_from_iterator(mixed_corpus(sources, weights, budget_bytes),
|
| 308 |
+
trainer=trainer)
|
| 309 |
+
tok.add_special_tokens(SPECIAL_TOKENS)
|
| 310 |
+
|
| 311 |
+
out.parent.mkdir(parents=True, exist_ok=True)
|
| 312 |
+
tok.save(str(out))
|
| 313 |
+
print(f"\nwrote {out} vocab {tok.get_vocab_size():,}")
|
| 314 |
+
return tok
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
# ------------------------------------------------------------------ check ---
|
| 318 |
+
|
| 319 |
+
def check(path: Path, sample: Path = None):
|
| 320 |
+
"""The acceptance gate. Do not build shards until this is clean."""
|
| 321 |
+
_require_tokenizers()
|
| 322 |
+
from tokenizers import Tokenizer
|
| 323 |
+
|
| 324 |
+
tok = Tokenizer.from_file(str(path))
|
| 325 |
+
passed = True
|
| 326 |
+
|
| 327 |
+
def check_one(label, condition):
|
| 328 |
+
nonlocal passed
|
| 329 |
+
passed &= bool(condition)
|
| 330 |
+
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
| 331 |
+
|
| 332 |
+
print("size")
|
| 333 |
+
size = tok.get_vocab_size()
|
| 334 |
+
print(f" vocab {size:,}")
|
| 335 |
+
check_one(f"exactly {VOCAB_SIZE:,}, the size the model is built for",
|
| 336 |
+
size == VOCAB_SIZE)
|
| 337 |
+
check_one("divisible by 128 for the lm head kernel", size % 128 == 0)
|
| 338 |
+
check_one("every id fits in uint16", size <= 65_536)
|
| 339 |
+
try:
|
| 340 |
+
from mimans.model.model import ModelConfig
|
| 341 |
+
check_one("agrees with ModelConfig.vocab", ModelConfig().vocab == size)
|
| 342 |
+
except Exception:
|
| 343 |
+
print(" (torch not importable here, skipping the ModelConfig cross check)")
|
| 344 |
+
|
| 345 |
+
print("\nspecial tokens")
|
| 346 |
+
for i, name in enumerate(SPECIAL_TOKENS[:6]):
|
| 347 |
+
ids = tok.encode(name, add_special_tokens=False).ids
|
| 348 |
+
check_one(f"{name:16s} -> {ids} at the expected id {i}",
|
| 349 |
+
ids == [i])
|
| 350 |
+
glued = f"{REPO_NAME}owner/repo{FILE_SEP}import os\n{ENDOFTEXT}"
|
| 351 |
+
ids = tok.encode(glued, add_special_tokens=False).ids
|
| 352 |
+
check_one("markers stay atomic with no whitespace around them",
|
| 353 |
+
ids[0] == 4 and ids.count(5) == 1 and ids[-1] == 0)
|
| 354 |
+
|
| 355 |
+
print("\nround trip")
|
| 356 |
+
samples = []
|
| 357 |
+
if sample:
|
| 358 |
+
for i, text in enumerate(documents(Path(sample))):
|
| 359 |
+
samples.append(text)
|
| 360 |
+
if i >= 400:
|
| 361 |
+
break
|
| 362 |
+
samples += [
|
| 363 |
+
"def f(x):\n return x + 1\n",
|
| 364 |
+
"\tif (a && b) { return 'x\\n'; }\n",
|
| 365 |
+
"# emoji \U0001f600 and accents: café, naïve, \u4e2d\u6587\n",
|
| 366 |
+
"x = 0b1010 + 0x1F + 3.14159e-10\n",
|
| 367 |
+
"\r\nwindows\r\nline\r\nendings\r\n",
|
| 368 |
+
"".join(chr(i) for i in range(1, 256)),
|
| 369 |
+
]
|
| 370 |
+
bad = [s for s in samples
|
| 371 |
+
if tok.decode(tok.encode(s, add_special_tokens=False).ids,
|
| 372 |
+
skip_special_tokens=False) != s]
|
| 373 |
+
check_one(f"{len(samples)} samples decode back exactly"
|
| 374 |
+
f"{'' if not bad else f', {len(bad)} failed'}", not bad)
|
| 375 |
+
|
| 376 |
+
print("\nfim, the way data.py builds it")
|
| 377 |
+
rng = __import__("numpy").random.default_rng(0)
|
| 378 |
+
doc = "class A:\n def b(self):\n return 42\n"
|
| 379 |
+
transformed, did = apply_fim(doc, rng, rate=1.0)
|
| 380 |
+
ids = tok.encode(transformed, add_special_tokens=False).ids
|
| 381 |
+
back = tok.decode(ids, skip_special_tokens=False)
|
| 382 |
+
check_one("a fim sample survives encode and decode", back == transformed)
|
| 383 |
+
check_one("and still reconstructs the original file", undo_fim(back) == doc)
|
| 384 |
+
check_one("all three markers are one id each",
|
| 385 |
+
sum(i in (1, 2, 3) for i in ids) == 3)
|
| 386 |
+
|
| 387 |
+
print("\nindentation")
|
| 388 |
+
ids = tok.encode(" return x\n deeper\n",
|
| 389 |
+
add_special_tokens=False).ids
|
| 390 |
+
print(f" two indented lines cost {len(ids)} tokens")
|
| 391 |
+
check_one("indentation is not one token per space", len(ids) < 14)
|
| 392 |
+
|
| 393 |
+
if samples:
|
| 394 |
+
print("\nfertility")
|
| 395 |
+
raw = sum(len(s.encode("utf-8")) for s in samples)
|
| 396 |
+
count = sum(len(tok.encode(s, add_special_tokens=False).ids)
|
| 397 |
+
for s in samples)
|
| 398 |
+
rate = raw / max(count, 1)
|
| 399 |
+
print(f" {rate:.2f} bytes per token over {raw/1e6:.1f} MB")
|
| 400 |
+
print(f" train.py assumes 3.4 for bits per byte; 40B tokens is "
|
| 401 |
+
f"{40e9*rate/1e9:.0f} GB of text at this rate")
|
| 402 |
+
check_one("in the range a code tokenizer should land in",
|
| 403 |
+
2.5 < rate < 5.0)
|
| 404 |
+
|
| 405 |
+
print("\n" + ("tokenizer ok, safe to build shards"
|
| 406 |
+
if passed else "FAILURES ABOVE"))
|
| 407 |
+
return passed
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
def main():
|
| 411 |
+
parser = argparse.ArgumentParser()
|
| 412 |
+
parser.add_argument("--train", action="store_true")
|
| 413 |
+
parser.add_argument("--check", action="store_true")
|
| 414 |
+
parser.add_argument("--corpus", action="append", default=[],
|
| 415 |
+
metavar="NAME=PATH",
|
| 416 |
+
help="repeatable, e.g. --corpus code=D:/other/place. "
|
| 417 |
+
"Omit entirely to use paths.TOKENIZER_RAW/{code,text,math}.")
|
| 418 |
+
parser.add_argument("--out", type=Path,
|
| 419 |
+
default=paths.TOKENIZER / "tokenizer.json")
|
| 420 |
+
parser.add_argument("--budget-gb", type=float, default=2.0,
|
| 421 |
+
help="bytes of corpus to train on, split by the mixture")
|
| 422 |
+
parser.add_argument("--sample", type=Path, default=paths.TOKENIZER_HOLDOUT,
|
| 423 |
+
help="held out files for the round trip and fertility checks")
|
| 424 |
+
parser.add_argument("--vocab", type=int, default=VOCAB_SIZE)
|
| 425 |
+
parser.add_argument("--code-languages", default=",".join(TOP_CODE_LANGUAGES),
|
| 426 |
+
help="comma separated the-stack-smol folder names, "
|
| 427 |
+
"or 'all' to read the entire code source unfiltered")
|
| 428 |
+
args = parser.parse_args()
|
| 429 |
+
|
| 430 |
+
if args.train:
|
| 431 |
+
sources = {}
|
| 432 |
+
for item in args.corpus:
|
| 433 |
+
if "=" not in item:
|
| 434 |
+
raise SystemExit(f"--corpus wants NAME=PATH, got {item!r}")
|
| 435 |
+
name, path = item.split("=", 1)
|
| 436 |
+
sources[name] = Path(path)
|
| 437 |
+
if not sources:
|
| 438 |
+
# No --corpus given: use the standard layout directly.
|
| 439 |
+
sources = {"code": paths.TOKENIZER_CODE, "text": paths.TOKENIZER_TEXT,
|
| 440 |
+
"math": paths.TOKENIZER_MATH}
|
| 441 |
+
sources = {k: v for k, v in sources.items() if any(v.iterdir())}
|
| 442 |
+
if "code" in sources and args.code_languages != "all":
|
| 443 |
+
languages = [s.strip() for s in args.code_languages.split(",") if s.strip()]
|
| 444 |
+
dirs = select_code_dirs(sources["code"], languages)
|
| 445 |
+
parquets = select_code_parquets(sources["code"], languages)
|
| 446 |
+
print(f"code restricted to {languages}: "
|
| 447 |
+
f"{[d.name for d in dirs]} + {[p.name for p in parquets]}")
|
| 448 |
+
sources["code"] = dirs + parquets
|
| 449 |
+
if not sources:
|
| 450 |
+
raise SystemExit(
|
| 451 |
+
f"nothing found in {paths.TOKENIZER_RAW} and no --corpus given.\n"
|
| 452 |
+
f"put files under {paths.TOKENIZER_CODE}, {paths.TOKENIZER_TEXT}, "
|
| 453 |
+
f"{paths.TOKENIZER_MATH}")
|
| 454 |
+
weights = {k: DEFAULT_WEIGHTS.get(k, 0.0) for k in sources}
|
| 455 |
+
total = sum(weights.values())
|
| 456 |
+
if abs(total - 1.0) > 1e-6:
|
| 457 |
+
weights = {k: v / total for k, v in weights.items()}
|
| 458 |
+
print(f"renormalised mixture to {weights}")
|
| 459 |
+
train(sources, args.out, int(args.budget_gb * 1e9), weights, args.vocab)
|
| 460 |
+
|
| 461 |
+
if args.check or not args.train:
|
| 462 |
+
return check(args.out, args.sample)
|
| 463 |
+
return True
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
if __name__ == "__main__":
|
| 467 |
+
raise SystemExit(0 if main() else 1)
|