Upload folder using huggingface_hub
Browse files- README.md +189 -0
- chat.py +194 -0
- chat_format.py +101 -0
- config.json +22 -0
- generation_config.json +9 -0
- model.py +350 -0
- model.safetensors +3 -0
- modeling_nanoi1.py +51 -0
- tokenizer.json +0 -0
- tokenizer_config.json +11 -0
README.md
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
- hi
|
| 6 |
+
- bn
|
| 7 |
+
- mr
|
| 8 |
+
- ta
|
| 9 |
+
- te
|
| 10 |
+
- gu
|
| 11 |
+
- kn
|
| 12 |
+
- ml
|
| 13 |
+
- pa
|
| 14 |
+
library_name: pytorch
|
| 15 |
+
pipeline_tag: text-generation
|
| 16 |
+
tags:
|
| 17 |
+
- small-language-model
|
| 18 |
+
- multi-head-latent-attention
|
| 19 |
+
- indic
|
| 20 |
+
- kitefish
|
| 21 |
+
datasets:
|
| 22 |
+
- HuggingFaceFW/fineweb-edu
|
| 23 |
+
- HuggingFaceTB/finemath
|
| 24 |
+
- HuggingFaceTB/smollm-corpus
|
| 25 |
+
- HuggingFaceFW/fineweb-2
|
| 26 |
+
- allenai/dolmino-mix-1124
|
| 27 |
+
- OpenCoder-LLM/opc-annealing-corpus
|
| 28 |
+
- institutional/institutional-books-1.0
|
| 29 |
+
---
|
| 30 |
+
|
| 31 |
+
# NanoI1-155M (base)
|
| 32 |
+
|
| 33 |
+
**NanoI1** is a 155M-parameter decoder-only language model trained from scratch by
|
| 34 |
+
[KiteFish AI](https://kitefishai.com) on ~1.0T tokens. It is built for low-cost, on-premise deployment in
|
| 35 |
+
Indian regulated industries (banking, insurance, financial services). Its main uses are structured extraction,
|
| 36 |
+
classification and grounded Q&A, where data residency and inference cost matter more than open-ended
|
| 37 |
+
general knowledge.
|
| 38 |
+
|
| 39 |
+
This repository holds the **base (pretrained) model**, best used as a starting point for fine-tuning. For chat
|
| 40 |
+
and instruction following use [KiteFishAI/NanoI1-155M-Instruct](https://huggingface.co/KiteFishAI/NanoI1-155M-Instruct).
|
| 41 |
+
|
| 42 |
+
| | |
|
| 43 |
+
|---|---|
|
| 44 |
+
| Parameters | 155.3M total: 78.0M transformer layers, 38.6M input embedding, 38.7M output projection |
|
| 45 |
+
| Architecture | Decoder-only Transformer, Multi-Head Latent Attention (MLA), SwiGLU, RMSNorm, RoPE |
|
| 46 |
+
| Training tokens | ~998B (pretraining 629B + extension 319B + anneal 50B) |
|
| 47 |
+
| Context length | 2,048 tokens (trained) |
|
| 48 |
+
| Tokenizer | Byte-level BPE, 50,304 tokens, trained on our own mixture |
|
| 49 |
+
| Languages | English; Hindi and 8 other Indian languages (limited) |
|
| 50 |
+
| Licence | Apache-2.0 |
|
| 51 |
+
|
| 52 |
+
## Quick start
|
| 53 |
+
|
| 54 |
+
The model uses a custom architecture, so loading needs the Python files in this repository (no `transformers`
|
| 55 |
+
class yet).
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
pip install torch safetensors tokenizers huggingface_hub
|
| 59 |
+
```
|
| 60 |
+
```python
|
| 61 |
+
import sys
|
| 62 |
+
from huggingface_hub import snapshot_download
|
| 63 |
+
|
| 64 |
+
path = snapshot_download("KiteFishAI/NanoI1-155M")
|
| 65 |
+
sys.path.insert(0, path)
|
| 66 |
+
import torch
|
| 67 |
+
from modeling_nanoi1 import from_pretrained
|
| 68 |
+
from chat import generate
|
| 69 |
+
|
| 70 |
+
model, tok = from_pretrained(path)
|
| 71 |
+
ids = [tok.token_to_id("<|endoftext|>")] + tok.encode("The Reserve Bank of India regulates", add_special_tokens=False).ids
|
| 72 |
+
out = generate(model, ids, max_new_tokens=40, temperature=0) # greedy continuation
|
| 73 |
+
print(tok.decode(out))
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
To fine-tune, load the weights the same way and train with your own loop, or use `sft.py` from the training
|
| 77 |
+
repository with the original checkpoint.
|
| 78 |
+
|
| 79 |
+
## Architecture
|
| 80 |
+
|
| 81 |
+
| Hyper-parameter | Value |
|
| 82 |
+
|---|---|
|
| 83 |
+
| Layers | 12 |
|
| 84 |
+
| Hidden size | 768 |
|
| 85 |
+
| Attention heads | 12 (head dim 64) |
|
| 86 |
+
| KV latent dimension (MLA) | 256 |
|
| 87 |
+
| FFN | SwiGLU, 2,048 hidden |
|
| 88 |
+
| Normalisation | Pre-norm RMSNorm (ε = 1e-6), QK-norm |
|
| 89 |
+
| Position encoding | RoPE, θ = 5,000,000 |
|
| 90 |
+
| Embeddings | Untied input / output |
|
| 91 |
+
| Vocabulary | 50,304 (byte-level BPE, 32 special tokens) |
|
| 92 |
+
|
| 93 |
+
Multi-Head Latent Attention caches one 256-d latent vector per token per layer instead of full keys and values
|
| 94 |
+
(2 × 768), so the KV cache is **6× smaller** than standard multi-head attention at the same width.
|
| 95 |
+
|
| 96 |
+
## Training
|
| 97 |
+
|
| 98 |
+
| Stage | Tokens | Data | LR schedule |
|
| 99 |
+
|---|---|---|---|
|
| 100 |
+
| 1. Pretraining | 629B (steps 0–600k) | Our 798B-token corpus, each token at most once: English web, code, peS2o papers, multilingual web, Project Gutenberg, Proof-Pile math, English Wikipedia | linear warmup, then constant 6e-4 |
|
| 101 |
+
| 2. Extension | 319B (600k–904k) | Unseen remainder of the stage-1 corpus (169B) + FineWeb-Edu (60B), The Stack v2 code (50B), Institutional Books 1.0 English (25B), FineMath (15B) | constant 6e-4 |
|
| 102 |
+
| 3. Anneal | 50B (904k–952k) | High-quality mix: FineWeb-Edu, Cosmopedia, FineMath, Dolmino math & FLAN, OpenCoder annealing corpus, StackExchange, FineWeb-2 Indic, plus ~35% replay of stage-1 sources | linear decay 6e-4 → 0 |
|
| 103 |
+
|
| 104 |
+
- **Benchmark decontamination:** every document added in stages 2–3 that shares a 13-word sequence with the
|
| 105 |
+
test set of any benchmark below was removed.
|
| 106 |
+
- **Optimiser:** AdamW (β = 0.9/0.95, weight decay 0.1, gradient clip 1.0), bf16 mixed precision.
|
| 107 |
+
- **Batch:** global batch of 1,048,576 tokens (512 × 2,048).
|
| 108 |
+
- **Hardware:** 8× NVIDIA H100 NVL, ~1.45M tokens/s.
|
| 109 |
+
- **Data loading:** a deterministic exactly-once sampler, so every training window is used once per pass and
|
| 110 |
+
restarts resume to the exact sample.
|
| 111 |
+
|
| 112 |
+
## Evaluation (base model)
|
| 113 |
+
|
| 114 |
+
Base checkpoint (stage 3), evaluated with lm-evaluation-harness 0.4.13 at 2,048 context. All models used the
|
| 115 |
+
same prompts, shots and greedy decoding. Scores in %, Pile in bits per byte (lower is better).
|
| 116 |
+
|
| 117 |
+
| Benchmark | Shots | **NanoI1-155M** | Gemma-3-270M | Qwen3-0.6B-Base |
|
| 118 |
+
|---|---|---|---|---|
|
| 119 |
+
| HellaSwag | 10 | 38.5 | 42.0 | 53.5 |
|
| 120 |
+
| PIQA | 0 | 65.9 | 68.1 | 70.3 |
|
| 121 |
+
| WinoGrande | 5 | 52.6 | 54.0 | 59.9 |
|
| 122 |
+
| ARC-Easy | 25 | 58.9 | 59.0 | 74.9 |
|
| 123 |
+
| ARC-Challenge | 25 | 28.1 | 30.0 | 45.1 |
|
| 124 |
+
| OpenBookQA | 0 | **32.2** | 30.6 | 34.6 |
|
| 125 |
+
| RACE-High | 5 | **33.1** | 31.4 | 37.3 |
|
| 126 |
+
| TriviaQA | 5 | 10.2 | 14.3 | 20.1 |
|
| 127 |
+
| NaturalQuestions | 5 | 3.2 | 4.1 | 7.6 |
|
| 128 |
+
| MMLU-Redux | 5 | 27.8 | 28.2 | 32.1 |
|
| 129 |
+
| MMLU-Pro | 5 | 7.8 | 8.5 | 25.5 |
|
| 130 |
+
| GPQA | 5 | **26.6** | 25.9 | 29.9 |
|
| 131 |
+
| BBH | 3 | 18.4 | 22.7 | 39.4 |
|
| 132 |
+
| GSM8K | 8 | **4.0** | 1.8 | 48.9 |
|
| 133 |
+
| MATH | 4 | 2.2 | 2.4 | 29.9 |
|
| 134 |
+
| HumanEval | 0 | **9.1** | 2.4 | 29.3 |
|
| 135 |
+
| EvalPlus | 0 | **4.3** | 1.2 | 13.0 |
|
| 136 |
+
| INCLUDE (44 languages) | – | **26.0** | 25.5 | 36.3 |
|
| 137 |
+
| MMMLU | 5 | 23.6 | 25.6 | 38.7 |
|
| 138 |
+
| Pile test (BPB ↓) | – | 0.870 | 0.868 | 0.716 |
|
| 139 |
+
|
| 140 |
+
**Reading the table:**
|
| 141 |
+
- **Against Gemma-3-270M:** NanoI1 is ahead on code (HumanEval ~4×), grade-school maths, reading
|
| 142 |
+
comprehension and OpenBookQA. It is behind on commonsense benchmarks (HellaSwag, PIQA) and factual recall
|
| 143 |
+
(TriviaQA). It has fewer parameters in its transformer layers (78M vs ~100M) and saw roughly 1/6 of
|
| 144 |
+
Gemma's training tokens.
|
| 145 |
+
- **Against Qwen3-0.6B-Base:** Qwen3-0.6B, with about 4× the parameters and ~36T tokens, is stronger
|
| 146 |
+
everywhere.
|
| 147 |
+
- **Near chance:** many knowledge benchmarks (MMLU-Pro, GPQA, SuperGPQA, MMMLU) sit close to random guessing
|
| 148 |
+
for all models under 300M parameters, so their differences are small.
|
| 149 |
+
|
| 150 |
+
## Intended use
|
| 151 |
+
|
| 152 |
+
- **Good for (after fine-tuning):** structured extraction, classification, routing and short grounded Q&A over provided documents,
|
| 153 |
+
especially after fine-tuning on your own task data. Also a fast draft model, on-device assistant or research
|
| 154 |
+
baseline.
|
| 155 |
+
- **Not for:** open-ended factual questions without context, maths and multi-step reasoning, legal, medical or
|
| 156 |
+
financial advice, or any decision about a person without human review.
|
| 157 |
+
|
| 158 |
+
## Limitations
|
| 159 |
+
|
| 160 |
+
- **It invents content.** At 155M parameters the model makes up facts and can add fields that are not in the
|
| 161 |
+
source text when extracting JSON. For production extraction, use schema-constrained decoding (e.g. llguidance,
|
| 162 |
+
Outlines or XGrammar) and validate outputs.
|
| 163 |
+
- **Short context.** It was trained at 2,048 tokens; the RoPE base supports longer contexts but quality beyond
|
| 164 |
+
2k is untested.
|
| 165 |
+
- **Limited Indian-language ability.** Indian languages are a small share of training, so quality there is
|
| 166 |
+
well below English.
|
| 167 |
+
- **Bias.** The model reflects biases present in web, book and code data.
|
| 168 |
+
- **No safety training.** This base model has no instruction, preference or safety tuning. Add your own guardrails.
|
| 169 |
+
|
| 170 |
+
## Training data and licences
|
| 171 |
+
|
| 172 |
+
All third-party datasets were used under their published terms:
|
| 173 |
+
- **ODC-By:** FineWeb-Edu, FineMath, FineWeb-2, smollm-corpus (Cosmopedia v2), OpenCoder annealing corpus,
|
| 174 |
+
Dolmino mix.
|
| 175 |
+
- **The Stack v2:** per-file licences.
|
| 176 |
+
- **Institutional Books 1.0:** Institutional Data Initiative terms.
|
| 177 |
+
|
| 178 |
+
See each dataset card for details. Some anneal data (Cosmopedia, parts of FLAN and Dolmino math) was generated
|
| 179 |
+
by other language models.
|
| 180 |
+
|
| 181 |
+
## Citation
|
| 182 |
+
```bibtex
|
| 183 |
+
@techreport{kitefish2026nanoi1,
|
| 184 |
+
title = {NanoI1: A 155M-Parameter Language Model Trained on One Trillion Tokens for Indian Regulated Industries},
|
| 185 |
+
author = {Anuj and {KiteFish AI}},
|
| 186 |
+
institution = {KiteFish AI},
|
| 187 |
+
year = {2026}
|
| 188 |
+
}
|
| 189 |
+
```
|
chat.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Chat with a NanoI1 model (SFT checkpoint or final/ export).
|
| 3 |
+
|
| 4 |
+
python chat.py --ckpt runs/nanoi1-155m-sft/final/model.pt # interactive
|
| 5 |
+
python chat.py --ckpt ... --prompt "Summarise KYC in one line" # one-shot
|
| 6 |
+
python chat.py --ckpt ... --system "You are a concise banking assistant." # with a system prompt
|
| 7 |
+
|
| 8 |
+
Commands in interactive mode: /reset (new conversation), /system <text>, /exit.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import os
|
| 13 |
+
import sys
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
from chat_format import prompt_ids, sanitize, stop_ids
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@torch.no_grad()
|
| 21 |
+
def generate(model, ids, max_new_tokens=256, temperature=0.7, top_p=0.9, top_k=0,
|
| 22 |
+
repetition_penalty=1.1, stop=(), context_len=2048, stream=None):
|
| 23 |
+
"""Sample a continuation of `ids` using the model's latent KV cache. Returns new token ids."""
|
| 24 |
+
device = next(model.parameters()).device
|
| 25 |
+
ids = list(ids)[-(context_len - max_new_tokens):] if len(ids) + max_new_tokens > context_len else list(ids)
|
| 26 |
+
model.reset_cache()
|
| 27 |
+
logits = model(torch.tensor([ids], device=device), use_cache=True)[0, -1].float()
|
| 28 |
+
out = []
|
| 29 |
+
try:
|
| 30 |
+
for _ in range(max_new_tokens):
|
| 31 |
+
if repetition_penalty != 1.0 and (ids or out):
|
| 32 |
+
seen = torch.tensor(list(set(ids[-256:] + out)), device=device)
|
| 33 |
+
vals = logits[seen]
|
| 34 |
+
logits[seen] = torch.where(vals > 0, vals / repetition_penalty, vals * repetition_penalty)
|
| 35 |
+
if temperature <= 0:
|
| 36 |
+
nxt = int(logits.argmax())
|
| 37 |
+
else:
|
| 38 |
+
lg = logits / temperature
|
| 39 |
+
if top_k:
|
| 40 |
+
kth = torch.topk(lg, top_k).values[-1]
|
| 41 |
+
lg[lg < kth] = -float("inf")
|
| 42 |
+
probs = torch.softmax(lg, -1)
|
| 43 |
+
if 0 < top_p < 1:
|
| 44 |
+
sp, si = torch.sort(probs, descending=True)
|
| 45 |
+
cut = torch.cumsum(sp, 0) - sp > top_p
|
| 46 |
+
sp[cut] = 0
|
| 47 |
+
probs = torch.zeros_like(probs).scatter_(0, si, sp)
|
| 48 |
+
probs /= probs.sum()
|
| 49 |
+
nxt = int(torch.multinomial(probs, 1))
|
| 50 |
+
if nxt in stop:
|
| 51 |
+
break
|
| 52 |
+
out.append(nxt)
|
| 53 |
+
if stream:
|
| 54 |
+
stream(nxt)
|
| 55 |
+
logits = model(torch.tensor([[nxt]], device=device), use_cache=True)[0, -1].float()
|
| 56 |
+
finally:
|
| 57 |
+
model.reset_cache()
|
| 58 |
+
return out
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@torch.no_grad()
|
| 62 |
+
def generate_batch(model, prompts, max_new_tokens, stop=(), pad_id=0, until=None, tok=None):
|
| 63 |
+
"""Greedy-decode several prompts at once with the latent KV cache.
|
| 64 |
+
|
| 65 |
+
Prompts are left-padded and a key-padding mask keeps padding out of attention (RoPE is relative,
|
| 66 |
+
so the shift from left padding does not change any real token's attention). A row stops at a stop
|
| 67 |
+
token or once any `until` string appears in its decoded text (needs `tok`). Returns new ids per row."""
|
| 68 |
+
device = next(model.parameters()).device
|
| 69 |
+
B = len(prompts)
|
| 70 |
+
L = max(len(p) for p in prompts)
|
| 71 |
+
x = torch.full((B, L), pad_id, dtype=torch.long, device=device)
|
| 72 |
+
mask = torch.zeros((B, L), dtype=torch.bool, device=device)
|
| 73 |
+
for i, p in enumerate(prompts):
|
| 74 |
+
x[i, L - len(p):] = torch.tensor(p, device=device)
|
| 75 |
+
mask[i, L - len(p):] = True
|
| 76 |
+
out = [[] for _ in range(B)]
|
| 77 |
+
done = torch.zeros(B, dtype=torch.bool, device=device)
|
| 78 |
+
stop_t = torch.tensor(list(stop) or [-1], device=device)
|
| 79 |
+
model.reset_cache()
|
| 80 |
+
try:
|
| 81 |
+
logits = model(x, mask=mask, use_cache=True)[:, -1]
|
| 82 |
+
for _ in range(max_new_tokens):
|
| 83 |
+
nxt = logits.float().argmax(-1)
|
| 84 |
+
done |= torch.isin(nxt, stop_t)
|
| 85 |
+
nxt_l = nxt.tolist()
|
| 86 |
+
done_l = done.tolist()
|
| 87 |
+
for i in range(B):
|
| 88 |
+
if not done_l[i]:
|
| 89 |
+
out[i].append(nxt_l[i])
|
| 90 |
+
if until and tok is not None and any(u in tok.decode(out[i][-64:]) for u in until):
|
| 91 |
+
done[i] = True
|
| 92 |
+
if bool(done.all()):
|
| 93 |
+
break
|
| 94 |
+
mask = torch.cat([mask, torch.ones((B, 1), dtype=torch.bool, device=device)], 1)
|
| 95 |
+
logits = model(nxt[:, None], mask=mask, use_cache=True)[:, -1]
|
| 96 |
+
finally:
|
| 97 |
+
model.reset_cache()
|
| 98 |
+
return out
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def generate_reply(model, tok, messages, max_new_tokens=256, temperature=0.7, top_p=0.9,
|
| 102 |
+
repetition_penalty=1.1, stream=False):
|
| 103 |
+
ids = prompt_ids(tok, messages)
|
| 104 |
+
printer = None
|
| 105 |
+
if stream:
|
| 106 |
+
buf = []
|
| 107 |
+
|
| 108 |
+
def printer(t): # decode incrementally, print only complete UTF-8
|
| 109 |
+
buf.append(t)
|
| 110 |
+
s = tok.decode(buf)
|
| 111 |
+
if not s.endswith("�"):
|
| 112 |
+
sys.stdout.write(s)
|
| 113 |
+
sys.stdout.flush()
|
| 114 |
+
buf.clear()
|
| 115 |
+
out = generate(model, ids, max_new_tokens, temperature, top_p, 0,
|
| 116 |
+
repetition_penalty if temperature > 0 else 1.0, stop_ids(tok),
|
| 117 |
+
model.config.get("context_len", 2048) if hasattr(model, "config") else 2048, printer)
|
| 118 |
+
return tok.decode(out).strip()
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def load(ckpt, device):
|
| 122 |
+
from model import NanoI1Model
|
| 123 |
+
state = torch.load(ckpt, map_location="cpu", weights_only=False)
|
| 124 |
+
model = NanoI1Model(**state["model_config"])
|
| 125 |
+
model.load_state_dict(state["model"])
|
| 126 |
+
model.to(device).eval()
|
| 127 |
+
seq = (state.get("train_args") or {}).get("seq_len", 2048)
|
| 128 |
+
model.config = {**model.config, "context_len": min(model.config["context_len"], seq)}
|
| 129 |
+
return model
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def main():
|
| 133 |
+
ap = argparse.ArgumentParser()
|
| 134 |
+
ap.add_argument("--ckpt", required=True)
|
| 135 |
+
ap.add_argument("--tokenizer", default=None, help="default: tokenizer.json next to the checkpoint, else tokenizer/")
|
| 136 |
+
ap.add_argument("--prompt", default=None)
|
| 137 |
+
ap.add_argument("--system", default=None)
|
| 138 |
+
ap.add_argument("--max_new_tokens", type=int, default=512)
|
| 139 |
+
ap.add_argument("--temperature", type=float, default=0.7)
|
| 140 |
+
ap.add_argument("--top_p", type=float, default=0.9)
|
| 141 |
+
ap.add_argument("--repetition_penalty", type=float, default=1.1)
|
| 142 |
+
a = ap.parse_args()
|
| 143 |
+
|
| 144 |
+
from tokenizers import Tokenizer
|
| 145 |
+
tok_path = a.tokenizer or next((p for p in (os.path.join(os.path.dirname(a.ckpt), "tokenizer.json"),
|
| 146 |
+
"tokenizer/tokenizer.json") if os.path.exists(p)), None)
|
| 147 |
+
if not tok_path:
|
| 148 |
+
raise SystemExit("tokenizer.json not found: pass --tokenizer")
|
| 149 |
+
tok = Tokenizer.from_file(tok_path)
|
| 150 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 151 |
+
model = load(a.ckpt, device)
|
| 152 |
+
amp = torch.autocast("cuda", dtype=torch.bfloat16) if device.type == "cuda" else torch.autocast("cpu", enabled=False)
|
| 153 |
+
|
| 154 |
+
system = a.system
|
| 155 |
+
history = []
|
| 156 |
+
|
| 157 |
+
def ask(text):
|
| 158 |
+
msgs = ([{"role": "system", "content": sanitize(system, tok)}] if system else []) + history + \
|
| 159 |
+
[{"role": "user", "content": sanitize(text, tok)}]
|
| 160 |
+
with amp:
|
| 161 |
+
reply = generate_reply(model, tok, msgs, a.max_new_tokens, a.temperature, a.top_p,
|
| 162 |
+
a.repetition_penalty, stream=True)
|
| 163 |
+
print()
|
| 164 |
+
return reply
|
| 165 |
+
|
| 166 |
+
if a.prompt:
|
| 167 |
+
ask(a.prompt)
|
| 168 |
+
return
|
| 169 |
+
print("NanoI1 chat - /reset, /system <text>, /exit")
|
| 170 |
+
while True:
|
| 171 |
+
try:
|
| 172 |
+
text = input("\nyou> ").strip()
|
| 173 |
+
except (EOFError, KeyboardInterrupt):
|
| 174 |
+
break
|
| 175 |
+
if not text:
|
| 176 |
+
continue
|
| 177 |
+
if text == "/exit":
|
| 178 |
+
break
|
| 179 |
+
if text == "/reset":
|
| 180 |
+
history.clear()
|
| 181 |
+
print("(new conversation)")
|
| 182 |
+
continue
|
| 183 |
+
if text.startswith("/system"):
|
| 184 |
+
system = text[len("/system"):].strip() or None
|
| 185 |
+
history.clear()
|
| 186 |
+
print(f"(system prompt {'set' if system else 'cleared'}; new conversation)")
|
| 187 |
+
continue
|
| 188 |
+
sys.stdout.write("bot> ")
|
| 189 |
+
reply = ask(text)
|
| 190 |
+
history += [{"role": "user", "content": text}, {"role": "assistant", "content": reply}]
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
if __name__ == "__main__":
|
| 194 |
+
main()
|
chat_format.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NanoI1 chat format (ChatML on the special tokens reserved in the tokenizer). Every conversation
|
| 3 |
+
starts with <|endoftext|>, the same boundary token that preceded each document in pretraining:
|
| 4 |
+
|
| 5 |
+
<|endoftext|><|im_start|>system\\n{system}<|im_end|>\\n
|
| 6 |
+
<|im_start|>user\\n{question}<|im_end|>\\n
|
| 7 |
+
<|im_start|>assistant\\n{answer}<|im_end|>\\n
|
| 8 |
+
|
| 9 |
+
Training, inference (chat.py) and evaluation (evaluate.py / eval_chat.py) all go through this one
|
| 10 |
+
module, so the token sequence the model is trained on is exactly what it sees in use:
|
| 11 |
+
|
| 12 |
+
* `render(messages)` builds the text; `encode(tokenizer, messages)` tokenizes it and returns a
|
| 13 |
+
loss mask that is 1 only on assistant content and the closing <|im_end|> of assistant turns.
|
| 14 |
+
* Encoding a rendered string as a whole gives the same ids as `encode` (the tokenizer splits on
|
| 15 |
+
special tokens, and labels are assigned by character offsets), so there is no train/serve skew.
|
| 16 |
+
* Special-token strings inside user/assistant text are removed (`sanitize`) so data or users
|
| 17 |
+
cannot inject turn boundaries.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from typing import Dict, List, Optional, Tuple
|
| 21 |
+
|
| 22 |
+
IM_START, IM_END, EOT, PAD = "<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|pad|>"
|
| 23 |
+
ROLES = ("system", "user", "assistant")
|
| 24 |
+
ROLE_ALIASES = {"human": "user", "gpt": "assistant", "bot": "assistant", "model": "assistant",
|
| 25 |
+
"chatgpt": "assistant", "prompter": "user", "instruction": "user", "response": "assistant"}
|
| 26 |
+
|
| 27 |
+
_SPECIALS: List[str] = []
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _specials(tokenizer) -> List[str]:
|
| 31 |
+
global _SPECIALS
|
| 32 |
+
if not _SPECIALS:
|
| 33 |
+
vocab = tokenizer.get_vocab()
|
| 34 |
+
_SPECIALS = [t for t in vocab if t.startswith("<|") and t.endswith("|>")] + ["<think>", "</think>"]
|
| 35 |
+
return _SPECIALS
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def sanitize(text: str, tokenizer=None) -> str:
|
| 39 |
+
toks = _specials(tokenizer) if tokenizer is not None else [IM_START, IM_END, EOT, PAD]
|
| 40 |
+
for t in toks:
|
| 41 |
+
if t in text:
|
| 42 |
+
text = text.replace(t, "")
|
| 43 |
+
return text
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def normalize_role(role: str) -> Optional[str]:
|
| 47 |
+
r = (role or "").strip().lower()
|
| 48 |
+
r = ROLE_ALIASES.get(r, r)
|
| 49 |
+
return r if r in ROLES else None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def render(messages: List[Dict[str, str]], add_generation_prompt: bool = False) -> str:
|
| 53 |
+
out = [EOT]
|
| 54 |
+
for m in messages:
|
| 55 |
+
out.append(f"{IM_START}{m['role']}\n{m['content']}{IM_END}\n")
|
| 56 |
+
if add_generation_prompt:
|
| 57 |
+
out.append(f"{IM_START}assistant\n")
|
| 58 |
+
return "".join(out)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def encode(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool = False
|
| 62 |
+
) -> Tuple[List[int], List[int]]:
|
| 63 |
+
"""Token ids of render(messages) and a same-length 0/1 mask marking the assistant tokens to train on."""
|
| 64 |
+
text = render(messages, add_generation_prompt)
|
| 65 |
+
# character spans that should be learned: assistant content + its closing <|im_end|>
|
| 66 |
+
spans, pos = [], len(EOT)
|
| 67 |
+
for m in messages:
|
| 68 |
+
head = f"{IM_START}{m['role']}\n"
|
| 69 |
+
start = pos + len(head)
|
| 70 |
+
end = start + len(m["content"]) + len(IM_END)
|
| 71 |
+
if m["role"] == "assistant":
|
| 72 |
+
spans.append((start, end))
|
| 73 |
+
pos = end + 1 # trailing "\n"
|
| 74 |
+
enc = tokenizer.encode(text, add_special_tokens=False)
|
| 75 |
+
ids, mask = list(enc.ids), []
|
| 76 |
+
si = 0
|
| 77 |
+
for (a, b) in enc.offsets:
|
| 78 |
+
while si < len(spans) and spans[si][1] <= a:
|
| 79 |
+
si += 1
|
| 80 |
+
inside = si < len(spans) and a >= spans[si][0] and b <= spans[si][1] and b > a
|
| 81 |
+
mask.append(1 if inside else 0)
|
| 82 |
+
return ids, mask
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def prompt_ids(tokenizer, messages: List[Dict[str, str]]) -> List[int]:
|
| 86 |
+
"""Ids for generation: the conversation so far followed by '<|im_start|>assistant\\n'."""
|
| 87 |
+
return tokenizer.encode(render(messages, add_generation_prompt=True), add_special_tokens=False).ids
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def stop_ids(tokenizer) -> List[int]:
|
| 91 |
+
return [i for i in (tokenizer.token_to_id(IM_END), tokenizer.token_to_id(EOT)) if i is not None]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# Jinja equivalent, written into tokenizer_config.json so HF `apply_chat_template` matches exactly.
|
| 95 |
+
JINJA_TEMPLATE = (
|
| 96 |
+
"{{ '<|endoftext|>' }}"
|
| 97 |
+
"{% for message in messages %}"
|
| 98 |
+
"{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}"
|
| 99 |
+
"{% endfor %}"
|
| 100 |
+
"{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"
|
| 101 |
+
)
|
config.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "nanoi1",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"NanoI1Model"
|
| 5 |
+
],
|
| 6 |
+
"vocab_size": 50304,
|
| 7 |
+
"embed_dim": 768,
|
| 8 |
+
"context_len": 131072,
|
| 9 |
+
"num_heads": 12,
|
| 10 |
+
"head_dim": 64,
|
| 11 |
+
"qk_norm": true,
|
| 12 |
+
"latent_dim": 256,
|
| 13 |
+
"ffn_hidden_dim": 2048,
|
| 14 |
+
"transformer_layers": 12,
|
| 15 |
+
"rope_theta": 5000000,
|
| 16 |
+
"rms_norm_eps": 1e-06,
|
| 17 |
+
"tie_embeddings": false,
|
| 18 |
+
"trained_context_len": 2048,
|
| 19 |
+
"torch_dtype": "bfloat16",
|
| 20 |
+
"num_parameters": 155293056,
|
| 21 |
+
"num_non_embedding_parameters": 116659584
|
| 22 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"max_new_tokens": 512,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"temperature": 0.7,
|
| 5 |
+
"top_p": 0.9,
|
| 6 |
+
"repetition_penalty": 1.1,
|
| 7 |
+
"eos_token_id": 0,
|
| 8 |
+
"pad_token_id": 1
|
| 9 |
+
}
|
model.py
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class InputEmbedding(nn.Module):
|
| 8 |
+
def __init__(self, vocab_size, embedding_dim):
|
| 9 |
+
super().__init__()
|
| 10 |
+
self.embedding = nn.Embedding(vocab_size, embedding_dim)
|
| 11 |
+
|
| 12 |
+
def forward(self, x):
|
| 13 |
+
return self.embedding(x)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class RMSNorm(nn.Module):
|
| 17 |
+
def __init__(self, embed_dim, rms_norm_eps=1e-9):
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.rms_norm_eps = rms_norm_eps
|
| 20 |
+
self.weight = nn.Parameter(torch.ones(embed_dim))
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
variance = x.pow(2).mean(dim=-1, keepdim=True)
|
| 24 |
+
std = torch.sqrt(variance + self.rms_norm_eps)
|
| 25 |
+
x_norm = x / std
|
| 26 |
+
return x_norm * self.weight
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class RoPE(nn.Module):
|
| 30 |
+
def __init__(self, embed_dim, context_len, rope_theta):
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.context_len = context_len
|
| 33 |
+
self.embed_dim = embed_dim
|
| 34 |
+
|
| 35 |
+
N = rope_theta
|
| 36 |
+
inv_freq = (1. / N ** (torch.arange(0, embed_dim, 2).float() / embed_dim))
|
| 37 |
+
inv_freq = torch.cat((inv_freq, inv_freq), dim=-1)
|
| 38 |
+
positions = torch.arange(context_len)
|
| 39 |
+
|
| 40 |
+
rotation_angle = positions.unsqueeze(-1) * inv_freq.unsqueeze(0)
|
| 41 |
+
|
| 42 |
+
# non-persistent: recomputed at init, kept out of checkpoints
|
| 43 |
+
self.register_buffer('cos', torch.cos(rotation_angle), persistent=False)
|
| 44 |
+
self.register_buffer('sin', torch.sin(rotation_angle), persistent=False)
|
| 45 |
+
|
| 46 |
+
def forward(self, x, pos_offset=0):
|
| 47 |
+
seq_len = x.size(2)
|
| 48 |
+
x1, x2 = x.chunk(2, dim=-1)
|
| 49 |
+
|
| 50 |
+
adj_cos = self.cos[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype)
|
| 51 |
+
adj_sin = self.sin[pos_offset: pos_offset + seq_len].unsqueeze(0).unsqueeze(0).to(x.dtype)
|
| 52 |
+
|
| 53 |
+
rotate = torch.cat((-x2, x1), dim=-1)
|
| 54 |
+
|
| 55 |
+
x_adjusted = (x * adj_cos) + ( rotate * adj_sin )
|
| 56 |
+
|
| 57 |
+
return x_adjusted
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class MultiHeadLatentAttention(nn.Module):
|
| 61 |
+
def __init__(self, embed_dim, context_len, num_heads, head_dim=None, qk_norm=False, latent_dim=None,
|
| 62 |
+
rope_theta=10000, rms_norm_eps=1e-6, rope=None):
|
| 63 |
+
super().__init__()
|
| 64 |
+
self.embed_dim = embed_dim
|
| 65 |
+
self.context_len = context_len
|
| 66 |
+
self.num_heads = num_heads
|
| 67 |
+
self.qk_norm = qk_norm
|
| 68 |
+
|
| 69 |
+
if head_dim is None:
|
| 70 |
+
head_dim = embed_dim // num_heads
|
| 71 |
+
|
| 72 |
+
self.head_dim = head_dim
|
| 73 |
+
self.hidden_dim = self.head_dim * num_heads
|
| 74 |
+
|
| 75 |
+
self.latent_dim = latent_dim if latent_dim is not None else max(16, embed_dim // 8)
|
| 76 |
+
|
| 77 |
+
self.w_q = nn.Linear(embed_dim, self.hidden_dim)
|
| 78 |
+
self.w_dkv = nn.Linear(embed_dim, self.latent_dim)
|
| 79 |
+
self.w_uk = nn.Linear(self.latent_dim, self.hidden_dim)
|
| 80 |
+
self.w_uv = nn.Linear(self.latent_dim, self.hidden_dim)
|
| 81 |
+
self.w_o = nn.Linear(self.hidden_dim, embed_dim)
|
| 82 |
+
|
| 83 |
+
# one RoPE table shared by all layers (a 131072-long table per layer would waste ~770 MB)
|
| 84 |
+
self.rope = rope if rope is not None else RoPE(self.head_dim, context_len, rope_theta)
|
| 85 |
+
|
| 86 |
+
if self.qk_norm:
|
| 87 |
+
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
| 88 |
+
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
| 89 |
+
else:
|
| 90 |
+
self.q_norm = self.k_norm = None
|
| 91 |
+
|
| 92 |
+
self.register_buffer("cache_ckv", None, persistent=False)
|
| 93 |
+
self.ptr_current_pos = 0
|
| 94 |
+
|
| 95 |
+
def reset_cache(self):
|
| 96 |
+
self.cache_ckv = None
|
| 97 |
+
self.ptr_current_pos = 0
|
| 98 |
+
|
| 99 |
+
@staticmethod
|
| 100 |
+
def _get_causal_mask(seq_len, kv_len, device):
|
| 101 |
+
q_pos = torch.arange(seq_len, device=device).unsqueeze(1) + (kv_len - seq_len)
|
| 102 |
+
k_pos = torch.arange(kv_len, device=device).unsqueeze(0)
|
| 103 |
+
return (k_pos <= q_pos).float()
|
| 104 |
+
|
| 105 |
+
def forward(self, x, mask=None, use_cache=False):
|
| 106 |
+
b, seq_len, embed_dim = x.shape
|
| 107 |
+
|
| 108 |
+
q = self.w_q(x)
|
| 109 |
+
ckv = self.w_dkv(x)
|
| 110 |
+
|
| 111 |
+
if use_cache:
|
| 112 |
+
if self.cache_ckv is None:
|
| 113 |
+
updated_cache = ckv
|
| 114 |
+
else:
|
| 115 |
+
updated_cache = torch.cat((self.cache_ckv, ckv), dim=1)
|
| 116 |
+
self.cache_ckv = updated_cache
|
| 117 |
+
pos_offset = self.ptr_current_pos
|
| 118 |
+
self.ptr_current_pos += seq_len
|
| 119 |
+
else:
|
| 120 |
+
updated_cache = ckv
|
| 121 |
+
pos_offset = 0
|
| 122 |
+
|
| 123 |
+
kv_len = updated_cache.shape[1]
|
| 124 |
+
|
| 125 |
+
k = self.w_uk(updated_cache)
|
| 126 |
+
v = self.w_uv(updated_cache)
|
| 127 |
+
|
| 128 |
+
q = q.view(b, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 129 |
+
k = k.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 130 |
+
v = v.view(b, kv_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 131 |
+
|
| 132 |
+
if self.qk_norm:
|
| 133 |
+
q = self.q_norm(q)
|
| 134 |
+
k = self.k_norm(k)
|
| 135 |
+
|
| 136 |
+
q = self.rope(q, pos_offset=pos_offset)
|
| 137 |
+
k = self.rope(k, pos_offset=0)
|
| 138 |
+
|
| 139 |
+
# attn_score = (q @ k.transpose(2, 3)) / math.sqrt(self.head_dim)
|
| 140 |
+
# if mask is None:
|
| 141 |
+
# mask = self._get_causal_mask(seq_len, kv_len, x.device)
|
| 142 |
+
# attn_score = attn_score.masked_fill(mask == 0, -1e9)
|
| 143 |
+
# weights = F.softmax(attn_score, dim=-1)
|
| 144 |
+
#
|
| 145 |
+
# context_vec = torch.matmul(weights, v).transpose(1, 2).contiguous()
|
| 146 |
+
|
| 147 |
+
if mask is None and seq_len == kv_len:
|
| 148 |
+
context_vec = F.scaled_dot_product_attention(
|
| 149 |
+
q, k, v,
|
| 150 |
+
attn_mask=None,
|
| 151 |
+
is_causal=True,
|
| 152 |
+
dropout_p=0.0,
|
| 153 |
+
)
|
| 154 |
+
else:
|
| 155 |
+
attn_mask = self._get_causal_mask(seq_len, kv_len, x.device).bool()
|
| 156 |
+
if mask is not None:
|
| 157 |
+
# mask: [b, kv_len] key-padding mask, True = real token (batched generation with left padding).
|
| 158 |
+
# Every query may also see itself, so rows that are all padding stay finite (never NaN);
|
| 159 |
+
# real queries never attend to pad keys, so padding cannot leak into real outputs.
|
| 160 |
+
q_pos = torch.arange(seq_len, device=x.device).unsqueeze(1) + (kv_len - seq_len)
|
| 161 |
+
k_pos = torch.arange(kv_len, device=x.device).unsqueeze(0)
|
| 162 |
+
attn_mask = (attn_mask[None, None] & mask[:, None, None, :].bool()) | (k_pos == q_pos)[None, None]
|
| 163 |
+
context_vec = F.scaled_dot_product_attention(
|
| 164 |
+
q, k, v,
|
| 165 |
+
attn_mask=attn_mask,
|
| 166 |
+
is_causal=False,
|
| 167 |
+
dropout_p=0.0,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
context_vec = context_vec.transpose(1, 2).contiguous()
|
| 171 |
+
context_vec = context_vec.view(b, seq_len, self.hidden_dim)
|
| 172 |
+
|
| 173 |
+
logits = self.w_o(context_vec)
|
| 174 |
+
return logits
|
| 175 |
+
|
| 176 |
+
class SkipConnection(nn.Module):
|
| 177 |
+
def __init__(self):
|
| 178 |
+
super().__init__()
|
| 179 |
+
|
| 180 |
+
def forward(self, x1, x2):
|
| 181 |
+
return x1+x2
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class FeedForwardLayer(nn.Module):
|
| 185 |
+
def __init__(self, embed_dim, hidden_dim):
|
| 186 |
+
super().__init__()
|
| 187 |
+
self.fc1 = nn.Linear(embed_dim, hidden_dim)
|
| 188 |
+
self.fc2 = nn.Linear(embed_dim, hidden_dim)
|
| 189 |
+
self.fc3 = nn.Linear(hidden_dim, embed_dim)
|
| 190 |
+
|
| 191 |
+
def forward(self, x):
|
| 192 |
+
up = self.fc1(x)
|
| 193 |
+
gate = self.fc2(x)
|
| 194 |
+
|
| 195 |
+
down = F.silu(up) * gate
|
| 196 |
+
output = self.fc3(down)
|
| 197 |
+
return output
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
class OutputLinearLayer(nn.Module):
|
| 201 |
+
def __init__(self, embedding_dim, vocab_size):
|
| 202 |
+
super().__init__()
|
| 203 |
+
self.linear = nn.Linear(embedding_dim, vocab_size)
|
| 204 |
+
|
| 205 |
+
def forward(self, x):
|
| 206 |
+
return self.linear(x)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
class TransformerBlock(nn.Module):
|
| 210 |
+
def __init__(self, embed_dim,
|
| 211 |
+
context_len,
|
| 212 |
+
num_heads,
|
| 213 |
+
head_dim=None,
|
| 214 |
+
qk_norm=False,
|
| 215 |
+
latent_dim=None,
|
| 216 |
+
ffn_hidden_dim=None,
|
| 217 |
+
rope_theta=10000,
|
| 218 |
+
rms_norm_eps=1e-6,
|
| 219 |
+
rope=None):
|
| 220 |
+
super().__init__()
|
| 221 |
+
self.embed_dim = embed_dim
|
| 222 |
+
self.num_heads = num_heads
|
| 223 |
+
|
| 224 |
+
if ffn_hidden_dim is None:
|
| 225 |
+
ffn_hidden_dim = embed_dim
|
| 226 |
+
|
| 227 |
+
self.rms_norm1 = RMSNorm(embed_dim, rms_norm_eps)
|
| 228 |
+
self.mla_block = MultiHeadLatentAttention(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim,
|
| 229 |
+
rope_theta, rms_norm_eps, rope)
|
| 230 |
+
self.skip_connection = SkipConnection()
|
| 231 |
+
self.rms_norm2 = RMSNorm(embed_dim, rms_norm_eps)
|
| 232 |
+
self.ffn = FeedForwardLayer(embed_dim, ffn_hidden_dim)
|
| 233 |
+
|
| 234 |
+
def forward(self, x, mask=None, use_cache=False):
|
| 235 |
+
residual_1 = x
|
| 236 |
+
x = self.rms_norm1(x)
|
| 237 |
+
x = self.mla_block(x, mask, use_cache)
|
| 238 |
+
|
| 239 |
+
x = self.skip_connection(x, residual_1)
|
| 240 |
+
|
| 241 |
+
residual_2 = x
|
| 242 |
+
x = self.rms_norm2(x)
|
| 243 |
+
x = self.ffn(x)
|
| 244 |
+
|
| 245 |
+
x = self.skip_connection(x, residual_2)
|
| 246 |
+
return x
|
| 247 |
+
|
| 248 |
+
class NanoI1Model(nn.Module):
|
| 249 |
+
def __init__(self, vocab_size,
|
| 250 |
+
embed_dim,
|
| 251 |
+
context_len,
|
| 252 |
+
num_heads,
|
| 253 |
+
head_dim=None,
|
| 254 |
+
qk_norm=False,
|
| 255 |
+
latent_dim=None,
|
| 256 |
+
ffn_hidden_dim=None,
|
| 257 |
+
transformer_layers=4,
|
| 258 |
+
rope_theta=10000,
|
| 259 |
+
rms_norm_eps=1e-6,
|
| 260 |
+
tie_embeddings=False):
|
| 261 |
+
super().__init__()
|
| 262 |
+
self.config = dict(vocab_size=vocab_size, embed_dim=embed_dim, context_len=context_len,
|
| 263 |
+
num_heads=num_heads, head_dim=head_dim, qk_norm=qk_norm, latent_dim=latent_dim,
|
| 264 |
+
ffn_hidden_dim=ffn_hidden_dim, transformer_layers=transformer_layers,
|
| 265 |
+
rope_theta=rope_theta, rms_norm_eps=rms_norm_eps, tie_embeddings=tie_embeddings)
|
| 266 |
+
hd = head_dim if head_dim is not None else embed_dim // num_heads
|
| 267 |
+
self.rope = RoPE(hd, context_len, rope_theta)
|
| 268 |
+
|
| 269 |
+
self.input_embedding = nn.Embedding(vocab_size, embed_dim)
|
| 270 |
+
self.transformer_blocks = nn.ModuleList([
|
| 271 |
+
TransformerBlock(embed_dim, context_len, num_heads, head_dim, qk_norm, latent_dim, ffn_hidden_dim,
|
| 272 |
+
rope_theta, rms_norm_eps, self.rope) for _ in range(transformer_layers)
|
| 273 |
+
])
|
| 274 |
+
|
| 275 |
+
self.final_norm = RMSNorm(embed_dim, rms_norm_eps)
|
| 276 |
+
self.output_linear = OutputLinearLayer(embed_dim, vocab_size)
|
| 277 |
+
if tie_embeddings:
|
| 278 |
+
self.output_linear.linear.weight = self.input_embedding.weight
|
| 279 |
+
|
| 280 |
+
self.apply(self._init_weights)
|
| 281 |
+
# GPT-2 / OLMo-style scaled init for projections that write into the residual stream
|
| 282 |
+
std_res = 0.02 / math.sqrt(2 * transformer_layers)
|
| 283 |
+
for blk in self.transformer_blocks:
|
| 284 |
+
nn.init.normal_(blk.mla_block.w_o.weight, mean=0.0, std=std_res)
|
| 285 |
+
nn.init.normal_(blk.ffn.fc3.weight, mean=0.0, std=std_res)
|
| 286 |
+
|
| 287 |
+
@staticmethod
|
| 288 |
+
def _init_weights(m):
|
| 289 |
+
if isinstance(m, nn.Linear):
|
| 290 |
+
nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
| 291 |
+
if m.bias is not None:
|
| 292 |
+
nn.init.zeros_(m.bias)
|
| 293 |
+
elif isinstance(m, nn.Embedding):
|
| 294 |
+
nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
| 295 |
+
|
| 296 |
+
@classmethod
|
| 297 |
+
def from_config(cls, path_or_dict, **overrides):
|
| 298 |
+
cfg = load_config(path_or_dict) if isinstance(path_or_dict, str) else dict(path_or_dict)
|
| 299 |
+
cfg.update(overrides)
|
| 300 |
+
return cls(**cfg)
|
| 301 |
+
|
| 302 |
+
def num_params(self, non_embedding=True):
|
| 303 |
+
n = sum(p.numel() for p in self.parameters())
|
| 304 |
+
if non_embedding:
|
| 305 |
+
n -= self.input_embedding.weight.numel()
|
| 306 |
+
return n
|
| 307 |
+
|
| 308 |
+
def reset_cache(self):
|
| 309 |
+
for blk in self.transformer_blocks:
|
| 310 |
+
blk.mla_block.reset_cache()
|
| 311 |
+
|
| 312 |
+
def forward(self, x, mask=None, use_cache=False, targets=None, reduction="mean"):
|
| 313 |
+
"""Returns logits; or, when `targets` is given, the next-token loss.
|
| 314 |
+
Targets equal to -100 are ignored (SFT: only assistant tokens are learned), and the output
|
| 315 |
+
projection is computed only for the kept positions. With self.fused_loss set (fused_loss.py)
|
| 316 |
+
the full logits are never materialised. reduction: 'mean' over kept tokens, or 'sum'."""
|
| 317 |
+
x = self.input_embedding(x)
|
| 318 |
+
for block in self.transformer_blocks:
|
| 319 |
+
x = block(x, mask, use_cache)
|
| 320 |
+
x = self.final_norm(x)
|
| 321 |
+
if targets is not None:
|
| 322 |
+
return self._loss(x, targets, reduction)
|
| 323 |
+
x = self.output_linear(x)
|
| 324 |
+
return x
|
| 325 |
+
|
| 326 |
+
@torch.compiler.disable
|
| 327 |
+
def _loss(self, h, targets, reduction):
|
| 328 |
+
h = h.reshape(-1, h.size(-1))
|
| 329 |
+
t = targets.reshape(-1)
|
| 330 |
+
keep = t != -100
|
| 331 |
+
if not bool(keep.all()):
|
| 332 |
+
h, t = h[keep], t[keep]
|
| 333 |
+
lin = self.output_linear.linear
|
| 334 |
+
if t.numel() == 0: # nothing to learn in this batch: zero loss that still touches every param (DDP)
|
| 335 |
+
zero = h.sum() * 0.0 + lin.weight.sum() * 0.0
|
| 336 |
+
return zero + lin.bias.sum() * 0.0 if lin.bias is not None else zero
|
| 337 |
+
fused = getattr(self, "fused_loss", None)
|
| 338 |
+
if fused is not None:
|
| 339 |
+
loss = fused(lin.weight, h, t, lin.bias)
|
| 340 |
+
else:
|
| 341 |
+
loss = F.cross_entropy(self.output_linear(h).float(), t)
|
| 342 |
+
return loss * t.numel() if reduction == "sum" else loss
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def load_config(path):
|
| 346 |
+
"""config.json loader that tolerates // line comments."""
|
| 347 |
+
import json
|
| 348 |
+
with open(path) as f:
|
| 349 |
+
text = "\n".join(l for l in f if not l.lstrip().startswith("//"))
|
| 350 |
+
return json.loads(text)
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4ece7a181cb85e205bee56e41002946f8cded8abed78e154a1816dbb7c96c04e
|
| 3 |
+
size 310613312
|
modeling_nanoi1.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Load NanoI1 from a Hugging Face repo or a local folder exported by export_hf.py.
|
| 3 |
+
|
| 4 |
+
from modeling_nanoi1 import from_pretrained, chat
|
| 5 |
+
model, tok = from_pretrained("KiteFishAI/NanoI1-155M-Instruct") # or a local folder
|
| 6 |
+
print(chat(model, tok, [{"role": "user", "content": "Explain KYC in two sentences."}]))
|
| 7 |
+
|
| 8 |
+
Base model: use model(...) for logits, or chat.generate(...) on plain token ids.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _local_dir(name_or_path, revision=None):
|
| 18 |
+
if os.path.isdir(name_or_path):
|
| 19 |
+
return name_or_path
|
| 20 |
+
from huggingface_hub import snapshot_download
|
| 21 |
+
return snapshot_download(name_or_path, revision=revision,
|
| 22 |
+
allow_patterns=["*.json", "*.safetensors", "*.py"])
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def from_pretrained(name_or_path, device=None, dtype=None, revision=None):
|
| 26 |
+
"""Returns (model in eval mode, tokenizers.Tokenizer)."""
|
| 27 |
+
from safetensors.torch import load_file
|
| 28 |
+
from tokenizers import Tokenizer
|
| 29 |
+
from model import NanoI1Model
|
| 30 |
+
|
| 31 |
+
d = _local_dir(name_or_path, revision)
|
| 32 |
+
cfg = json.load(open(os.path.join(d, "config.json")))
|
| 33 |
+
keys = ("vocab_size", "embed_dim", "num_heads", "transformer_layers", "head_dim", "context_len", "qk_norm",
|
| 34 |
+
"ffn_hidden_dim", "latent_dim", "rope_theta", "rms_norm_eps", "tie_embeddings")
|
| 35 |
+
model = NanoI1Model(**{k: cfg[k] for k in keys if k in cfg})
|
| 36 |
+
model.load_state_dict(load_file(os.path.join(d, "model.safetensors")))
|
| 37 |
+
device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
|
| 38 |
+
dtype = dtype or (torch.bfloat16 if device.type == "cuda" else torch.float32)
|
| 39 |
+
model.to(device=device, dtype=dtype).eval()
|
| 40 |
+
# generate within the context length the model was trained at
|
| 41 |
+
model.config = {**model.config, "context_len": min(cfg["context_len"], cfg.get("trained_context_len", 2048))}
|
| 42 |
+
return model, Tokenizer.from_file(os.path.join(d, "tokenizer.json"))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@torch.no_grad()
|
| 46 |
+
def chat(model, tok, messages, max_new_tokens=512, temperature=0.7, top_p=0.9, repetition_penalty=1.1):
|
| 47 |
+
"""messages: [{"role": "system"|"user"|"assistant", "content": str}, ...]. temperature=0 -> greedy."""
|
| 48 |
+
from chat import generate_reply
|
| 49 |
+
from chat_format import sanitize
|
| 50 |
+
msgs = [{**m, "content": sanitize(m["content"], tok)} for m in messages]
|
| 51 |
+
return generate_reply(model, tok, msgs, max_new_tokens, temperature, top_p, repetition_penalty)
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"bos_token": "<|bos|>",
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
+
"pad_token": "<|pad|>",
|
| 6 |
+
"unk_token": null,
|
| 7 |
+
"model_max_length": 2048,
|
| 8 |
+
"add_bos_token": false,
|
| 9 |
+
"add_eos_token": false,
|
| 10 |
+
"clean_up_tokenization_spaces": false
|
| 11 |
+
}
|