Export TinyIndianLM checkpoint in HF format
Browse files- README.md +18 -0
- __init__.py +4 -0
- config.json +25 -0
- configuration_tinyindianlm.py +39 -0
- generation_config.json +9 -0
- model.py +340 -0
- modeling_tinyindianlm.py +82 -0
- pytorch_model.bin +3 -0
- tokenizer.json +0 -0
- tokenizer_config.json +69 -0
README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AdvaithMagic/tinyindianlm-te
|
| 2 |
+
|
| 3 |
+
Custom TinyIndianLM checkpoint exported in Hugging Face repository format.
|
| 4 |
+
|
| 5 |
+
## Loading
|
| 6 |
+
|
| 7 |
+
```python
|
| 8 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 9 |
+
|
| 10 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 11 |
+
"AdvaithMagic/tinyindianlm-te",
|
| 12 |
+
trust_remote_code=True,
|
| 13 |
+
)
|
| 14 |
+
tokenizer = AutoTokenizer.from_pretrained("AdvaithMagic/tinyindianlm-te")
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
Note: this model uses custom architecture code included in this repository,
|
| 18 |
+
so `trust_remote_code=True` is required.
|
__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .configuration_tinyindianlm import TinyIndianLMConfig
|
| 2 |
+
from .modeling_tinyindianlm import TinyIndianLMForCausalLM
|
| 3 |
+
|
| 4 |
+
__all__ = ["TinyIndianLMConfig", "TinyIndianLMForCausalLM"]
|
config.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "tinyindianlm",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"TinyIndianLMForCausalLM"
|
| 5 |
+
],
|
| 6 |
+
"auto_map": {
|
| 7 |
+
"AutoConfig": "configuration_tinyindianlm.TinyIndianLMConfig",
|
| 8 |
+
"AutoModelForCausalLM": "modeling_tinyindianlm.TinyIndianLMForCausalLM"
|
| 9 |
+
},
|
| 10 |
+
"vocab_size": 4192,
|
| 11 |
+
"d_model": 512,
|
| 12 |
+
"n_layers": 4,
|
| 13 |
+
"n_heads": 8,
|
| 14 |
+
"n_kv_heads": 4,
|
| 15 |
+
"max_seq_len": 768,
|
| 16 |
+
"d_ffn": 1408,
|
| 17 |
+
"dropout": 0.0,
|
| 18 |
+
"rope_theta": 10000.0,
|
| 19 |
+
"pad_token_id": 0,
|
| 20 |
+
"unk_token_id": 1,
|
| 21 |
+
"bos_token_id": 2,
|
| 22 |
+
"eos_token_id": 3,
|
| 23 |
+
"torch_dtype": "float32",
|
| 24 |
+
"transformers_version": "4.0.0"
|
| 25 |
+
}
|
configuration_tinyindianlm.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import PretrainedConfig
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class TinyIndianLMConfig(PretrainedConfig):
|
| 5 |
+
model_type = "tinyindianlm"
|
| 6 |
+
|
| 7 |
+
def __init__(
|
| 8 |
+
self,
|
| 9 |
+
vocab_size=16000,
|
| 10 |
+
d_model=512,
|
| 11 |
+
n_layers=4,
|
| 12 |
+
n_heads=8,
|
| 13 |
+
n_kv_heads=4,
|
| 14 |
+
max_seq_len=512,
|
| 15 |
+
d_ffn=1408,
|
| 16 |
+
dropout=0.0,
|
| 17 |
+
rope_theta=10000.0,
|
| 18 |
+
pad_token_id=0,
|
| 19 |
+
unk_token_id=1,
|
| 20 |
+
bos_token_id=2,
|
| 21 |
+
eos_token_id=3,
|
| 22 |
+
**kwargs,
|
| 23 |
+
):
|
| 24 |
+
self.vocab_size = vocab_size
|
| 25 |
+
self.d_model = d_model
|
| 26 |
+
self.n_layers = n_layers
|
| 27 |
+
self.n_heads = n_heads
|
| 28 |
+
self.n_kv_heads = n_kv_heads
|
| 29 |
+
self.max_seq_len = max_seq_len
|
| 30 |
+
self.d_ffn = d_ffn
|
| 31 |
+
self.dropout = dropout
|
| 32 |
+
self.rope_theta = rope_theta
|
| 33 |
+
super().__init__(
|
| 34 |
+
pad_token_id=pad_token_id,
|
| 35 |
+
unk_token_id=unk_token_id,
|
| 36 |
+
bos_token_id=bos_token_id,
|
| 37 |
+
eos_token_id=eos_token_id,
|
| 38 |
+
**kwargs,
|
| 39 |
+
)
|
generation_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 2,
|
| 3 |
+
"eos_token_id": 3,
|
| 4 |
+
"pad_token_id": 0,
|
| 5 |
+
"do_sample": false,
|
| 6 |
+
"temperature": 1.0,
|
| 7 |
+
"top_k": 50,
|
| 8 |
+
"top_p": 1.0
|
| 9 |
+
}
|
model.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Decoder-only Transformer with RoPE positional encoding.
|
| 3 |
+
Target: ~30M parameters.
|
| 4 |
+
|
| 5 |
+
Architecture choices (informed by MobileLLM paper + Vizuara results):
|
| 6 |
+
- d_model = 512, n_layers = 4 → ~30M params
|
| 7 |
+
- n_heads = 8, head_dim = 64
|
| 8 |
+
- FFN: SwiGLU activation (d_ffn = 4 * d_model, but gated so 2/3 effective)
|
| 9 |
+
Actually: d_ffn = int(2/3 * 4 * d_model) rounded to nearest 64 → 1408
|
| 10 |
+
- RoPE positional encoding (no learned position embeddings)
|
| 11 |
+
- RMSNorm (no bias, more stable than LayerNorm for small models)
|
| 12 |
+
- No dropout during training (small model on small data, dropout hurts)
|
| 13 |
+
- Causal (autoregressive) mask
|
| 14 |
+
|
| 15 |
+
Parameter count breakdown (vocab=16000, d=512, layers=4):
|
| 16 |
+
Embedding: 16000 × 512 = 8.19M
|
| 17 |
+
Each layer:
|
| 18 |
+
Attention: 4 × 512 × 512 = 1.05M
|
| 19 |
+
FFN: 512×1408 + 1408×512 + 512×1408 = ~2.16M
|
| 20 |
+
Norms: 2 × 512 negligible
|
| 21 |
+
4 layers: 4 × 3.21M = 12.84M
|
| 22 |
+
Output head: tied to embedding = 0M (weight tying)
|
| 23 |
+
TOTAL: ~21M (with weight tying) → add unembedding = ~29M without
|
| 24 |
+
|
| 25 |
+
With weight tying (output = embedding.T): ~21M ← we use this
|
| 26 |
+
This is standard practice for small models (GPT-2 style).
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
import math
|
| 30 |
+
import torch
|
| 31 |
+
import torch.nn as nn
|
| 32 |
+
import torch.nn.functional as F
|
| 33 |
+
from dataclasses import dataclass
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
# ── Config ────────────────────────────────────────────────────────────────────
|
| 37 |
+
|
| 38 |
+
@dataclass
|
| 39 |
+
class ModelConfig:
|
| 40 |
+
vocab_size: int = 16000
|
| 41 |
+
d_model: int = 512
|
| 42 |
+
n_layers: int = 4
|
| 43 |
+
n_heads: int = 8
|
| 44 |
+
n_kv_heads: int = 4 # GQA: 4 KV heads, 8 Q heads (reduces params)
|
| 45 |
+
max_seq_len: int = 512
|
| 46 |
+
# FFN hidden dim: SwiGLU convention = 2/3 * 4 * d_model, rounded to 64
|
| 47 |
+
d_ffn: int = 1408 # = round(2/3 * 4 * 512 / 64) * 64
|
| 48 |
+
|
| 49 |
+
# Regularization
|
| 50 |
+
dropout: float = 0.0 # set to 0.1 for fine-tuning if needed
|
| 51 |
+
|
| 52 |
+
# RoPE
|
| 53 |
+
rope_theta: float = 10000.0
|
| 54 |
+
|
| 55 |
+
def __post_init__(self):
|
| 56 |
+
assert self.d_model % self.n_heads == 0
|
| 57 |
+
assert self.n_heads % self.n_kv_heads == 0
|
| 58 |
+
self.head_dim = self.d_model // self.n_heads
|
| 59 |
+
self.n_rep = self.n_heads // self.n_kv_heads # for GQA repeat
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# ── RoPE ─────────────────────────────────────────────────────────────────────
|
| 63 |
+
|
| 64 |
+
def precompute_rope_freqs(head_dim: int, max_seq_len: int, theta: float = 10000.0):
|
| 65 |
+
"""
|
| 66 |
+
Precompute RoPE frequency tensor.
|
| 67 |
+
Returns: (max_seq_len, head_dim//2) complex tensor.
|
| 68 |
+
"""
|
| 69 |
+
freqs = 1.0 / (
|
| 70 |
+
theta ** (torch.arange(0, head_dim, 2).float() / head_dim)
|
| 71 |
+
)
|
| 72 |
+
t = torch.arange(max_seq_len)
|
| 73 |
+
freqs = torch.outer(t, freqs)
|
| 74 |
+
return torch.polar(torch.ones_like(freqs), freqs) # complex
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
| 78 |
+
"""
|
| 79 |
+
Apply RoPE to query or key tensor.
|
| 80 |
+
x: (batch, seq_len, n_heads, head_dim)
|
| 81 |
+
freqs: (seq_len, head_dim//2) complex
|
| 82 |
+
"""
|
| 83 |
+
# Reshape to pairs for complex multiplication
|
| 84 |
+
x_r = x.float().reshape(*x.shape[:-1], -1, 2)
|
| 85 |
+
x_c = torch.view_as_complex(x_r)
|
| 86 |
+
freqs = freqs[:x.shape[1]].unsqueeze(0).unsqueeze(2) # (1, seq, 1, dim//2)
|
| 87 |
+
x_out = torch.view_as_real(x_c * freqs).flatten(-2)
|
| 88 |
+
return x_out.type_as(x)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# ── RMSNorm ───────────────────────────────────────────────────────────────────
|
| 92 |
+
|
| 93 |
+
class RMSNorm(nn.Module):
|
| 94 |
+
def __init__(self, d_model: int, eps: float = 1e-6):
|
| 95 |
+
super().__init__()
|
| 96 |
+
self.weight = nn.Parameter(torch.ones(d_model))
|
| 97 |
+
self.eps = eps
|
| 98 |
+
|
| 99 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 100 |
+
norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 101 |
+
return norm * self.weight
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
# ── Attention ─────────────────────────────────────────────────────────────────
|
| 105 |
+
|
| 106 |
+
class GroupedQueryAttention(nn.Module):
|
| 107 |
+
def __init__(self, cfg: ModelConfig):
|
| 108 |
+
super().__init__()
|
| 109 |
+
self.n_heads = cfg.n_heads
|
| 110 |
+
self.n_kv_heads = cfg.n_kv_heads
|
| 111 |
+
self.n_rep = cfg.n_rep
|
| 112 |
+
self.head_dim = cfg.head_dim
|
| 113 |
+
self.d_model = cfg.d_model
|
| 114 |
+
|
| 115 |
+
self.Wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False)
|
| 116 |
+
self.Wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
|
| 117 |
+
self.Wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
|
| 118 |
+
self.Wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False)
|
| 119 |
+
|
| 120 |
+
self.dropout = nn.Dropout(cfg.dropout)
|
| 121 |
+
|
| 122 |
+
def forward(
|
| 123 |
+
self,
|
| 124 |
+
x: torch.Tensor, # (B, T, d_model)
|
| 125 |
+
freqs: torch.Tensor, # (T, head_dim//2) complex
|
| 126 |
+
mask: torch.Tensor | None = None, # (T, T) causal mask
|
| 127 |
+
) -> torch.Tensor:
|
| 128 |
+
B, T, _ = x.shape
|
| 129 |
+
|
| 130 |
+
q = self.Wq(x).view(B, T, self.n_heads, self.head_dim)
|
| 131 |
+
k = self.Wk(x).view(B, T, self.n_kv_heads, self.head_dim)
|
| 132 |
+
v = self.Wv(x).view(B, T, self.n_kv_heads, self.head_dim)
|
| 133 |
+
|
| 134 |
+
# RoPE
|
| 135 |
+
q = apply_rope(q, freqs)
|
| 136 |
+
k = apply_rope(k, freqs)
|
| 137 |
+
|
| 138 |
+
# GQA: repeat K/V to match Q heads
|
| 139 |
+
if self.n_rep > 1:
|
| 140 |
+
k = k.repeat_interleave(self.n_rep, dim=2)
|
| 141 |
+
v = v.repeat_interleave(self.n_rep, dim=2)
|
| 142 |
+
|
| 143 |
+
# Attention: (B, n_heads, T, head_dim)
|
| 144 |
+
q = q.transpose(1, 2)
|
| 145 |
+
k = k.transpose(1, 2)
|
| 146 |
+
v = v.transpose(1, 2)
|
| 147 |
+
|
| 148 |
+
# Use PyTorch's flash attention when available (much faster on GPU)
|
| 149 |
+
if hasattr(F, "scaled_dot_product_attention"):
|
| 150 |
+
# is_causal=True handles the mask automatically and uses FlashAttention
|
| 151 |
+
out = F.scaled_dot_product_attention(
|
| 152 |
+
q, k, v,
|
| 153 |
+
attn_mask=None,
|
| 154 |
+
dropout_p=self.dropout.p if self.training else 0.0,
|
| 155 |
+
is_causal=True,
|
| 156 |
+
)
|
| 157 |
+
else:
|
| 158 |
+
scale = self.head_dim ** -0.5
|
| 159 |
+
scores = torch.matmul(q, k.transpose(-2, -1)) * scale
|
| 160 |
+
if mask is not None:
|
| 161 |
+
scores = scores + mask
|
| 162 |
+
scores = F.softmax(scores.float(), dim=-1).type_as(q)
|
| 163 |
+
scores = self.dropout(scores)
|
| 164 |
+
out = torch.matmul(scores, v)
|
| 165 |
+
|
| 166 |
+
# Merge heads
|
| 167 |
+
out = out.transpose(1, 2).contiguous().view(B, T, -1)
|
| 168 |
+
return self.Wo(out)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
# ── SwiGLU FFN ────────────────────────────────────────────────────────────────
|
| 172 |
+
|
| 173 |
+
class SwiGLUFFN(nn.Module):
|
| 174 |
+
def __init__(self, cfg: ModelConfig):
|
| 175 |
+
super().__init__()
|
| 176 |
+
self.gate = nn.Linear(cfg.d_model, cfg.d_ffn, bias=False)
|
| 177 |
+
self.up = nn.Linear(cfg.d_model, cfg.d_ffn, bias=False)
|
| 178 |
+
self.down = nn.Linear(cfg.d_ffn, cfg.d_model, bias=False)
|
| 179 |
+
self.dropout = nn.Dropout(cfg.dropout)
|
| 180 |
+
|
| 181 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 182 |
+
return self.dropout(self.down(F.silu(self.gate(x)) * self.up(x)))
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
# ── Transformer Block ─────────────────────────────────────────────────────────
|
| 186 |
+
|
| 187 |
+
class TransformerBlock(nn.Module):
|
| 188 |
+
def __init__(self, cfg: ModelConfig):
|
| 189 |
+
super().__init__()
|
| 190 |
+
self.attn_norm = RMSNorm(cfg.d_model)
|
| 191 |
+
self.attn = GroupedQueryAttention(cfg)
|
| 192 |
+
self.ffn_norm = RMSNorm(cfg.d_model)
|
| 193 |
+
self.ffn = SwiGLUFFN(cfg)
|
| 194 |
+
|
| 195 |
+
def forward(
|
| 196 |
+
self,
|
| 197 |
+
x: torch.Tensor,
|
| 198 |
+
freqs: torch.Tensor,
|
| 199 |
+
mask: torch.Tensor | None = None,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
# Pre-norm (LLaMA style)
|
| 202 |
+
x = x + self.attn(self.attn_norm(x), freqs, mask)
|
| 203 |
+
x = x + self.ffn(self.ffn_norm(x))
|
| 204 |
+
return x
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ── Full Model ────────────────────────────────────────────────────────────────
|
| 208 |
+
|
| 209 |
+
class TinyIndianLM(nn.Module):
|
| 210 |
+
def __init__(self, cfg: ModelConfig):
|
| 211 |
+
super().__init__()
|
| 212 |
+
self.cfg = cfg
|
| 213 |
+
|
| 214 |
+
self.embedding = nn.Embedding(cfg.vocab_size, cfg.d_model, padding_idx=0)
|
| 215 |
+
self.layers = nn.ModuleList([TransformerBlock(cfg) for _ in range(cfg.n_layers)])
|
| 216 |
+
self.norm = RMSNorm(cfg.d_model)
|
| 217 |
+
self.output = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)
|
| 218 |
+
|
| 219 |
+
# Weight tying: output projection shares weights with embedding
|
| 220 |
+
self.output.weight = self.embedding.weight
|
| 221 |
+
|
| 222 |
+
# Precompute RoPE frequencies (register as buffer → moves to device)
|
| 223 |
+
freqs = precompute_rope_freqs(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
|
| 224 |
+
self.register_buffer("rope_freqs", freqs, persistent=False)
|
| 225 |
+
|
| 226 |
+
# Causal mask (optional fallback when not using F.scaled_dot_product_attention)
|
| 227 |
+
mask = torch.full((cfg.max_seq_len, cfg.max_seq_len), float("-inf"))
|
| 228 |
+
mask = torch.triu(mask, diagonal=1)
|
| 229 |
+
self.register_buffer("causal_mask", mask, persistent=False)
|
| 230 |
+
|
| 231 |
+
# Init weights
|
| 232 |
+
self.apply(self._init_weights)
|
| 233 |
+
# Scale residual projections (GPT-2 style)
|
| 234 |
+
for pn, p in self.named_parameters():
|
| 235 |
+
if pn.endswith(("Wo.weight", "down.weight")):
|
| 236 |
+
nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * cfg.n_layers))
|
| 237 |
+
|
| 238 |
+
def _init_weights(self, module):
|
| 239 |
+
if isinstance(module, nn.Linear):
|
| 240 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 241 |
+
if module.bias is not None:
|
| 242 |
+
nn.init.zeros_(module.bias)
|
| 243 |
+
elif isinstance(module, nn.Embedding):
|
| 244 |
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
| 245 |
+
|
| 246 |
+
def forward(
|
| 247 |
+
self,
|
| 248 |
+
input_ids: torch.Tensor, # (B, T)
|
| 249 |
+
targets: torch.Tensor | None = None, # (B, T) for training
|
| 250 |
+
pad_id: int = 0,
|
| 251 |
+
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
| 252 |
+
B, T = input_ids.shape
|
| 253 |
+
assert T <= self.cfg.max_seq_len, f"Sequence length {T} > max {self.cfg.max_seq_len}"
|
| 254 |
+
|
| 255 |
+
x = self.embedding(input_ids) # (B, T, d_model)
|
| 256 |
+
freqs = self.rope_freqs[:T]
|
| 257 |
+
|
| 258 |
+
for layer in self.layers:
|
| 259 |
+
x = layer(x, freqs)
|
| 260 |
+
|
| 261 |
+
x = self.norm(x)
|
| 262 |
+
logits = self.output(x) # (B, T, vocab_size)
|
| 263 |
+
|
| 264 |
+
loss = None
|
| 265 |
+
if targets is not None:
|
| 266 |
+
# Shift: predict token[i+1] from token[i]
|
| 267 |
+
# input: [BOS, t1, t2, ..., tN, EOS]
|
| 268 |
+
# targets: [t1, t2, ..., tN, EOS, PAD]
|
| 269 |
+
# But we already have aligned input/target from dataloader
|
| 270 |
+
# Mask out PAD tokens in the loss
|
| 271 |
+
loss = F.cross_entropy(
|
| 272 |
+
logits.view(-1, self.cfg.vocab_size),
|
| 273 |
+
targets.view(-1),
|
| 274 |
+
ignore_index=pad_id,
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
return logits, loss
|
| 278 |
+
|
| 279 |
+
@torch.no_grad()
|
| 280 |
+
def generate(
|
| 281 |
+
self,
|
| 282 |
+
input_ids: torch.Tensor, # (1, T) prompt
|
| 283 |
+
max_new_tokens: int = 200,
|
| 284 |
+
temperature: float = 1.0,
|
| 285 |
+
top_k: int = 50,
|
| 286 |
+
eos_id: int = 3,
|
| 287 |
+
pad_id: int = 0,
|
| 288 |
+
) -> list[int]:
|
| 289 |
+
self.eval()
|
| 290 |
+
generated = input_ids.tolist()[0]
|
| 291 |
+
|
| 292 |
+
for _ in range(max_new_tokens):
|
| 293 |
+
ids_tensor = torch.tensor([generated], device=input_ids.device)
|
| 294 |
+
# Truncate to max_seq_len
|
| 295 |
+
if ids_tensor.shape[1] > self.cfg.max_seq_len:
|
| 296 |
+
ids_tensor = ids_tensor[:, -self.cfg.max_seq_len:]
|
| 297 |
+
|
| 298 |
+
logits, _ = self.forward(ids_tensor)
|
| 299 |
+
logits = logits[0, -1, :] / temperature # (vocab_size,)
|
| 300 |
+
|
| 301 |
+
# Remove PAD from generation
|
| 302 |
+
logits[pad_id] = float("-inf")
|
| 303 |
+
|
| 304 |
+
# Top-k sampling
|
| 305 |
+
if top_k > 0:
|
| 306 |
+
top_vals, _ = torch.topk(logits, min(top_k, logits.size(-1)))
|
| 307 |
+
logits[logits < top_vals[-1]] = float("-inf")
|
| 308 |
+
|
| 309 |
+
probs = F.softmax(logits, dim=-1)
|
| 310 |
+
next_id = torch.multinomial(probs, num_samples=1).item()
|
| 311 |
+
generated.append(next_id)
|
| 312 |
+
|
| 313 |
+
if next_id == eos_id:
|
| 314 |
+
break
|
| 315 |
+
|
| 316 |
+
return generated
|
| 317 |
+
|
| 318 |
+
def num_parameters(self, exclude_embeddings: bool = False) -> int:
|
| 319 |
+
if exclude_embeddings:
|
| 320 |
+
return sum(p.numel() for n, p in self.named_parameters()
|
| 321 |
+
if "embedding" not in n and p.requires_grad)
|
| 322 |
+
return sum(p.numel() for p in self.parameters() if p.requires_grad)
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
# ── Quick test ────────────────────────────────────────────────────────────────
|
| 326 |
+
|
| 327 |
+
if __name__ == "__main__":
|
| 328 |
+
cfg = ModelConfig()
|
| 329 |
+
model = TinyIndianLM(cfg)
|
| 330 |
+
total = model.num_parameters()
|
| 331 |
+
print(f"Model config: d_model={cfg.d_model}, n_layers={cfg.n_layers}, "
|
| 332 |
+
f"n_heads={cfg.n_heads}, d_ffn={cfg.d_ffn}")
|
| 333 |
+
print(f"Total parameters: {total:,} ({total/1e6:.1f}M)")
|
| 334 |
+
|
| 335 |
+
# Forward pass test
|
| 336 |
+
B, T = 2, 64
|
| 337 |
+
x = torch.randint(0, cfg.vocab_size, (B, T))
|
| 338 |
+
logits, loss = model(x, targets=x)
|
| 339 |
+
print(f"Logits shape: {logits.shape}")
|
| 340 |
+
print(f"Initial loss (should be ~ln({cfg.vocab_size})={math.log(cfg.vocab_size):.2f}): {loss.item():.4f}")
|
modeling_tinyindianlm.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from transformers import PreTrainedModel
|
| 4 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 5 |
+
|
| 6 |
+
from .configuration_tinyindianlm import TinyIndianLMConfig
|
| 7 |
+
from .model import ModelConfig, TinyIndianLM
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class TinyIndianLMForCausalLM(PreTrainedModel):
|
| 11 |
+
config_class = TinyIndianLMConfig
|
| 12 |
+
base_model_prefix = "model"
|
| 13 |
+
supports_gradient_checkpointing = False
|
| 14 |
+
|
| 15 |
+
def __init__(self, config: TinyIndianLMConfig):
|
| 16 |
+
super().__init__(config)
|
| 17 |
+
internal_cfg = ModelConfig(
|
| 18 |
+
vocab_size=config.vocab_size,
|
| 19 |
+
d_model=config.d_model,
|
| 20 |
+
n_layers=config.n_layers,
|
| 21 |
+
n_heads=config.n_heads,
|
| 22 |
+
n_kv_heads=config.n_kv_heads,
|
| 23 |
+
max_seq_len=config.max_seq_len,
|
| 24 |
+
d_ffn=config.d_ffn,
|
| 25 |
+
dropout=config.dropout,
|
| 26 |
+
rope_theta=config.rope_theta,
|
| 27 |
+
)
|
| 28 |
+
self.model = TinyIndianLM(internal_cfg)
|
| 29 |
+
self.post_init()
|
| 30 |
+
|
| 31 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 32 |
+
return self.model.embedding
|
| 33 |
+
|
| 34 |
+
def set_input_embeddings(self, value: nn.Embedding) -> None:
|
| 35 |
+
self.model.embedding = value
|
| 36 |
+
|
| 37 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 38 |
+
return self.model.output
|
| 39 |
+
|
| 40 |
+
def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
|
| 41 |
+
self.model.output = new_embeddings
|
| 42 |
+
|
| 43 |
+
def forward(
|
| 44 |
+
self,
|
| 45 |
+
input_ids=None,
|
| 46 |
+
attention_mask=None,
|
| 47 |
+
labels=None,
|
| 48 |
+
**kwargs,
|
| 49 |
+
):
|
| 50 |
+
del attention_mask, kwargs
|
| 51 |
+
|
| 52 |
+
# Safety guard for exported repos where tokenizer vocab and
|
| 53 |
+
# model embedding size diverge. Any out-of-range token id is
|
| 54 |
+
# mapped to unk to avoid embedding lookup crashes.
|
| 55 |
+
if input_ids is not None:
|
| 56 |
+
vocab_size = self.model.embedding.num_embeddings
|
| 57 |
+
if (input_ids >= vocab_size).any():
|
| 58 |
+
unk_id = getattr(self.config, "unk_token_id", 1)
|
| 59 |
+
input_ids = input_ids.clone()
|
| 60 |
+
input_ids[input_ids >= vocab_size] = unk_id
|
| 61 |
+
|
| 62 |
+
logits, loss = self.model(
|
| 63 |
+
input_ids,
|
| 64 |
+
targets=labels,
|
| 65 |
+
pad_id=self.config.pad_token_id,
|
| 66 |
+
)
|
| 67 |
+
return CausalLMOutputWithPast(
|
| 68 |
+
loss=loss,
|
| 69 |
+
logits=logits,
|
| 70 |
+
past_key_values=None,
|
| 71 |
+
hidden_states=None,
|
| 72 |
+
attentions=None,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
def prepare_inputs_for_generation(self, input_ids, **kwargs):
|
| 76 |
+
del kwargs
|
| 77 |
+
return {"input_ids": input_ids}
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def _reorder_cache(past_key_values, beam_idx):
|
| 81 |
+
del beam_idx
|
| 82 |
+
return past_key_values
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:008aab4469016fe595f003d9157ac37a9f35ccec09bc33df148e56dbf47fa26e
|
| 3 |
+
size 55802169
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"vocab_size": 4192,
|
| 3 |
+
"special_tokens": [
|
| 4 |
+
"<pad>",
|
| 5 |
+
"<unk>",
|
| 6 |
+
"<bos>",
|
| 7 |
+
"<eos>"
|
| 8 |
+
],
|
| 9 |
+
"pad_id": 0,
|
| 10 |
+
"unk_id": 1,
|
| 11 |
+
"bos_id": 2,
|
| 12 |
+
"eos_id": 3,
|
| 13 |
+
"train_ratio": 0.8,
|
| 14 |
+
"seed": 42,
|
| 15 |
+
"languages": [
|
| 16 |
+
"ml"
|
| 17 |
+
],
|
| 18 |
+
"normalizer": "NFC",
|
| 19 |
+
"pre_tokenizer": "ByteLevel",
|
| 20 |
+
"recommended_seq_len": 768,
|
| 21 |
+
"fertility_per_lang": {
|
| 22 |
+
"ml": {
|
| 23 |
+
"mean_fertility": 0.6979,
|
| 24 |
+
"mean_tokens": 391.4,
|
| 25 |
+
"p50_tokens": 394,
|
| 26 |
+
"p90_tokens": 576,
|
| 27 |
+
"p95_tokens": 688,
|
| 28 |
+
"p99_tokens": 1086,
|
| 29 |
+
"max_tokens": 1316
|
| 30 |
+
}
|
| 31 |
+
},
|
| 32 |
+
"token_overlap": {
|
| 33 |
+
"pairwise": {},
|
| 34 |
+
"family_overlap": {
|
| 35 |
+
"indo_aryan_vs_indo_aryan": {
|
| 36 |
+
"shared_tokens": 0,
|
| 37 |
+
"jaccard_similarity": 0
|
| 38 |
+
},
|
| 39 |
+
"dravidian_vs_indo_aryan": {
|
| 40 |
+
"shared_tokens": 0,
|
| 41 |
+
"jaccard_similarity": 0.0
|
| 42 |
+
},
|
| 43 |
+
"dravidian_vs_dravidian": {
|
| 44 |
+
"shared_tokens": 3628,
|
| 45 |
+
"jaccard_similarity": 1.0
|
| 46 |
+
}
|
| 47 |
+
},
|
| 48 |
+
"per_lang_unique_tokens": {
|
| 49 |
+
"ml": 3628
|
| 50 |
+
}
|
| 51 |
+
},
|
| 52 |
+
"vocab_utilization": {
|
| 53 |
+
"vocab_size": 4192,
|
| 54 |
+
"used_tokens": 3628,
|
| 55 |
+
"unused_tokens": 564,
|
| 56 |
+
"utilization_rate": 0.8655,
|
| 57 |
+
"shared_all_langs": 3628,
|
| 58 |
+
"unique_per_lang": {
|
| 59 |
+
"ml": 3628
|
| 60 |
+
}
|
| 61 |
+
},
|
| 62 |
+
"oov_rates": {
|
| 63 |
+
"ml": {
|
| 64 |
+
"total_tokens": 880083,
|
| 65 |
+
"unk_tokens": 0,
|
| 66 |
+
"oov_rate": 0.0
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
}
|