AdvaithMagic commited on
Commit
a15230d
·
verified ·
1 Parent(s): 73ff74b

Export TinyIndianLM checkpoint in HF format

Browse files
README.md ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AdvaithMagic/tinyindianlm-ml
2
+
3
+ Custom TinyIndianLM checkpoint exported in Hugging Face repository format.
4
+
5
+ ## Loading
6
+
7
+ ```python
8
+ from transformers import AutoModelForCausalLM, AutoTokenizer
9
+
10
+ model = AutoModelForCausalLM.from_pretrained(
11
+ "AdvaithMagic/tinyindianlm-ml",
12
+ trust_remote_code=True,
13
+ )
14
+ tokenizer = AutoTokenizer.from_pretrained("AdvaithMagic/tinyindianlm-ml")
15
+ ```
16
+
17
+ Note: this model uses custom architecture code included in this repository,
18
+ so `trust_remote_code=True` is required.
__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from .configuration_tinyindianlm import TinyIndianLMConfig
2
+ from .modeling_tinyindianlm import TinyIndianLMForCausalLM
3
+
4
+ __all__ = ["TinyIndianLMConfig", "TinyIndianLMForCausalLM"]
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "tinyindianlm",
3
+ "architectures": [
4
+ "TinyIndianLMForCausalLM"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_tinyindianlm.TinyIndianLMConfig",
8
+ "AutoModelForCausalLM": "modeling_tinyindianlm.TinyIndianLMForCausalLM"
9
+ },
10
+ "vocab_size": 4192,
11
+ "d_model": 512,
12
+ "n_layers": 4,
13
+ "n_heads": 8,
14
+ "n_kv_heads": 4,
15
+ "max_seq_len": 768,
16
+ "d_ffn": 1408,
17
+ "dropout": 0.0,
18
+ "rope_theta": 10000.0,
19
+ "pad_token_id": 0,
20
+ "unk_token_id": 1,
21
+ "bos_token_id": 2,
22
+ "eos_token_id": 3,
23
+ "torch_dtype": "float32",
24
+ "transformers_version": "4.0.0"
25
+ }
configuration_tinyindianlm.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+
4
+ class TinyIndianLMConfig(PretrainedConfig):
5
+ model_type = "tinyindianlm"
6
+
7
+ def __init__(
8
+ self,
9
+ vocab_size=16000,
10
+ d_model=512,
11
+ n_layers=4,
12
+ n_heads=8,
13
+ n_kv_heads=4,
14
+ max_seq_len=512,
15
+ d_ffn=1408,
16
+ dropout=0.0,
17
+ rope_theta=10000.0,
18
+ pad_token_id=0,
19
+ unk_token_id=1,
20
+ bos_token_id=2,
21
+ eos_token_id=3,
22
+ **kwargs,
23
+ ):
24
+ self.vocab_size = vocab_size
25
+ self.d_model = d_model
26
+ self.n_layers = n_layers
27
+ self.n_heads = n_heads
28
+ self.n_kv_heads = n_kv_heads
29
+ self.max_seq_len = max_seq_len
30
+ self.d_ffn = d_ffn
31
+ self.dropout = dropout
32
+ self.rope_theta = rope_theta
33
+ super().__init__(
34
+ pad_token_id=pad_token_id,
35
+ unk_token_id=unk_token_id,
36
+ bos_token_id=bos_token_id,
37
+ eos_token_id=eos_token_id,
38
+ **kwargs,
39
+ )
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "eos_token_id": 3,
4
+ "pad_token_id": 0,
5
+ "do_sample": false,
6
+ "temperature": 1.0,
7
+ "top_k": 50,
8
+ "top_p": 1.0
9
+ }
model.py ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Decoder-only Transformer with RoPE positional encoding.
3
+ Target: ~30M parameters.
4
+
5
+ Architecture choices (informed by MobileLLM paper + Vizuara results):
6
+ - d_model = 512, n_layers = 4 → ~30M params
7
+ - n_heads = 8, head_dim = 64
8
+ - FFN: SwiGLU activation (d_ffn = 4 * d_model, but gated so 2/3 effective)
9
+ Actually: d_ffn = int(2/3 * 4 * d_model) rounded to nearest 64 → 1408
10
+ - RoPE positional encoding (no learned position embeddings)
11
+ - RMSNorm (no bias, more stable than LayerNorm for small models)
12
+ - No dropout during training (small model on small data, dropout hurts)
13
+ - Causal (autoregressive) mask
14
+
15
+ Parameter count breakdown (vocab=16000, d=512, layers=4):
16
+ Embedding: 16000 × 512 = 8.19M
17
+ Each layer:
18
+ Attention: 4 × 512 × 512 = 1.05M
19
+ FFN: 512×1408 + 1408×512 + 512×1408 = ~2.16M
20
+ Norms: 2 × 512 negligible
21
+ 4 layers: 4 × 3.21M = 12.84M
22
+ Output head: tied to embedding = 0M (weight tying)
23
+ TOTAL: ~21M (with weight tying) → add unembedding = ~29M without
24
+
25
+ With weight tying (output = embedding.T): ~21M ← we use this
26
+ This is standard practice for small models (GPT-2 style).
27
+ """
28
+
29
+ import math
30
+ import torch
31
+ import torch.nn as nn
32
+ import torch.nn.functional as F
33
+ from dataclasses import dataclass
34
+
35
+
36
+ # ── Config ────────────────────────────────────────────────────────────────────
37
+
38
+ @dataclass
39
+ class ModelConfig:
40
+ vocab_size: int = 16000
41
+ d_model: int = 512
42
+ n_layers: int = 4
43
+ n_heads: int = 8
44
+ n_kv_heads: int = 4 # GQA: 4 KV heads, 8 Q heads (reduces params)
45
+ max_seq_len: int = 512
46
+ # FFN hidden dim: SwiGLU convention = 2/3 * 4 * d_model, rounded to 64
47
+ d_ffn: int = 1408 # = round(2/3 * 4 * 512 / 64) * 64
48
+
49
+ # Regularization
50
+ dropout: float = 0.0 # set to 0.1 for fine-tuning if needed
51
+
52
+ # RoPE
53
+ rope_theta: float = 10000.0
54
+
55
+ def __post_init__(self):
56
+ assert self.d_model % self.n_heads == 0
57
+ assert self.n_heads % self.n_kv_heads == 0
58
+ self.head_dim = self.d_model // self.n_heads
59
+ self.n_rep = self.n_heads // self.n_kv_heads # for GQA repeat
60
+
61
+
62
+ # ── RoPE ─────────────────────────────────────────────────────────────────────
63
+
64
+ def precompute_rope_freqs(head_dim: int, max_seq_len: int, theta: float = 10000.0):
65
+ """
66
+ Precompute RoPE frequency tensor.
67
+ Returns: (max_seq_len, head_dim//2) complex tensor.
68
+ """
69
+ freqs = 1.0 / (
70
+ theta ** (torch.arange(0, head_dim, 2).float() / head_dim)
71
+ )
72
+ t = torch.arange(max_seq_len)
73
+ freqs = torch.outer(t, freqs)
74
+ return torch.polar(torch.ones_like(freqs), freqs) # complex
75
+
76
+
77
+ def apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
78
+ """
79
+ Apply RoPE to query or key tensor.
80
+ x: (batch, seq_len, n_heads, head_dim)
81
+ freqs: (seq_len, head_dim//2) complex
82
+ """
83
+ # Reshape to pairs for complex multiplication
84
+ x_r = x.float().reshape(*x.shape[:-1], -1, 2)
85
+ x_c = torch.view_as_complex(x_r)
86
+ freqs = freqs[:x.shape[1]].unsqueeze(0).unsqueeze(2) # (1, seq, 1, dim//2)
87
+ x_out = torch.view_as_real(x_c * freqs).flatten(-2)
88
+ return x_out.type_as(x)
89
+
90
+
91
+ # ── RMSNorm ───────────────────────────────────────────────────────────────────
92
+
93
+ class RMSNorm(nn.Module):
94
+ def __init__(self, d_model: int, eps: float = 1e-6):
95
+ super().__init__()
96
+ self.weight = nn.Parameter(torch.ones(d_model))
97
+ self.eps = eps
98
+
99
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
100
+ norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
101
+ return norm * self.weight
102
+
103
+
104
+ # ── Attention ─────────────────────────────────────────────────────────────────
105
+
106
+ class GroupedQueryAttention(nn.Module):
107
+ def __init__(self, cfg: ModelConfig):
108
+ super().__init__()
109
+ self.n_heads = cfg.n_heads
110
+ self.n_kv_heads = cfg.n_kv_heads
111
+ self.n_rep = cfg.n_rep
112
+ self.head_dim = cfg.head_dim
113
+ self.d_model = cfg.d_model
114
+
115
+ self.Wq = nn.Linear(cfg.d_model, cfg.n_heads * cfg.head_dim, bias=False)
116
+ self.Wk = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
117
+ self.Wv = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=False)
118
+ self.Wo = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model, bias=False)
119
+
120
+ self.dropout = nn.Dropout(cfg.dropout)
121
+
122
+ def forward(
123
+ self,
124
+ x: torch.Tensor, # (B, T, d_model)
125
+ freqs: torch.Tensor, # (T, head_dim//2) complex
126
+ mask: torch.Tensor | None = None, # (T, T) causal mask
127
+ ) -> torch.Tensor:
128
+ B, T, _ = x.shape
129
+
130
+ q = self.Wq(x).view(B, T, self.n_heads, self.head_dim)
131
+ k = self.Wk(x).view(B, T, self.n_kv_heads, self.head_dim)
132
+ v = self.Wv(x).view(B, T, self.n_kv_heads, self.head_dim)
133
+
134
+ # RoPE
135
+ q = apply_rope(q, freqs)
136
+ k = apply_rope(k, freqs)
137
+
138
+ # GQA: repeat K/V to match Q heads
139
+ if self.n_rep > 1:
140
+ k = k.repeat_interleave(self.n_rep, dim=2)
141
+ v = v.repeat_interleave(self.n_rep, dim=2)
142
+
143
+ # Attention: (B, n_heads, T, head_dim)
144
+ q = q.transpose(1, 2)
145
+ k = k.transpose(1, 2)
146
+ v = v.transpose(1, 2)
147
+
148
+ # Use PyTorch's flash attention when available (much faster on GPU)
149
+ if hasattr(F, "scaled_dot_product_attention"):
150
+ # is_causal=True handles the mask automatically and uses FlashAttention
151
+ out = F.scaled_dot_product_attention(
152
+ q, k, v,
153
+ attn_mask=None,
154
+ dropout_p=self.dropout.p if self.training else 0.0,
155
+ is_causal=True,
156
+ )
157
+ else:
158
+ scale = self.head_dim ** -0.5
159
+ scores = torch.matmul(q, k.transpose(-2, -1)) * scale
160
+ if mask is not None:
161
+ scores = scores + mask
162
+ scores = F.softmax(scores.float(), dim=-1).type_as(q)
163
+ scores = self.dropout(scores)
164
+ out = torch.matmul(scores, v)
165
+
166
+ # Merge heads
167
+ out = out.transpose(1, 2).contiguous().view(B, T, -1)
168
+ return self.Wo(out)
169
+
170
+
171
+ # ── SwiGLU FFN ────────────────────────────────────────────────────────────────
172
+
173
+ class SwiGLUFFN(nn.Module):
174
+ def __init__(self, cfg: ModelConfig):
175
+ super().__init__()
176
+ self.gate = nn.Linear(cfg.d_model, cfg.d_ffn, bias=False)
177
+ self.up = nn.Linear(cfg.d_model, cfg.d_ffn, bias=False)
178
+ self.down = nn.Linear(cfg.d_ffn, cfg.d_model, bias=False)
179
+ self.dropout = nn.Dropout(cfg.dropout)
180
+
181
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
182
+ return self.dropout(self.down(F.silu(self.gate(x)) * self.up(x)))
183
+
184
+
185
+ # ── Transformer Block ─────────────────────────────────────────────────────────
186
+
187
+ class TransformerBlock(nn.Module):
188
+ def __init__(self, cfg: ModelConfig):
189
+ super().__init__()
190
+ self.attn_norm = RMSNorm(cfg.d_model)
191
+ self.attn = GroupedQueryAttention(cfg)
192
+ self.ffn_norm = RMSNorm(cfg.d_model)
193
+ self.ffn = SwiGLUFFN(cfg)
194
+
195
+ def forward(
196
+ self,
197
+ x: torch.Tensor,
198
+ freqs: torch.Tensor,
199
+ mask: torch.Tensor | None = None,
200
+ ) -> torch.Tensor:
201
+ # Pre-norm (LLaMA style)
202
+ x = x + self.attn(self.attn_norm(x), freqs, mask)
203
+ x = x + self.ffn(self.ffn_norm(x))
204
+ return x
205
+
206
+
207
+ # ── Full Model ────────────────────────────────────────────────────────────────
208
+
209
+ class TinyIndianLM(nn.Module):
210
+ def __init__(self, cfg: ModelConfig):
211
+ super().__init__()
212
+ self.cfg = cfg
213
+
214
+ self.embedding = nn.Embedding(cfg.vocab_size, cfg.d_model, padding_idx=0)
215
+ self.layers = nn.ModuleList([TransformerBlock(cfg) for _ in range(cfg.n_layers)])
216
+ self.norm = RMSNorm(cfg.d_model)
217
+ self.output = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)
218
+
219
+ # Weight tying: output projection shares weights with embedding
220
+ self.output.weight = self.embedding.weight
221
+
222
+ # Precompute RoPE frequencies (register as buffer → moves to device)
223
+ freqs = precompute_rope_freqs(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
224
+ self.register_buffer("rope_freqs", freqs, persistent=False)
225
+
226
+ # Causal mask (optional fallback when not using F.scaled_dot_product_attention)
227
+ mask = torch.full((cfg.max_seq_len, cfg.max_seq_len), float("-inf"))
228
+ mask = torch.triu(mask, diagonal=1)
229
+ self.register_buffer("causal_mask", mask, persistent=False)
230
+
231
+ # Init weights
232
+ self.apply(self._init_weights)
233
+ # Scale residual projections (GPT-2 style)
234
+ for pn, p in self.named_parameters():
235
+ if pn.endswith(("Wo.weight", "down.weight")):
236
+ nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * cfg.n_layers))
237
+
238
+ def _init_weights(self, module):
239
+ if isinstance(module, nn.Linear):
240
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
241
+ if module.bias is not None:
242
+ nn.init.zeros_(module.bias)
243
+ elif isinstance(module, nn.Embedding):
244
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
245
+
246
+ def forward(
247
+ self,
248
+ input_ids: torch.Tensor, # (B, T)
249
+ targets: torch.Tensor | None = None, # (B, T) for training
250
+ pad_id: int = 0,
251
+ ) -> tuple[torch.Tensor, torch.Tensor | None]:
252
+ B, T = input_ids.shape
253
+ assert T <= self.cfg.max_seq_len, f"Sequence length {T} > max {self.cfg.max_seq_len}"
254
+
255
+ x = self.embedding(input_ids) # (B, T, d_model)
256
+ freqs = self.rope_freqs[:T]
257
+
258
+ for layer in self.layers:
259
+ x = layer(x, freqs)
260
+
261
+ x = self.norm(x)
262
+ logits = self.output(x) # (B, T, vocab_size)
263
+
264
+ loss = None
265
+ if targets is not None:
266
+ # Shift: predict token[i+1] from token[i]
267
+ # input: [BOS, t1, t2, ..., tN, EOS]
268
+ # targets: [t1, t2, ..., tN, EOS, PAD]
269
+ # But we already have aligned input/target from dataloader
270
+ # Mask out PAD tokens in the loss
271
+ loss = F.cross_entropy(
272
+ logits.view(-1, self.cfg.vocab_size),
273
+ targets.view(-1),
274
+ ignore_index=pad_id,
275
+ )
276
+
277
+ return logits, loss
278
+
279
+ @torch.no_grad()
280
+ def generate(
281
+ self,
282
+ input_ids: torch.Tensor, # (1, T) prompt
283
+ max_new_tokens: int = 200,
284
+ temperature: float = 1.0,
285
+ top_k: int = 50,
286
+ eos_id: int = 3,
287
+ pad_id: int = 0,
288
+ ) -> list[int]:
289
+ self.eval()
290
+ generated = input_ids.tolist()[0]
291
+
292
+ for _ in range(max_new_tokens):
293
+ ids_tensor = torch.tensor([generated], device=input_ids.device)
294
+ # Truncate to max_seq_len
295
+ if ids_tensor.shape[1] > self.cfg.max_seq_len:
296
+ ids_tensor = ids_tensor[:, -self.cfg.max_seq_len:]
297
+
298
+ logits, _ = self.forward(ids_tensor)
299
+ logits = logits[0, -1, :] / temperature # (vocab_size,)
300
+
301
+ # Remove PAD from generation
302
+ logits[pad_id] = float("-inf")
303
+
304
+ # Top-k sampling
305
+ if top_k > 0:
306
+ top_vals, _ = torch.topk(logits, min(top_k, logits.size(-1)))
307
+ logits[logits < top_vals[-1]] = float("-inf")
308
+
309
+ probs = F.softmax(logits, dim=-1)
310
+ next_id = torch.multinomial(probs, num_samples=1).item()
311
+ generated.append(next_id)
312
+
313
+ if next_id == eos_id:
314
+ break
315
+
316
+ return generated
317
+
318
+ def num_parameters(self, exclude_embeddings: bool = False) -> int:
319
+ if exclude_embeddings:
320
+ return sum(p.numel() for n, p in self.named_parameters()
321
+ if "embedding" not in n and p.requires_grad)
322
+ return sum(p.numel() for p in self.parameters() if p.requires_grad)
323
+
324
+
325
+ # ── Quick test ────────────────────────────────────────────────────────────────
326
+
327
+ if __name__ == "__main__":
328
+ cfg = ModelConfig()
329
+ model = TinyIndianLM(cfg)
330
+ total = model.num_parameters()
331
+ print(f"Model config: d_model={cfg.d_model}, n_layers={cfg.n_layers}, "
332
+ f"n_heads={cfg.n_heads}, d_ffn={cfg.d_ffn}")
333
+ print(f"Total parameters: {total:,} ({total/1e6:.1f}M)")
334
+
335
+ # Forward pass test
336
+ B, T = 2, 64
337
+ x = torch.randint(0, cfg.vocab_size, (B, T))
338
+ logits, loss = model(x, targets=x)
339
+ print(f"Logits shape: {logits.shape}")
340
+ print(f"Initial loss (should be ~ln({cfg.vocab_size})={math.log(cfg.vocab_size):.2f}): {loss.item():.4f}")
modeling_tinyindianlm.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from transformers import PreTrainedModel
4
+ from transformers.modeling_outputs import CausalLMOutputWithPast
5
+
6
+ from .configuration_tinyindianlm import TinyIndianLMConfig
7
+ from .model import ModelConfig, TinyIndianLM
8
+
9
+
10
+ class TinyIndianLMForCausalLM(PreTrainedModel):
11
+ config_class = TinyIndianLMConfig
12
+ base_model_prefix = "model"
13
+ supports_gradient_checkpointing = False
14
+
15
+ def __init__(self, config: TinyIndianLMConfig):
16
+ super().__init__(config)
17
+ internal_cfg = ModelConfig(
18
+ vocab_size=config.vocab_size,
19
+ d_model=config.d_model,
20
+ n_layers=config.n_layers,
21
+ n_heads=config.n_heads,
22
+ n_kv_heads=config.n_kv_heads,
23
+ max_seq_len=config.max_seq_len,
24
+ d_ffn=config.d_ffn,
25
+ dropout=config.dropout,
26
+ rope_theta=config.rope_theta,
27
+ )
28
+ self.model = TinyIndianLM(internal_cfg)
29
+ self.post_init()
30
+
31
+ def get_input_embeddings(self) -> nn.Embedding:
32
+ return self.model.embedding
33
+
34
+ def set_input_embeddings(self, value: nn.Embedding) -> None:
35
+ self.model.embedding = value
36
+
37
+ def get_output_embeddings(self) -> nn.Linear:
38
+ return self.model.output
39
+
40
+ def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
41
+ self.model.output = new_embeddings
42
+
43
+ def forward(
44
+ self,
45
+ input_ids=None,
46
+ attention_mask=None,
47
+ labels=None,
48
+ **kwargs,
49
+ ):
50
+ del attention_mask, kwargs
51
+
52
+ # Safety guard for exported repos where tokenizer vocab and
53
+ # model embedding size diverge. Any out-of-range token id is
54
+ # mapped to unk to avoid embedding lookup crashes.
55
+ if input_ids is not None:
56
+ vocab_size = self.model.embedding.num_embeddings
57
+ if (input_ids >= vocab_size).any():
58
+ unk_id = getattr(self.config, "unk_token_id", 1)
59
+ input_ids = input_ids.clone()
60
+ input_ids[input_ids >= vocab_size] = unk_id
61
+
62
+ logits, loss = self.model(
63
+ input_ids,
64
+ targets=labels,
65
+ pad_id=self.config.pad_token_id,
66
+ )
67
+ return CausalLMOutputWithPast(
68
+ loss=loss,
69
+ logits=logits,
70
+ past_key_values=None,
71
+ hidden_states=None,
72
+ attentions=None,
73
+ )
74
+
75
+ def prepare_inputs_for_generation(self, input_ids, **kwargs):
76
+ del kwargs
77
+ return {"input_ids": input_ids}
78
+
79
+ @staticmethod
80
+ def _reorder_cache(past_key_values, beam_idx):
81
+ del beam_idx
82
+ return past_key_values
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c90da8ac2854c499ad33ebc7e5eb479fdc31691d1267522b73883d46bc6dce0b
3
+ size 55802169
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 4192,
3
+ "special_tokens": [
4
+ "<pad>",
5
+ "<unk>",
6
+ "<bos>",
7
+ "<eos>"
8
+ ],
9
+ "pad_id": 0,
10
+ "unk_id": 1,
11
+ "bos_id": 2,
12
+ "eos_id": 3,
13
+ "train_ratio": 0.8,
14
+ "seed": 42,
15
+ "languages": [
16
+ "ml"
17
+ ],
18
+ "normalizer": "NFC",
19
+ "pre_tokenizer": "ByteLevel",
20
+ "recommended_seq_len": 768,
21
+ "fertility_per_lang": {
22
+ "ml": {
23
+ "mean_fertility": 0.6979,
24
+ "mean_tokens": 391.4,
25
+ "p50_tokens": 394,
26
+ "p90_tokens": 576,
27
+ "p95_tokens": 688,
28
+ "p99_tokens": 1086,
29
+ "max_tokens": 1316
30
+ }
31
+ },
32
+ "token_overlap": {
33
+ "pairwise": {},
34
+ "family_overlap": {
35
+ "indo_aryan_vs_indo_aryan": {
36
+ "shared_tokens": 0,
37
+ "jaccard_similarity": 0
38
+ },
39
+ "dravidian_vs_indo_aryan": {
40
+ "shared_tokens": 0,
41
+ "jaccard_similarity": 0.0
42
+ },
43
+ "dravidian_vs_dravidian": {
44
+ "shared_tokens": 3628,
45
+ "jaccard_similarity": 1.0
46
+ }
47
+ },
48
+ "per_lang_unique_tokens": {
49
+ "ml": 3628
50
+ }
51
+ },
52
+ "vocab_utilization": {
53
+ "vocab_size": 4192,
54
+ "used_tokens": 3628,
55
+ "unused_tokens": 564,
56
+ "utilization_rate": 0.8655,
57
+ "shared_all_langs": 3628,
58
+ "unique_per_lang": {
59
+ "ml": 3628
60
+ }
61
+ },
62
+ "oov_rates": {
63
+ "ml": {
64
+ "total_tokens": 880083,
65
+ "unk_tokens": 0,
66
+ "oov_rate": 0.0
67
+ }
68
+ }
69
+ }