mikecovlee commited on
Commit
7a9602f
·
verified ·
1 Parent(s): 60ba37f

Add TinyMixtral v3.0 (477.5M total / 276.1M active MoE)

Browse files
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Michael Lee (李登淳)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
chat_template.jinja ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% for message in messages %}
2
+ {% if message['role'] == 'user' %}
3
+ {{ '<|user|>
4
+ ' + message['content'] + eos_token }}
5
+ {% elif message['role'] == 'system' %}
6
+ {{ '<|system|>
7
+ ' + message['content'] + eos_token }}
8
+ {% elif message['role'] == 'assistant' %}
9
+ {{ '<|assistant|>
10
+ ' + message['content'] + eos_token }}
11
+ {% endif %}
12
+ {% if loop.last and add_generation_prompt %}
13
+ {{ '<|assistant|>' }}
14
+ {% endif %}
15
+ {% endfor %}
config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "TinyMixtralForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "dtype": "float32",
7
+ "eos_token_id": 2,
8
+ "expert_intermediate_size": 2048,
9
+ "head_dim": 64,
10
+ "hidden_size": 1024,
11
+ "initializer_range": 0.02,
12
+ "max_position_embeddings": 2048,
13
+ "model_type": "tinymixtral",
14
+ "num_attention_heads": 16,
15
+ "num_experts_per_tok": 2,
16
+ "num_hidden_layers": 16,
17
+ "num_key_value_heads": 4,
18
+ "num_local_experts": 4,
19
+ "pad_token_id": 2,
20
+ "rms_norm_eps": 1e-06,
21
+ "rope_theta": 1000000.0,
22
+ "router_aux_loss_coef": 0.001,
23
+ "router_jitter_noise": 0.0,
24
+ "tie_word_embeddings": true,
25
+ "transformers_version": "5.17.0",
26
+ "use_qk_norm": true,
27
+ "vocab_size": 32000,
28
+ "auto_map": {
29
+ "AutoConfig": "configuration_tinymixtral.TinyMixtralConfig",
30
+ "AutoModelForCausalLM": "modeling_tinymixtral.TinyMixtralForCausalLM"
31
+ }
32
+ }
configuration_tinymixtral.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) Michael Lee (李登淳) 2026. All rights reserved.
2
+ # Open-source under the MIT License. See LICENSE for details.
3
+
4
+ from transformers import PretrainedConfig
5
+
6
+
7
+ class TinyMixtralConfig(PretrainedConfig):
8
+ model_type = "tinymixtral"
9
+
10
+ def __init__(
11
+ self,
12
+ vocab_size: int = 32000,
13
+ hidden_size: int = 896,
14
+ num_hidden_layers: int = 10,
15
+ num_attention_heads: int = 14,
16
+ num_key_value_heads: int = 2,
17
+ head_dim: int = 64,
18
+ max_position_embeddings: int = 2048,
19
+ num_local_experts: int = 6,
20
+ num_experts_per_tok: int = 2,
21
+ expert_intermediate_size: int = 2389,
22
+ router_aux_loss_coef: float = 0.01,
23
+ router_jitter_noise: float = 0.01,
24
+ use_qk_norm: bool = False,
25
+ rms_norm_eps: float = 1e-6,
26
+ rope_theta: float = 1_000_000.0,
27
+ attention_dropout: float = 0.0,
28
+ tie_word_embeddings: bool = True,
29
+ initializer_range: float = 0.02,
30
+ **kwargs,
31
+ ):
32
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
33
+ self.vocab_size = vocab_size
34
+ self.hidden_size = hidden_size
35
+ self.num_hidden_layers = num_hidden_layers
36
+ self.num_attention_heads = num_attention_heads
37
+ self.num_key_value_heads = num_key_value_heads
38
+ self.head_dim = head_dim
39
+ self.max_position_embeddings = max_position_embeddings
40
+ self.num_local_experts = num_local_experts
41
+ self.num_experts_per_tok = num_experts_per_tok
42
+ self.expert_intermediate_size = expert_intermediate_size
43
+ self.router_aux_loss_coef = router_aux_loss_coef
44
+ self.router_jitter_noise = router_jitter_noise
45
+ self.use_qk_norm = use_qk_norm
46
+ self.rms_norm_eps = rms_norm_eps
47
+ self.rope_theta = rope_theta
48
+ self.attention_dropout = attention_dropout
49
+ self.initializer_range = initializer_range
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "output_attentions": false,
4
+ "output_hidden_states": false,
5
+ "transformers_version": "5.17.0"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45441ed3826f64840be1956320f74de6023d5c2f40148d58e60ff572054aca50
3
+ size 1909882768
modeling_tinymixtral.py ADDED
@@ -0,0 +1,321 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) Michael Lee (李登淳) 2026. All rights reserved.
2
+ # Open-source under the MIT License. See LICENSE for details.
3
+
4
+ from dataclasses import dataclass
5
+ from typing import Optional
6
+
7
+ import torch
8
+ import torch.nn as nn
9
+ import torch.nn.functional as F
10
+ from torch.utils.checkpoint import checkpoint
11
+ from transformers import PreTrainedModel, GenerationMixin
12
+ from transformers.modeling_outputs import ModelOutput
13
+
14
+ from .configuration_tinymixtral import TinyMixtralConfig
15
+
16
+
17
+ # ============================================================
18
+ # Layers
19
+ # ============================================================
20
+
21
+ class RMSNorm(nn.Module):
22
+ def __init__(self, dim: int, eps: float = 1e-6):
23
+ super().__init__()
24
+ self.weight = nn.Parameter(torch.ones(dim))
25
+ self.eps = eps
26
+
27
+ def forward(self, x):
28
+ dtype = x.dtype
29
+ x = x.float()
30
+ norm = x.pow(2).mean(-1, keepdim=True)
31
+ x = x * torch.rsqrt(norm + self.eps)
32
+ return (x * self.weight).to(dtype)
33
+
34
+
35
+ class RotaryEmbedding(nn.Module):
36
+ def __init__(self, dim, max_position_embeddings=2048, theta=10000.0):
37
+ super().__init__()
38
+ self.dim = dim
39
+ self.max_position_embeddings = max_position_embeddings
40
+ self.theta = theta
41
+ self._build_cache()
42
+
43
+ def _build_cache(self):
44
+ inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.dim, 2).float() / self.dim))
45
+ t = torch.arange(self.max_position_embeddings).float()
46
+ freqs = torch.outer(t, inv_freq)
47
+ emb = torch.cat((freqs, freqs), dim=-1)
48
+ self.register_buffer("cos_cached", emb.cos(), persistent=False)
49
+ self.register_buffer("sin_cached", emb.sin(), persistent=False)
50
+
51
+ def forward(self, x, position_ids):
52
+ cos = self.cos_cached[position_ids].unsqueeze(1)
53
+ sin = self.sin_cached[position_ids].unsqueeze(1)
54
+ x_rot = x.float()
55
+ x1, x2 = x_rot.chunk(2, dim=-1)
56
+ rotated = torch.cat((-x2, x1), dim=-1)
57
+ return (x_rot * cos + rotated * sin).to(x.dtype)
58
+
59
+
60
+ class GQAAttention(nn.Module):
61
+ def __init__(self, config):
62
+ super().__init__()
63
+ self.hidden_size = config.hidden_size
64
+ self.num_heads = config.num_attention_heads
65
+ self.num_kv_heads = config.num_key_value_heads
66
+ self.head_dim = config.head_dim
67
+ self.num_groups = self.num_heads // self.num_kv_heads
68
+ assert self.num_heads % self.num_kv_heads == 0
69
+
70
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
71
+ self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
72
+ self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
73
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
74
+ self.rotary_emb = RotaryEmbedding(self.head_dim, config.max_position_embeddings, config.rope_theta)
75
+ self.attention_dropout = config.attention_dropout
76
+ if getattr(config, "use_qk_norm", False):
77
+ self.q_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
78
+ self.k_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
79
+ else:
80
+ self.q_norm = None
81
+ self.k_norm = None
82
+
83
+ def forward(self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, use_cache=False):
84
+ B, S, _ = hidden_states.shape
85
+ q = self.q_proj(hidden_states).view(B, S, self.num_heads, self.head_dim).transpose(1, 2)
86
+ k = self.k_proj(hidden_states).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
87
+ v = self.v_proj(hidden_states).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
88
+
89
+ if self.q_norm is not None:
90
+ q = self.q_norm(q)
91
+ k = self.k_norm(k)
92
+
93
+ cache_len = past_key_value[0].shape[2] if past_key_value is not None else 0
94
+ if position_ids is None:
95
+ position_ids = torch.arange(cache_len, cache_len + S, device=hidden_states.device).unsqueeze(0).expand(B, -1)
96
+ q, k = self.rotary_emb(q, position_ids), self.rotary_emb(k, position_ids)
97
+
98
+ if past_key_value is not None:
99
+ k = torch.cat([past_key_value[0], k], dim=2)
100
+ v = torch.cat([past_key_value[1], v], dim=2)
101
+ cache = (k, v) if use_cache else None
102
+ total_len = cache_len + S
103
+
104
+ if attention_mask is not None or cache_len > 0:
105
+ k_exp = k.unsqueeze(2).expand(-1, -1, self.num_groups, -1, -1).reshape(B, self.num_heads, total_len, self.head_dim)
106
+ v_exp = v.unsqueeze(2).expand(-1, -1, self.num_groups, -1, -1).reshape(B, self.num_heads, total_len, self.head_dim)
107
+ causal = torch.tril(torch.ones(S, total_len, device=hidden_states.device, dtype=torch.bool), diagonal=cache_len)
108
+ if attention_mask is not None:
109
+ mask = causal[None, None, :, :] & attention_mask[:, None, None, :]
110
+ else:
111
+ mask = causal[None, None, :, :]
112
+ attn = F.scaled_dot_product_attention(
113
+ q, k_exp, v_exp, attn_mask=mask,
114
+ dropout_p=self.attention_dropout if self.training else 0.0,
115
+ is_causal=False,
116
+ )
117
+ else:
118
+ attn = F.scaled_dot_product_attention(
119
+ q, k, v, attn_mask=None,
120
+ dropout_p=self.attention_dropout if self.training else 0.0,
121
+ is_causal=True,
122
+ enable_gqa=True,
123
+ )
124
+ return self.o_proj(attn.transpose(1, 2).reshape(B, S, -1)), cache
125
+
126
+
127
+ class SparseMoE(nn.Module):
128
+ def __init__(self, config):
129
+ super().__init__()
130
+ self.hidden_size = config.hidden_size
131
+ self.num_experts = config.num_local_experts
132
+ self.top_k = config.num_experts_per_tok
133
+ self.expert_intermediate = config.expert_intermediate_size
134
+ self.jitter_noise = config.router_jitter_noise
135
+ self.aux_loss_coef = config.router_aux_loss_coef
136
+ self.router = nn.Linear(self.hidden_size, self.num_experts, bias=False)
137
+ self.gate_proj = nn.Parameter(torch.empty(self.num_experts, self.expert_intermediate, self.hidden_size))
138
+ self.up_proj = nn.Parameter(torch.empty(self.num_experts, self.expert_intermediate, self.hidden_size))
139
+ self.down_proj = nn.Parameter(torch.empty(self.num_experts, self.hidden_size, self.expert_intermediate))
140
+ self._init_weights()
141
+
142
+ def _init_weights(self, std=0.02):
143
+ nn.init.normal_(self.gate_proj, std=std)
144
+ nn.init.normal_(self.up_proj, std=std)
145
+ nn.init.normal_(self.down_proj, std=std)
146
+
147
+ def forward(self, x):
148
+ B, S, D = x.shape
149
+ x_flat = x.view(-1, D)
150
+ N = B * S
151
+ logits = self.router(x_flat)
152
+ if self.training and self.jitter_noise > 0:
153
+ logits = logits * (1 + torch.randn_like(logits) * self.jitter_noise)
154
+ weights = F.softmax(logits.float(), dim=-1).to(x.dtype)
155
+ w_topk, experts = torch.topk(weights, self.top_k, dim=-1)
156
+ w_topk = w_topk / w_topk.sum(dim=-1, keepdim=True)
157
+
158
+ aux = torch.tensor(0.0, device=x.device, dtype=x.dtype)
159
+ if self.training and self.aux_loss_coef > 0:
160
+ with torch.no_grad():
161
+ mask = F.one_hot(experts, num_classes=self.num_experts).float()
162
+ f_i = mask.mean(dim=(0, 1))
163
+ P_i = weights.mean(dim=0)
164
+ aux = (f_i.detach() * P_i).sum() * self.num_experts
165
+
166
+ flat_experts = experts.view(-1)
167
+ flat_weights = w_topk.view(-1)
168
+ flat_token_idx = torch.arange(N, device=x.device).unsqueeze(1).expand(-1, self.top_k).reshape(-1)
169
+
170
+ sorted_indices = flat_experts.argsort(stable=True)
171
+ sorted_token_idx = flat_token_idx[sorted_indices]
172
+ sorted_weights = flat_weights[sorted_indices]
173
+ sorted_experts = flat_experts[sorted_indices]
174
+
175
+ expert_counts = torch.bincount(sorted_experts, minlength=self.num_experts).tolist()
176
+
177
+ out = torch.zeros(N, D, device=x.device, dtype=x.dtype)
178
+ start = 0
179
+ for e in range(self.num_experts):
180
+ count = expert_counts[e]
181
+ if count == 0:
182
+ continue
183
+ end = start + count
184
+ idx = sorted_token_idx[start:end]
185
+ w = sorted_weights[start:end]
186
+ ts = x_flat[idx]
187
+ gate = F.silu(ts @ self.gate_proj[e].T)
188
+ up = ts @ self.up_proj[e].T
189
+ out.index_add_(0, idx, ((gate * up @ self.down_proj[e].T) * w.unsqueeze(-1)).to(x.dtype))
190
+ start = end
191
+ if self.training:
192
+ keepalive = (self.gate_proj.sum() + self.up_proj.sum() + self.down_proj.sum()) * 0.0
193
+ out = out + keepalive.to(out.dtype)
194
+ return out.view(B, S, D), aux
195
+
196
+
197
+ class MoETransformerBlock(nn.Module):
198
+ def __init__(self, config):
199
+ super().__init__()
200
+ self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
201
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
202
+ self.self_attn = GQAAttention(config)
203
+ self.moe = SparseMoE(config)
204
+
205
+ def forward(self, x, attention_mask=None, position_ids=None, past_key_value=None, use_cache=False):
206
+ attn_out, new_cache = self.self_attn(
207
+ self.input_layernorm(x), attention_mask, position_ids, past_key_value, use_cache
208
+ )
209
+ x = x + attn_out
210
+ h, aux = self.moe(self.post_attention_layernorm(x))
211
+ return x + h, aux, new_cache
212
+
213
+
214
+ # ============================================================
215
+ # Causal LM
216
+ # ============================================================
217
+
218
+ @dataclass
219
+ class CausalLMOutputWithPast(ModelOutput):
220
+ loss: Optional[torch.Tensor] = None
221
+ logits: torch.Tensor = None
222
+ past_key_values: Optional[tuple] = None
223
+
224
+
225
+ class TinyMixtralForCausalLM(PreTrainedModel, GenerationMixin):
226
+ config_class = TinyMixtralConfig
227
+ base_model_prefix = "tinymixtral"
228
+ supports_gradient_checkpointing = True
229
+ _no_split_modules = ["MoETransformerBlock"]
230
+ _supports_cache_class = False
231
+ _supports_static_cache = False
232
+
233
+ def _supports_default_dynamic_cache(self):
234
+ return False
235
+
236
+ def __init__(self, config):
237
+ super().__init__(config)
238
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
239
+ self.layers = nn.ModuleList([MoETransformerBlock(config) for _ in range(config.num_hidden_layers)])
240
+ self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
241
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
242
+ if config.tie_word_embeddings:
243
+ self.lm_head.weight = self.embed_tokens.weight
244
+ self._tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"}
245
+ self._use_activation_checkpointing = False
246
+ self.post_init()
247
+ if getattr(self.config, "eos_token_id", None) is None:
248
+ self.config.eos_token_id = 2
249
+ self.config.pad_token_id = 2
250
+
251
+ def prepare_inputs_for_generation(self, input_ids, attention_mask=None, past_key_values=None, **kwargs):
252
+ return {
253
+ "input_ids": input_ids,
254
+ "attention_mask": attention_mask,
255
+ "past_key_values": past_key_values,
256
+ "use_cache": kwargs.get("use_cache", True),
257
+ }
258
+
259
+ def _reorder_cache(self, past_key_values, beam_idx):
260
+ return tuple(
261
+ tuple(past.index_select(0, beam_idx) for past in layer_past)
262
+ for layer_past in past_key_values
263
+ )
264
+
265
+ def _init_weights(self, module):
266
+ std = self.config.initializer_range
267
+ if isinstance(module, nn.Linear):
268
+ module.weight.data.normal_(mean=0.0, std=std)
269
+ if module.bias is not None:
270
+ module.bias.data.zero_()
271
+ elif isinstance(module, nn.Embedding):
272
+ module.weight.data.normal_(mean=0.0, std=std)
273
+
274
+ def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
275
+ self._use_activation_checkpointing = True
276
+
277
+ def gradient_checkpointing_disable(self):
278
+ self._use_activation_checkpointing = False
279
+
280
+ def forward(self, input_ids, attention_mask=None, labels=None, return_dict=True, past_key_values=None, use_cache=False, **kwargs):
281
+ B, S = input_ids.shape
282
+ has_cache = past_key_values is not None and len(past_key_values) > 0
283
+ past_len = past_key_values[0][0].shape[2] if has_cache else 0
284
+ if past_len > 0 and S > past_len:
285
+ input_ids = input_ids[:, past_len:]
286
+ S = input_ids.shape[1]
287
+ pos = torch.arange(past_len, past_len + S, device=input_ids.device).unsqueeze(0).expand(B, -1)
288
+
289
+ total_len = past_len + S
290
+ if attention_mask is not None and attention_mask.shape[1] < total_len:
291
+ pad = torch.ones(B, total_len - attention_mask.shape[1],
292
+ dtype=attention_mask.dtype, device=attention_mask.device)
293
+ attention_mask = torch.cat([pad, attention_mask], dim=1)
294
+ cmask = attention_mask.bool() if attention_mask is not None else None
295
+
296
+ h = self.embed_tokens(input_ids)
297
+ total_aux = torch.tensor(0.0, device=input_ids.device, dtype=torch.float32)
298
+ new_caches = []
299
+ for i, layer in enumerate(self.layers):
300
+ layer_cache = past_key_values[i] if has_cache else None
301
+ if self._use_activation_checkpointing and self.training:
302
+ h, aux, _ = checkpoint(layer, h, cmask, pos, None, False, use_reentrant=False)
303
+ else:
304
+ h, aux, layer_new_cache = layer(h, cmask, pos, layer_cache, use_cache)
305
+ new_caches.append(layer_new_cache)
306
+ total_aux = total_aux + aux
307
+ logits = self.lm_head(self.norm(h)).float()
308
+
309
+ loss = None
310
+ if labels is not None:
311
+ loss = F.cross_entropy(
312
+ logits.reshape(-1, logits.size(-1)),
313
+ labels.reshape(-1),
314
+ ignore_index=-100,
315
+ )
316
+ loss = loss + self.config.router_aux_loss_coef * (total_aux / len(self.layers))
317
+
318
+ past_key_values_out = tuple(new_caches) if use_cache else None
319
+ if not return_dict:
320
+ return (loss, logits, past_key_values_out) if loss is not None else (logits, past_key_values_out)
321
+ return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=past_key_values_out)
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": null,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "extra_special_tokens": {},
35
+ "legacy": false,
36
+ "model_max_length": 2048,
37
+ "pad_token": "</s>",
38
+ "padding_side": "right",
39
+ "sp_model_kwargs": {},
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false
43
+ }