Jtapsa commited on
Commit
eb41dc4
·
verified ·
1 Parent(s): 5182004

Create moep_gemma.py

Browse files
Files changed (1) hide show
  1. moep_gemma.py +555 -0
moep_gemma.py ADDED
@@ -0,0 +1,555 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from dataclasses import dataclass
3
+ from typing import Optional, Tuple, List, Union
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+
9
+ from transformers import PreTrainedModel, PretrainedConfig
10
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoModel
11
+
12
+
13
+ from transformers.modeling_outputs import (
14
+ BaseModelOutput,
15
+ CausalLMOutput,
16
+ )
17
+
18
+
19
+ @dataclass
20
+ class BaseModelOutputWithAux(BaseModelOutput):
21
+ aux_loss: Optional[torch.Tensor] = None
22
+
23
+
24
+ def create_causal_mask(seq_len: int, device=None, dtype=torch.float32):
25
+ mask = torch.full((seq_len, seq_len), torch.finfo(dtype).min, device=device, dtype=dtype)
26
+ mask = torch.triu(mask, diagonal=1)
27
+ return mask
28
+
29
+ def create_sw_mask(seq_len: int, window_size: Optional[int], device=None, dtype=torch.float32):
30
+ if not window_size:
31
+ return None
32
+ i = torch.arange(seq_len, device=device)
33
+ j = torch.arange(seq_len, device=device)
34
+ allowed = (j[None, :] <= i[:, None]) & (j[None, :] >= i[:, None] - (window_size - 1))
35
+ mask = torch.zeros((seq_len, seq_len), device=device, dtype=dtype)
36
+ mask = mask.masked_fill(~allowed, torch.finfo(dtype).min)
37
+ return mask
38
+
39
+
40
+ class Linear(nn.Module):
41
+ def __init__(self, in_dim: int, out_dim: int, b: bool = False):
42
+ super().__init__()
43
+ self.linear = nn.Linear(in_dim, out_dim, bias=b)
44
+
45
+ def forward(self, x: torch.Tensor):
46
+ return self.linear(x)
47
+
48
+
49
+ class RMS_norm(nn.Module):
50
+ def __init__(self, dim: int, eps: float = 1e-5, b=True):
51
+ super().__init__()
52
+ self.rms_norm = nn.RMSNorm(dim, eps=eps, elementwise_affine=b)
53
+
54
+ def forward(self, x: torch.Tensor):
55
+ return self.rms_norm(x)
56
+
57
+
58
+ class GeGLU(nn.Module):
59
+ def __init__(self, in_dim, hidden_dim, b=False):
60
+ super().__init__()
61
+ self.linear1 = nn.Linear(in_dim, hidden_dim, bias=b)
62
+ self.linear2 = nn.Linear(in_dim, hidden_dim, bias=b)
63
+ self.linear3 = nn.Linear(hidden_dim, in_dim, bias=b)
64
+
65
+ def forward(self, x: torch.Tensor):
66
+ gate = F.gelu(self.linear1(x), approximate='tanh')
67
+ return self.linear3(gate * self.linear2(x))
68
+
69
+
70
+ # --- RoPE --- #
71
+
72
+ def precompute_theta_pos_frequencies(emb_dim: int, seq_len: int, device: str, theta: float = 10000.0):
73
+ assert emb_dim % 2 == 0, "Dimension must be divisible by 2"
74
+ theta_numerator = torch.arange(0, emb_dim, 2, device=device).float()
75
+ theta_values = 1.0 / (theta ** (theta_numerator / emb_dim))
76
+ m = torch.arange(seq_len, device=device)
77
+ freqs_matrix = torch.outer(m, theta_values).float()
78
+ freqs_complex = torch.polar(torch.ones_like(freqs_matrix, device=device), freqs_matrix)
79
+ return freqs_complex
80
+
81
+ def RoPE(x: torch.Tensor, freqs: torch.Tensor, index: int = -1):
82
+ ndim = x.ndim
83
+ assert 1 < ndim
84
+ assert freqs.shape[-1] == x.shape[-1]
85
+
86
+ if index != -1:
87
+ # Ensure that index cannot be less than 1 --> (1 == first token position)
88
+ # and more than freqs.shape[-2] --> (index > max sequence length)
89
+ index = max(1, min(index, freqs.shape[-2]))
90
+ freqs = freqs[(index -1), :].unsqueeze(0)
91
+
92
+ if freqs.shape[-2] != x.shape[-2] and index == -1:
93
+ # if in x seq_len != max lenght, drop unneeded m frequence positions
94
+ freqs = freqs[:x.shape[-2], :]
95
+
96
+ # from (seq_len, head_dim) to (1, 1, seq_len, head_dim)
97
+ freqs = freqs.unsqueeze(0).unsqueeze(0)
98
+ rotated_x = x * freqs
99
+
100
+ return rotated_x
101
+
102
+ def apply_rotary_embeddings(x: torch.Tensor, freqs_complex: torch.Tensor):
103
+ x_reshaped = x.reshape(*x.shape[:-1], -1, 2)
104
+ x_complex = torch.view_as_complex(x_reshaped.float())
105
+ x_rotated = RoPE(x_complex, freqs_complex)
106
+ x_out = torch.view_as_real(x_rotated)
107
+ x_out = x_out.reshape(*x.shape).type_as(x)
108
+ return x_out
109
+
110
+
111
+ class SelfAttention(nn.Module):
112
+ def __init__(self, dim: int, n_heads: int):
113
+ super().__init__()
114
+ assert dim % n_heads == 0, "dim must be divisible by n_heads"
115
+ self.dim = dim
116
+ self.n_heads = n_heads
117
+ self.head_dim = dim // n_heads
118
+ self.Q = nn.Linear(dim, dim)
119
+ self.K = nn.Linear(dim, dim)
120
+ self.V = nn.Linear(dim, dim)
121
+ self.O = nn.Linear(dim, dim)
122
+
123
+ def forward(self, x: torch.Tensor, *, causal_mask: Optional[torch.Tensor] = None, key_padding_mask: Optional[torch.Tensor] = None, freqs: Optional[torch.Tensor] = None):
124
+ # x: [B,S,D]
125
+ B, S, D = x.shape
126
+ H = self.n_heads
127
+ Dh = self.head_dim
128
+
129
+ q = self.Q(x).view(B, S, H, Dh).transpose(1, 2) # [B,H,S,Dh]
130
+ k = self.K(x).view(B, S, H, Dh).transpose(1, 2)
131
+ v = self.V(x).view(B, S, H, Dh).transpose(1, 2)
132
+
133
+ q = apply_rotary_embeddings(q, freqs)
134
+ k = apply_rotary_embeddings(k, freqs)
135
+
136
+ attn = (q @ k.transpose(-2, -1)) / math.sqrt(Dh) # [B,H,S,S]
137
+ neg_inf = torch.finfo(attn.dtype).min
138
+
139
+ if causal_mask is not None:
140
+ attn = attn + causal_mask[None, None, :, :]
141
+
142
+ if key_padding_mask is not None: # True at PAD keys
143
+ attn = attn.masked_fill(key_padding_mask[:, None, None, :], neg_inf)
144
+
145
+ attn = F.softmax(attn, dim=-1)
146
+ out = (attn @ v).transpose(1, 2).contiguous().view(B, S, D)
147
+
148
+ if key_padding_mask is not None:
149
+ query_mask = (~key_padding_mask).to(out.dtype).unsqueeze(-1) # [B,S,1]
150
+ out = out * query_mask
151
+
152
+ return self.O(out)
153
+
154
+
155
+ class Layer_Block(nn.Module):
156
+ def __init__(self, dim: int, n_heads: int):
157
+ super().__init__()
158
+ self.norm_1 = RMS_norm(dim)
159
+ self.norm_2 = RMS_norm(dim)
160
+ self.attention = SelfAttention(dim, n_heads)
161
+ self.hidden_dim = int(dim * 4)
162
+ self.feedforward = GeGLU(dim, self.hidden_dim)
163
+
164
+ def forward(self, x: torch.Tensor, *, causal_mask=None, key_padding_mask=None, freqs=None):
165
+ residual = x
166
+ x = self.norm_1(x)
167
+ x = self.attention(x, causal_mask=causal_mask, key_padding_mask=key_padding_mask, freqs=freqs)
168
+ x = residual + x
169
+
170
+ residual = x
171
+ x = self.norm_2(x)
172
+ x = self.feedforward(x)
173
+ x = residual + x
174
+ return x
175
+
176
+
177
+ class Top_k_Router(nn.Module):
178
+ def __init__(self, dim: int, n_experts: int, k: int):
179
+ super().__init__()
180
+ self.n_experts = n_experts
181
+ self.k = k
182
+ self.router = Linear(dim, n_experts)
183
+
184
+ def forward(self, x: torch.Tensor):
185
+ # x: [T, D]
186
+ logits = self.router(x)
187
+ probs = F.softmax(logits, dim=-1)
188
+ topk_prob, topk_idx = torch.topk(probs, k=self.k, dim=-1)
189
+ topk_prob = topk_prob / (topk_prob.sum(dim=-1, keepdim=True) + 1e-9)
190
+
191
+ p_mean = probs.mean(dim=0)
192
+ T, E = probs.size()
193
+ one_hot = F.one_hot(topk_idx, num_classes=E).sum(dim=1)
194
+ f_mean = one_hot.float().mean(dim=0) / float(self.k)
195
+ aux_loss = (E * (p_mean * f_mean).sum())
196
+
197
+ return topk_idx, topk_prob, aux_loss
198
+
199
+
200
+ class MoE_Block(nn.Module):
201
+ def __init__(self, dim_in: int, n_experts: int, k: int, dim_hidden: int, dim_out: Optional[int] = None):
202
+ super().__init__()
203
+ self.n_experts = n_experts
204
+ self.k = k
205
+ self.dim_out = dim_in if dim_out is None else dim_out
206
+ self.router = Top_k_Router(dim_in, n_experts, k)
207
+ self.experts = nn.ModuleList([nn.Linear(dim_in, self.dim_out, bias=True) for _ in range(n_experts)])
208
+
209
+ def forward(self, x: torch.Tensor):
210
+ # x: [B,S,D]
211
+ B, S, D = x.shape
212
+ T = B * S
213
+ x_flat = x.view(T, D)
214
+
215
+ topk_idx, topk_prob, aux_loss = self.router(x_flat) # [T,k], [T,k]
216
+ y_flat = x_flat.new_zeros((T, self.dim_out))
217
+
218
+ for slot in range(self.k):
219
+ e_ids = topk_idx[:, slot]
220
+ e_prob = topk_prob[:, slot].unsqueeze(-1) # [T,1]
221
+ for e in range(self.n_experts):
222
+ idx = (e_ids == e).nonzero(as_tuple=False).squeeze(1)
223
+ if idx.numel() == 0:
224
+ continue
225
+ y_e = self.experts[e](x_flat.index_select(0, idx)) # [Ne, dim_out]
226
+ y_flat.index_add_(0, idx, y_e * e_prob.index_select(0, idx))
227
+
228
+ return y_flat.view(B, S, self.dim_out), aux_loss
229
+
230
+
231
+ class ParallelLayer(nn.Module):
232
+ def __init__(self, dim: int, n_heads: int, n_blocks: int, k: int):
233
+ super().__init__()
234
+ self.dim = dim
235
+ self.n_blocks = n_blocks
236
+ self.blocks = nn.ModuleList([Layer_Block(dim, n_heads) for _ in range(n_blocks)])
237
+ self.router = Top_k_Router(dim, n_blocks, k)
238
+
239
+ def forward(self, x: torch.Tensor, *, causal_mask=None, key_padding_mask=None, freqs=None):
240
+ B, S, D = x.shape
241
+ T = B * S
242
+ outs = [blk(x, causal_mask=causal_mask, key_padding_mask=key_padding_mask, freqs=freqs) for blk in self.blocks]
243
+
244
+ Y = torch.stack([y.view(T, D) for y in outs], dim=1) # [T, n_blocks, D]
245
+
246
+ x_flat = x.view(T, D)
247
+ topk_idx, topk_prob, aux_loss = self.router(x_flat) # [T,k], [T,k]
248
+
249
+ gather_idx = topk_idx.unsqueeze(-1).expand(-1, -1, D) # [T,k,D]
250
+ Y_topk = torch.gather(Y, dim=1, index=gather_idx) # [T,k,D]
251
+ y_flat = (Y_topk * topk_prob.unsqueeze(-1)).sum(dim=1) # [T,D]
252
+
253
+ return y_flat.view(B, S, D), aux_loss
254
+
255
+
256
+ class MoEPgemmaLMConfig(PretrainedConfig):
257
+ model_type = "moep_gemma_lm"
258
+
259
+ def __init__(
260
+ self,
261
+ vocab_size: int = 16384,
262
+ bos_token_id = 1,
263
+ eos_token_id = 2,
264
+ pad_token_id = 3,
265
+ mask_token_id = 4,
266
+ max_context_length: int = 1024,
267
+ n_layers: int = 12,
268
+ ffn_multp: int = 4,
269
+ dim: int = 348,
270
+ n_heads: int = 6,
271
+ # Parallel specifics
272
+ n_parallel_blocks: int = 4,
273
+ parallel_dim: int = 192,
274
+ parallel_n_heads: int = 3,
275
+ parallel_k: int = 2,
276
+ sliding_window: Optional[int] = None,
277
+ # MoE specifics
278
+ n_experts: int = 4,
279
+ moe_k: int = 2,
280
+ # Norms
281
+ layer_norm_epsilon: float = 1e-5,
282
+ tie_word_embeddings: bool = True,
283
+ **kwargs,
284
+ ):
285
+ super().__init__(
286
+ vocab_size=vocab_size,
287
+ pad_token_id=pad_token_id,
288
+ bos_token_id=bos_token_id,
289
+ eos_token_id=eos_token_id,
290
+ **kwargs,
291
+ )
292
+
293
+ # Special tokens
294
+ self.mask_token_id = mask_token_id
295
+
296
+ # Core dims
297
+ self.vocab_size = vocab_size
298
+ self.max_context_length = max_context_length
299
+ self.n_layers = n_layers
300
+ self.ffn_multp = ffn_multp
301
+ self.dim = dim
302
+ self.n_heads = n_heads
303
+
304
+ # Parallel specifics
305
+ self.n_parallel_blocks = n_parallel_blocks
306
+ self.parallel_dim = parallel_dim
307
+ self.parallel_n_heads = parallel_n_heads
308
+ self.parallel_k = parallel_k
309
+ self.sliding_window = sliding_window
310
+
311
+ # MoE specifics
312
+ self.n_experts = n_experts
313
+ self.moe_k = moe_k
314
+
315
+ # norms
316
+ self.layer_norm_epsilon = layer_norm_epsilon
317
+
318
+ # Tying
319
+ self.tie_word_embeddings = tie_word_embeddings
320
+
321
+ class MoEPgemmaModel(PreTrainedModel):
322
+ config_class = MoEPgemmaLMConfig
323
+ base_model_prefix = "model"
324
+
325
+ def __init__(self, config: MoEPgemmaLMConfig):
326
+ super().__init__(config)
327
+ self.config = config
328
+
329
+ # --- embeddings ---
330
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.dim, padding_idx=config.pad_token_id)
331
+
332
+ self.layers = nn.ModuleList()
333
+
334
+ # Large Layer
335
+ self.layers.append(Layer_Block(config.dim,
336
+ config.n_heads))
337
+
338
+ # MoE Linear
339
+ self.layers.append(MoE_Block(config.dim,
340
+ config.n_experts,
341
+ config.moe_k,
342
+ dim_hidden=config.dim * 2,
343
+ dim_out=config.parallel_dim))
344
+
345
+ # Parallel Layers
346
+ for _ in range(config.n_layers - 2):
347
+ self.layers.append(ParallelLayer(config.parallel_dim,
348
+ config.parallel_n_heads,
349
+ config.n_parallel_blocks,
350
+ config.parallel_k))
351
+
352
+ # MoE Linear
353
+ self.layers.append(MoE_Block(config.parallel_dim,
354
+ config.n_experts,
355
+ config.moe_k,
356
+ dim_hidden=config.parallel_dim * config.ffn_multp,
357
+ dim_out=config.dim))
358
+
359
+ # Large Layer
360
+ self.layers.append(Layer_Block(config.dim,
361
+ config.n_heads))
362
+
363
+ self.norm = RMS_norm(config.dim)
364
+ self.post_init()
365
+
366
+ self.sliding_window = config.sliding_window
367
+
368
+ self.freqs_p = precompute_theta_pos_frequencies(int(config.parallel_dim/config.parallel_n_heads),
369
+ config.max_context_length,
370
+ device=self.device)
371
+
372
+ self.freqs = precompute_theta_pos_frequencies(int(config.dim/config.n_heads),
373
+ config.max_context_length,
374
+ device=self.device)
375
+
376
+
377
+ def get_input_embeddings(self):
378
+ return self.embed_tokens
379
+
380
+ def set_input_embeddings(self, new_embeddings):
381
+ self.embed_tokens = new_embeddings
382
+
383
+ def _init_weights(self, module):
384
+ if isinstance(module, (nn.Linear, nn.Embedding)):
385
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
386
+ if isinstance(module, nn.Linear) and module.bias is not None:
387
+ nn.init.zeros_(module.bias)
388
+
389
+ def forward(
390
+ self,
391
+ input_ids: torch.LongTensor,
392
+ attention_mask: Optional[torch.Tensor] = None,
393
+ return_dict: bool = True,
394
+ return_aux: Optional[bool] = False,
395
+ return_aux_dict: Optional[bool] = False,
396
+ **unused,
397
+ ) -> Union[BaseModelOutputWithAux, BaseModelOutput, Tuple]:
398
+
399
+ device = input_ids.device
400
+ B, S = input_ids.shape
401
+
402
+ if self.freqs.device != device:
403
+ self.freqs = self.freqs.to(device)
404
+ self.freqs_p = self.freqs_p.to(device)
405
+
406
+ key_padding = input_ids.eq(self.config.pad_token_id) if attention_mask is None else (attention_mask == 0)
407
+
408
+ # embeddings
409
+ x = self.embed_tokens(input_ids)
410
+
411
+ mask_dtype = x.dtype
412
+ causal = create_causal_mask(S, device=device, dtype=mask_dtype)
413
+ sw = create_sw_mask(S, self.sliding_window, device=self.device, dtype=mask_dtype)
414
+ sw_mask = None if sw is None else torch.minimum(causal, sw)
415
+
416
+
417
+ aux_total = None
418
+
419
+ freqs = self.freqs
420
+
421
+ # First large
422
+ x = self.layers[0](x, causal_mask=causal, key_padding_mask=key_padding, freqs=freqs)
423
+
424
+ # MoE shrink
425
+ x, aux = self.layers[1](x)
426
+
427
+ if aux is not None:
428
+ aux_total = aux
429
+
430
+ freqs_p = self.freqs_p
431
+
432
+ idx = 2
433
+ for i in range(self.config.n_layers - 2):
434
+
435
+ use_sw = (self.sliding_window is not None) and (i % 2 == 1)
436
+ mask = sw_mask if (use_sw and sw_mask is not None) else causal
437
+
438
+ x, aux = self.layers[idx](x, causal_mask=mask, key_padding_mask=key_padding, freqs=freqs_p)
439
+ if aux is not None:
440
+ aux_total += aux
441
+ idx += 1
442
+
443
+ # MoE grow
444
+ x, aux = self.layers[idx](x)
445
+
446
+ if aux is not None:
447
+ aux_total += aux
448
+
449
+ idx += 1
450
+
451
+ # Second large
452
+ x = self.layers[idx](x, causal_mask=causal, key_padding_mask=key_padding, freqs=freqs)
453
+
454
+ last_hidden = self.norm(x)
455
+
456
+ if not return_dict:
457
+ return (last_hidden, aux_total) if return_aux else (last_hidden,)
458
+
459
+ if return_aux_dict:
460
+ return BaseModelOutputWithAux(
461
+ last_hidden_state=last_hidden,
462
+ hidden_states=None,
463
+ attentions=None,
464
+ aux_loss=aux_total,
465
+ )
466
+ else:
467
+ return BaseModelOutput(
468
+ last_hidden_state=last_hidden,
469
+ hidden_states=None,
470
+ attentions=None,
471
+ )
472
+
473
+
474
+ class MoEPgemmaForCausalLM(PreTrainedModel):
475
+ config_class = MoEPgemmaLMConfig
476
+ base_model_prefix = "model"
477
+
478
+ def __init__(self, config: MoEPgemmaLMConfig):
479
+ super().__init__(config)
480
+ self.config = config
481
+ self.model = MoEPgemmaModel(config)
482
+ self.lm_head = nn.Linear(config.dim, config.vocab_size, bias=False)
483
+ self.post_init()
484
+ if config.tie_word_embeddings:
485
+ self.lm_head.weight = self.model.embed_tokens.weight
486
+
487
+ def get_input_embeddings(self):
488
+ return self.model.get_input_embeddings()
489
+
490
+ def set_input_embeddings(self, new_embeddings):
491
+ self.model.set_input_embeddings(new_embeddings)
492
+
493
+ def _init_weights(self, module):
494
+ if isinstance(module, (nn.Linear, nn.Embedding)):
495
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
496
+ if isinstance(module, nn.Linear) and module.bias is not None:
497
+ nn.init.zeros_(module.bias)
498
+
499
+ def forward(
500
+ self,
501
+ input_ids: torch.LongTensor,
502
+ attention_mask: Optional[torch.Tensor] = None,
503
+ labels: Optional[torch.LongTensor] = None,
504
+ return_dict: bool = True,
505
+ return_aux: Optional[bool] = False,
506
+ aux_loss_weight: Optional[float] = 0.2,
507
+ **unused,
508
+ ) -> Union[CausalLMOutput, Tuple]:
509
+
510
+
511
+ if return_aux:
512
+ base_out = self.model(input_ids=input_ids,
513
+ attention_mask=attention_mask,
514
+ return_aux_dict=True,
515
+ return_aux=True)
516
+ else:
517
+ base_out = self.model(input_ids=input_ids,
518
+ attention_mask=attention_mask)
519
+
520
+ logits = self.lm_head(base_out.last_hidden_state).float()
521
+
522
+ loss = None
523
+
524
+ if labels is not None:
525
+
526
+ shift_logits = logits[:, :-1, :].contiguous()
527
+ shift_labels = labels[:, 1:].contiguous()
528
+
529
+ pad_id = self.config.pad_token_id
530
+ if pad_id is not None:
531
+ shift_labels = shift_labels.masked_fill(shift_labels == pad_id, -100)
532
+
533
+ loss = F.cross_entropy(
534
+ shift_logits.view(-1, shift_logits.size(-1)),
535
+ shift_labels.view(-1),
536
+ ignore_index=-100,
537
+ )
538
+
539
+ aux = getattr(base_out, "aux_loss", None)
540
+ if aux is not None:
541
+ loss = loss + aux_loss_weight * aux
542
+
543
+ if not return_dict:
544
+ return (loss, logits) if loss is not None else (logits,)
545
+
546
+ return CausalLMOutput(
547
+ loss=loss,
548
+ logits=logits,
549
+ hidden_states=base_out.hidden_states,
550
+ attentions=base_out.attentions,
551
+ )
552
+
553
+ AutoConfig.register("moep_gemma_lm", MoEPgemmaLMConfig)
554
+ AutoModelForCausalLM.register(MoEPgemmaLMConfig, MoEPgemmaForCausalLM)
555
+ AutoModel.register(MoEPgemmaLMConfig, MoEPgemmaModel)