freococo commited on
Commit
a438bb9
·
verified ·
1 Parent(s): 3bcdbab

Fix RoPE meta-buffer init & tie_weights for Transformers 5.x

Browse files
Files changed (1) hide show
  1. modeling_mytranslation.py +26 -17
modeling_mytranslation.py CHANGED
@@ -2,7 +2,6 @@ import math
2
  import torch
3
  import torch.nn as nn
4
  import torch.nn.functional as F
5
- import transformers
6
  from transformers import PreTrainedModel
7
  from transformers.modeling_outputs import Seq2SeqLMOutput
8
  from .configuration_mytranslation import MyTranslationConfig
@@ -11,7 +10,6 @@ PAD_ID = 3
11
  BOS_ID = 1
12
  EOS_ID = 2
13
  TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN = 4, 5, 6, 7
14
- _IS_TF5 = int(transformers.__version__.split(".")[0]) >= 5
15
 
16
  def _sdpa(q, k, v, attn_mask=None, is_causal=False):
17
  if q.size(1) != k.size(1):
@@ -40,16 +38,28 @@ class RMSNorm(nn.Module):
40
  class RotaryEmbedding(nn.Module):
41
  def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
42
  super().__init__()
43
- inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
44
- t = torch.arange(max_seq_len, dtype=torch.float32)
45
- freqs = torch.outer(t, inv_freq)
46
- self.register_buffer("cos_cached", freqs.cos()[None, None, :, :], persistent=False)
47
- self.register_buffer("sin_cached", freqs.sin()[None, None, :, :], persistent=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
  def forward(self, x: torch.Tensor) -> torch.Tensor:
50
- L = x.size(2)
51
- cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
52
- sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
53
  x1, x2 = x[..., ::2], x[..., 1::2]
54
  return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
55
 
@@ -190,7 +200,11 @@ class DecoderLayer(nn.Module):
190
  class MyTranslationForConditionalGeneration(PreTrainedModel):
191
  config_class = MyTranslationConfig
192
  base_model_prefix = ""
193
- _tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"} if hasattr(PreTrainedModel, "get_expanded_tied_weights_keys") else ["lm_head.weight"]
 
 
 
 
194
 
195
  def __init__(self, config: MyTranslationConfig):
196
  super().__init__(config)
@@ -225,7 +239,6 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
225
  self.dec_norm = RMSNorm(config.d_model)
226
  self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
227
  self.post_init()
228
- self.tie_weights()
229
 
230
  def get_input_embeddings(self):
231
  return self.embed_tokens
@@ -239,9 +252,6 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
239
  def set_output_embeddings(self, new_embeddings):
240
  self.lm_head = new_embeddings
241
 
242
- def tie_weights(self, **kwargs):
243
- self.lm_head.weight = self.embed_tokens.weight
244
-
245
  def encode_and_route(self, src_ids: torch.Tensor):
246
  enc_mask = (src_ids != self.config.pad_token_id).unsqueeze(1).unsqueeze(2)
247
  x = self.embed_tokens(src_ids) * self.embed_scale
@@ -316,8 +326,7 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
316
  k_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
317
  v_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
318
  cur_tok = torch.tensor([[BOS_ID]], dtype=torch.long, device=dev)
319
- cos_all = self.rope.cos_cached.to(dtype=dt, device=dev)
320
- sin_all = self.rope.sin_cached.to(dtype=dt, device=dev)
321
  ys = []
322
  for pos in range(max_new_tokens):
323
  x_dec = self.embed_tokens(cur_tok) * self.embed_scale
 
2
  import torch
3
  import torch.nn as nn
4
  import torch.nn.functional as F
 
5
  from transformers import PreTrainedModel
6
  from transformers.modeling_outputs import Seq2SeqLMOutput
7
  from .configuration_mytranslation import MyTranslationConfig
 
10
  BOS_ID = 1
11
  EOS_ID = 2
12
  TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN = 4, 5, 6, 7
 
13
 
14
  def _sdpa(q, k, v, attn_mask=None, is_causal=False):
15
  if q.size(1) != k.size(1):
 
38
  class RotaryEmbedding(nn.Module):
39
  def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
40
  super().__init__()
41
+ self.head_dim = dim
42
+ self.max_seq_len = max_seq_len
43
+ self.theta = theta
44
+ self._cos_live = None
45
+ self._sin_live = None
46
+
47
+ def get_cos_sin(self, seq_len: int, device: torch.device, dtype: torch.dtype):
48
+ if (
49
+ self._cos_live is None
50
+ or self._cos_live.device != device
51
+ or self._cos_live.size(2) < seq_len
52
+ ):
53
+ L = max(self.max_seq_len, seq_len)
54
+ inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.head_dim, 2, device=device, dtype=torch.float32) / self.head_dim))
55
+ t = torch.arange(L, device=device, dtype=torch.float32)
56
+ freqs = torch.outer(t, inv_freq)
57
+ self._cos_live = freqs.cos()[None, None, :, :]
58
+ self._sin_live = freqs.sin()[None, None, :, :]
59
+ return self._cos_live[:, :, :seq_len, :].to(dtype=dtype), self._sin_live[:, :, :seq_len, :].to(dtype=dtype)
60
 
61
  def forward(self, x: torch.Tensor) -> torch.Tensor:
62
+ cos, sin = self.get_cos_sin(x.size(2), x.device, x.dtype)
 
 
63
  x1, x2 = x[..., ::2], x[..., 1::2]
64
  return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
65
 
 
200
  class MyTranslationForConditionalGeneration(PreTrainedModel):
201
  config_class = MyTranslationConfig
202
  base_model_prefix = ""
203
+ _tied_weights_keys = (
204
+ {"lm_head.weight": "embed_tokens.weight"}
205
+ if hasattr(PreTrainedModel, "get_expanded_tied_weights_keys")
206
+ else ["lm_head.weight"]
207
+ )
208
 
209
  def __init__(self, config: MyTranslationConfig):
210
  super().__init__(config)
 
239
  self.dec_norm = RMSNorm(config.d_model)
240
  self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
241
  self.post_init()
 
242
 
243
  def get_input_embeddings(self):
244
  return self.embed_tokens
 
252
  def set_output_embeddings(self, new_embeddings):
253
  self.lm_head = new_embeddings
254
 
 
 
 
255
  def encode_and_route(self, src_ids: torch.Tensor):
256
  enc_mask = (src_ids != self.config.pad_token_id).unsqueeze(1).unsqueeze(2)
257
  x = self.embed_tokens(src_ids) * self.embed_scale
 
326
  k_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
327
  v_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
328
  cur_tok = torch.tensor([[BOS_ID]], dtype=torch.long, device=dev)
329
+ cos_all, sin_all = self.rope.get_cos_sin(max_new_tokens, dev, dt)
 
330
  ys = []
331
  for pos in range(max_new_tokens):
332
  x_dec = self.embed_tokens(cur_tok) * self.embed_scale