Translation
Transformers
Safetensors
Burmese
English
mytranslation
text2text-generation
Translation
Safetensors
Burmese
English
myanmartranslator
MyanmarLLM
custom_code
myanmar
Instructions to use freococo/MyTranslation_300M_Base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use freococo/MyTranslation_300M_Base with Transformers:
# Use a pipeline as a high-level helper # Warning: Pipeline type "translation" is no longer supported in transformers v5. # You must load the model directly (see below) or downgrade to v4.x with: # pip install "transformers<5.0.0" from transformers import pipeline pipe = pipeline("translation", model="freococo/MyTranslation_300M_Base", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForSeq2SeqLM model = AutoModelForSeq2SeqLM.from_pretrained("freococo/MyTranslation_300M_Base", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Fix RoPE meta-buffer init & tie_weights for Transformers 5.x
Browse files- modeling_mytranslation.py +26 -17
modeling_mytranslation.py
CHANGED
|
@@ -2,7 +2,6 @@ import math
|
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
| 4 |
import torch.nn.functional as F
|
| 5 |
-
import transformers
|
| 6 |
from transformers import PreTrainedModel
|
| 7 |
from transformers.modeling_outputs import Seq2SeqLMOutput
|
| 8 |
from .configuration_mytranslation import MyTranslationConfig
|
|
@@ -11,7 +10,6 @@ PAD_ID = 3
|
|
| 11 |
BOS_ID = 1
|
| 12 |
EOS_ID = 2
|
| 13 |
TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN = 4, 5, 6, 7
|
| 14 |
-
_IS_TF5 = int(transformers.__version__.split(".")[0]) >= 5
|
| 15 |
|
| 16 |
def _sdpa(q, k, v, attn_mask=None, is_causal=False):
|
| 17 |
if q.size(1) != k.size(1):
|
|
@@ -40,16 +38,28 @@ class RMSNorm(nn.Module):
|
|
| 40 |
class RotaryEmbedding(nn.Module):
|
| 41 |
def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
|
| 42 |
super().__init__()
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
self.
|
| 47 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 50 |
-
|
| 51 |
-
cos = self.cos_cached[:, :, :L, :].to(dtype=x.dtype)
|
| 52 |
-
sin = self.sin_cached[:, :, :L, :].to(dtype=x.dtype)
|
| 53 |
x1, x2 = x[..., ::2], x[..., 1::2]
|
| 54 |
return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
|
| 55 |
|
|
@@ -190,7 +200,11 @@ class DecoderLayer(nn.Module):
|
|
| 190 |
class MyTranslationForConditionalGeneration(PreTrainedModel):
|
| 191 |
config_class = MyTranslationConfig
|
| 192 |
base_model_prefix = ""
|
| 193 |
-
_tied_weights_keys =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
def __init__(self, config: MyTranslationConfig):
|
| 196 |
super().__init__(config)
|
|
@@ -225,7 +239,6 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
|
|
| 225 |
self.dec_norm = RMSNorm(config.d_model)
|
| 226 |
self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
|
| 227 |
self.post_init()
|
| 228 |
-
self.tie_weights()
|
| 229 |
|
| 230 |
def get_input_embeddings(self):
|
| 231 |
return self.embed_tokens
|
|
@@ -239,9 +252,6 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
|
|
| 239 |
def set_output_embeddings(self, new_embeddings):
|
| 240 |
self.lm_head = new_embeddings
|
| 241 |
|
| 242 |
-
def tie_weights(self, **kwargs):
|
| 243 |
-
self.lm_head.weight = self.embed_tokens.weight
|
| 244 |
-
|
| 245 |
def encode_and_route(self, src_ids: torch.Tensor):
|
| 246 |
enc_mask = (src_ids != self.config.pad_token_id).unsqueeze(1).unsqueeze(2)
|
| 247 |
x = self.embed_tokens(src_ids) * self.embed_scale
|
|
@@ -316,8 +326,7 @@ class MyTranslationForConditionalGeneration(PreTrainedModel):
|
|
| 316 |
k_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
|
| 317 |
v_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
|
| 318 |
cur_tok = torch.tensor([[BOS_ID]], dtype=torch.long, device=dev)
|
| 319 |
-
cos_all = self.rope.
|
| 320 |
-
sin_all = self.rope.sin_cached.to(dtype=dt, device=dev)
|
| 321 |
ys = []
|
| 322 |
for pos in range(max_new_tokens):
|
| 323 |
x_dec = self.embed_tokens(cur_tok) * self.embed_scale
|
|
|
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
| 4 |
import torch.nn.functional as F
|
|
|
|
| 5 |
from transformers import PreTrainedModel
|
| 6 |
from transformers.modeling_outputs import Seq2SeqLMOutput
|
| 7 |
from .configuration_mytranslation import MyTranslationConfig
|
|
|
|
| 10 |
BOS_ID = 1
|
| 11 |
EOS_ID = 2
|
| 12 |
TOK_2MY, TOK_2EN, TOK_FORMAL, TOK_SPOKEN = 4, 5, 6, 7
|
|
|
|
| 13 |
|
| 14 |
def _sdpa(q, k, v, attn_mask=None, is_causal=False):
|
| 15 |
if q.size(1) != k.size(1):
|
|
|
|
| 38 |
class RotaryEmbedding(nn.Module):
|
| 39 |
def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
|
| 40 |
super().__init__()
|
| 41 |
+
self.head_dim = dim
|
| 42 |
+
self.max_seq_len = max_seq_len
|
| 43 |
+
self.theta = theta
|
| 44 |
+
self._cos_live = None
|
| 45 |
+
self._sin_live = None
|
| 46 |
+
|
| 47 |
+
def get_cos_sin(self, seq_len: int, device: torch.device, dtype: torch.dtype):
|
| 48 |
+
if (
|
| 49 |
+
self._cos_live is None
|
| 50 |
+
or self._cos_live.device != device
|
| 51 |
+
or self._cos_live.size(2) < seq_len
|
| 52 |
+
):
|
| 53 |
+
L = max(self.max_seq_len, seq_len)
|
| 54 |
+
inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.head_dim, 2, device=device, dtype=torch.float32) / self.head_dim))
|
| 55 |
+
t = torch.arange(L, device=device, dtype=torch.float32)
|
| 56 |
+
freqs = torch.outer(t, inv_freq)
|
| 57 |
+
self._cos_live = freqs.cos()[None, None, :, :]
|
| 58 |
+
self._sin_live = freqs.sin()[None, None, :, :]
|
| 59 |
+
return self._cos_live[:, :, :seq_len, :].to(dtype=dtype), self._sin_live[:, :, :seq_len, :].to(dtype=dtype)
|
| 60 |
|
| 61 |
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 62 |
+
cos, sin = self.get_cos_sin(x.size(2), x.device, x.dtype)
|
|
|
|
|
|
|
| 63 |
x1, x2 = x[..., ::2], x[..., 1::2]
|
| 64 |
return torch.cat((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1)
|
| 65 |
|
|
|
|
| 200 |
class MyTranslationForConditionalGeneration(PreTrainedModel):
|
| 201 |
config_class = MyTranslationConfig
|
| 202 |
base_model_prefix = ""
|
| 203 |
+
_tied_weights_keys = (
|
| 204 |
+
{"lm_head.weight": "embed_tokens.weight"}
|
| 205 |
+
if hasattr(PreTrainedModel, "get_expanded_tied_weights_keys")
|
| 206 |
+
else ["lm_head.weight"]
|
| 207 |
+
)
|
| 208 |
|
| 209 |
def __init__(self, config: MyTranslationConfig):
|
| 210 |
super().__init__(config)
|
|
|
|
| 239 |
self.dec_norm = RMSNorm(config.d_model)
|
| 240 |
self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
|
| 241 |
self.post_init()
|
|
|
|
| 242 |
|
| 243 |
def get_input_embeddings(self):
|
| 244 |
return self.embed_tokens
|
|
|
|
| 252 |
def set_output_embeddings(self, new_embeddings):
|
| 253 |
self.lm_head = new_embeddings
|
| 254 |
|
|
|
|
|
|
|
|
|
|
| 255 |
def encode_and_route(self, src_ids: torch.Tensor):
|
| 256 |
enc_mask = (src_ids != self.config.pad_token_id).unsqueeze(1).unsqueeze(2)
|
| 257 |
x = self.embed_tokens(src_ids) * self.embed_scale
|
|
|
|
| 326 |
k_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
|
| 327 |
v_cache = torch.empty((self.dec_layers, 1, self.n_kv_heads, max_new_tokens, self.head_dim), dtype=dt, device=dev)
|
| 328 |
cur_tok = torch.tensor([[BOS_ID]], dtype=torch.long, device=dev)
|
| 329 |
+
cos_all, sin_all = self.rope.get_cos_sin(max_new_tokens, dev, dt)
|
|
|
|
| 330 |
ys = []
|
| 331 |
for pos in range(max_new_tokens):
|
| 332 |
x_dec = self.embed_tokens(cur_tok) * self.embed_scale
|