File size: 2,698 Bytes
a15230d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | import torch
import torch.nn as nn
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutputWithPast
from .configuration_tinyindianlm import TinyIndianLMConfig
from .model import ModelConfig, TinyIndianLM
class TinyIndianLMForCausalLM(PreTrainedModel):
config_class = TinyIndianLMConfig
base_model_prefix = "model"
supports_gradient_checkpointing = False
def __init__(self, config: TinyIndianLMConfig):
super().__init__(config)
internal_cfg = ModelConfig(
vocab_size=config.vocab_size,
d_model=config.d_model,
n_layers=config.n_layers,
n_heads=config.n_heads,
n_kv_heads=config.n_kv_heads,
max_seq_len=config.max_seq_len,
d_ffn=config.d_ffn,
dropout=config.dropout,
rope_theta=config.rope_theta,
)
self.model = TinyIndianLM(internal_cfg)
self.post_init()
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embedding
def set_input_embeddings(self, value: nn.Embedding) -> None:
self.model.embedding = value
def get_output_embeddings(self) -> nn.Linear:
return self.model.output
def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
self.model.output = new_embeddings
def forward(
self,
input_ids=None,
attention_mask=None,
labels=None,
**kwargs,
):
del attention_mask, kwargs
# Safety guard for exported repos where tokenizer vocab and
# model embedding size diverge. Any out-of-range token id is
# mapped to unk to avoid embedding lookup crashes.
if input_ids is not None:
vocab_size = self.model.embedding.num_embeddings
if (input_ids >= vocab_size).any():
unk_id = getattr(self.config, "unk_token_id", 1)
input_ids = input_ids.clone()
input_ids[input_ids >= vocab_size] = unk_id
logits, loss = self.model(
input_ids,
targets=labels,
pad_id=self.config.pad_token_id,
)
return CausalLMOutputWithPast(
loss=loss,
logits=logits,
past_key_values=None,
hidden_states=None,
attentions=None,
)
def prepare_inputs_for_generation(self, input_ids, **kwargs):
del kwargs
return {"input_ids": input_ids}
@staticmethod
def _reorder_cache(past_key_values, beam_idx):
del beam_idx
return past_key_values
|