File size: 3,649 Bytes
a3baf17 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
from dataclasses import dataclass
import torch
from torch import nn
from .utils import Embedding, FFN, LayerNorm, MHA, PositionalEncoding, UnEmbedding, MOE
@dataclass(frozen=True)
class TransformerConfig:
vocab_size: int
context_size: int = 256
dimension_model: int = 256
attention_heads: int = 8
decoder_blocks: int = 4
def __post_init__(self):
for name in ("vocab_size", "context_size", "dimension_model", "attention_heads", "decoder_blocks"):
value = getattr(self, name)
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise ValueError(f"{name} must be a positive integer")
if self.dimension_model % self.attention_heads:
raise ValueError("dimension_model must be divisible by attention_heads")
class DecoderBlock(nn.Module):
def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
super().__init__()
self.mha = MHA(config.dimension_model, config.attention_heads)
self.layernorm1 = LayerNorm(config.dimension_model)
if moe_flag == True:
self.ffn = MOE(config, experts, active, shared, hidden_dim=expert_hidden_dim)
else:
self.ffn = FFN(config.dimension_model)
self.layernorm2 = LayerNorm(config.dimension_model)
def forward(self, hidden_states, padding_mask=None):
hidden_states = self.layernorm1(
hidden_states + self.mha(hidden_states, padding_mask)
)
hidden_states = self.layernorm2(hidden_states + self.ffn(hidden_states))
if padding_mask is not None:
hidden_states = hidden_states.masked_fill(padding_mask[:, :, None], 0.0)
return hidden_states
class DecoderOnlyTransformer(nn.Module):
def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
super().__init__()
self.config = config
self.embedding = Embedding(config.vocab_size, config.dimension_model)
self.positional_encoding = PositionalEncoding(config.context_size, config.dimension_model)
self.decoder_list = nn.ModuleList(
DecoderBlock(config, moe_flag, experts, active, shared, expert_hidden_dim) for _ in range(config.decoder_blocks)
)
self.unembedding = UnEmbedding(config.vocab_size, config.dimension_model)
def forward(self, input_ids, padding_mask=None):
hidden_states = self.positional_encoding(self.embedding(input_ids))
for decoder in self.decoder_list:
hidden_states = decoder(hidden_states, padding_mask)
return self.unembedding(hidden_states)
@torch.no_grad()
def generate(self, input_ids, max_new_tokens=10, eos_token_id=None):
was_training = self.training
self.eval()
try:
generated = input_ids.clone()
finished = torch.zeros(input_ids.size(0), dtype=torch.bool, device=input_ids.device)
for _ in range(max_new_tokens):
logits = self(generated[:, -self.config.context_size:])
next_token = logits[:, -1, :].argmax(dim=-1)
if eos_token_id is not None:
next_token = next_token.masked_fill(finished, eos_token_id)
finished = finished | (next_token == eos_token_id)
generated = torch.cat((generated, next_token[:, None]), dim=1)
if eos_token_id is not None and finished.all():
break
return generated
finally:
self.train(was_training)
|