from dataclasses import dataclass import torch from torch import nn from .utils import Embedding, FFN, LayerNorm, MHA, PositionalEncoding, UnEmbedding, MOE @dataclass(frozen=True) class TransformerConfig: vocab_size: int context_size: int = 256 dimension_model: int = 256 attention_heads: int = 8 decoder_blocks: int = 4 def __post_init__(self): for name in ("vocab_size", "context_size", "dimension_model", "attention_heads", "decoder_blocks"): value = getattr(self, name) if isinstance(value, bool) or not isinstance(value, int) or value <= 0: raise ValueError(f"{name} must be a positive integer") if self.dimension_model % self.attention_heads: raise ValueError("dimension_model must be divisible by attention_heads") class DecoderBlock(nn.Module): def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None): super().__init__() self.mha = MHA(config.dimension_model, config.attention_heads) self.layernorm1 = LayerNorm(config.dimension_model) if moe_flag == True: self.ffn = MOE(config, experts, active, shared, hidden_dim=expert_hidden_dim) else: self.ffn = FFN(config.dimension_model) self.layernorm2 = LayerNorm(config.dimension_model) def forward(self, hidden_states, padding_mask=None): hidden_states = self.layernorm1( hidden_states + self.mha(hidden_states, padding_mask) ) hidden_states = self.layernorm2(hidden_states + self.ffn(hidden_states)) if padding_mask is not None: hidden_states = hidden_states.masked_fill(padding_mask[:, :, None], 0.0) return hidden_states class DecoderOnlyTransformer(nn.Module): def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None): super().__init__() self.config = config self.embedding = Embedding(config.vocab_size, config.dimension_model) self.positional_encoding = PositionalEncoding(config.context_size, config.dimension_model) self.decoder_list = nn.ModuleList( DecoderBlock(config, moe_flag, experts, active, shared, expert_hidden_dim) for _ in range(config.decoder_blocks) ) self.unembedding = UnEmbedding(config.vocab_size, config.dimension_model) def forward(self, input_ids, padding_mask=None): hidden_states = self.positional_encoding(self.embedding(input_ids)) for decoder in self.decoder_list: hidden_states = decoder(hidden_states, padding_mask) return self.unembedding(hidden_states) @torch.no_grad() def generate(self, input_ids, max_new_tokens=10, eos_token_id=None): was_training = self.training self.eval() try: generated = input_ids.clone() finished = torch.zeros(input_ids.size(0), dtype=torch.bool, device=input_ids.device) for _ in range(max_new_tokens): logits = self(generated[:, -self.config.context_size:]) next_token = logits[:, -1, :].argmax(dim=-1) if eos_token_id is not None: next_token = next_token.masked_fill(finished, eos_token_id) finished = finished | (next_token == eos_token_id) generated = torch.cat((generated, next_token[:, None]), dim=1) if eos_token_id is not None and finished.all(): break return generated finally: self.train(was_training)