walke007's picture
Upload Part 2 checkpoints and recorded results
a3baf17 verified
Raw History Blame Contribute Delete
3.65 kB
from dataclasses import dataclass
import torch
from torch import nn
from .utils import Embedding, FFN, LayerNorm, MHA, PositionalEncoding, UnEmbedding, MOE
@dataclass(frozen=True)
class TransformerConfig:
vocab_size: int
context_size: int = 256
dimension_model: int = 256
attention_heads: int = 8
decoder_blocks: int = 4
def __post_init__(self):
for name in ("vocab_size", "context_size", "dimension_model", "attention_heads", "decoder_blocks"):
value = getattr(self, name)
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise ValueError(f"{name} must be a positive integer")
if self.dimension_model % self.attention_heads:
raise ValueError("dimension_model must be divisible by attention_heads")
class DecoderBlock(nn.Module):
def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
super().__init__()
self.mha = MHA(config.dimension_model, config.attention_heads)
self.layernorm1 = LayerNorm(config.dimension_model)
if moe_flag == True:
self.ffn = MOE(config, experts, active, shared, hidden_dim=expert_hidden_dim)
else:
self.ffn = FFN(config.dimension_model)
self.layernorm2 = LayerNorm(config.dimension_model)
def forward(self, hidden_states, padding_mask=None):
hidden_states = self.layernorm1(
hidden_states + self.mha(hidden_states, padding_mask)
)
hidden_states = self.layernorm2(hidden_states + self.ffn(hidden_states))
if padding_mask is not None:
hidden_states = hidden_states.masked_fill(padding_mask[:, :, None], 0.0)
return hidden_states
class DecoderOnlyTransformer(nn.Module):
def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
super().__init__()
self.config = config
self.embedding = Embedding(config.vocab_size, config.dimension_model)
self.positional_encoding = PositionalEncoding(config.context_size, config.dimension_model)
self.decoder_list = nn.ModuleList(
DecoderBlock(config, moe_flag, experts, active, shared, expert_hidden_dim) for _ in range(config.decoder_blocks)
)
self.unembedding = UnEmbedding(config.vocab_size, config.dimension_model)
def forward(self, input_ids, padding_mask=None):
hidden_states = self.positional_encoding(self.embedding(input_ids))
for decoder in self.decoder_list:
hidden_states = decoder(hidden_states, padding_mask)
return self.unembedding(hidden_states)
@torch.no_grad()
def generate(self, input_ids, max_new_tokens=10, eos_token_id=None):
was_training = self.training
self.eval()
try:
generated = input_ids.clone()
finished = torch.zeros(input_ids.size(0), dtype=torch.bool, device=input_ids.device)
for _ in range(max_new_tokens):
logits = self(generated[:, -self.config.context_size:])
next_token = logits[:, -1, :].argmax(dim=-1)
if eos_token_id is not None:
next_token = next_token.masked_fill(finished, eos_token_id)
finished = finished | (next_token == eos_token_id)
generated = torch.cat((generated, next_token[:, None]), dim=1)
if eos_token_id is not None and finished.all():
break
return generated
finally:
self.train(was_training)