File size: 3,649 Bytes
a3baf17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

from dataclasses import dataclass

import torch
from torch import nn

from .utils import Embedding, FFN, LayerNorm, MHA, PositionalEncoding, UnEmbedding, MOE



@dataclass(frozen=True)
class TransformerConfig:
    vocab_size: int
    context_size: int = 256
    dimension_model: int = 256
    attention_heads: int = 8
    decoder_blocks: int = 4


    def __post_init__(self):
        for name in ("vocab_size", "context_size", "dimension_model", "attention_heads", "decoder_blocks"):
            value = getattr(self, name)
            if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
                raise ValueError(f"{name} must be a positive integer")
        if self.dimension_model % self.attention_heads:
            raise ValueError("dimension_model must be divisible by attention_heads")


class DecoderBlock(nn.Module):
    def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
        super().__init__()
        self.mha = MHA(config.dimension_model, config.attention_heads)
        self.layernorm1 = LayerNorm(config.dimension_model)
        if moe_flag == True:
            self.ffn = MOE(config, experts, active, shared, hidden_dim=expert_hidden_dim)
        else:
            self.ffn = FFN(config.dimension_model)
        self.layernorm2 = LayerNorm(config.dimension_model)

    def forward(self, hidden_states, padding_mask=None):

        hidden_states = self.layernorm1(
            hidden_states + self.mha(hidden_states, padding_mask)
        )
        hidden_states = self.layernorm2(hidden_states + self.ffn(hidden_states))
        if padding_mask is not None:
            hidden_states = hidden_states.masked_fill(padding_mask[:, :, None], 0.0)
        return hidden_states


class DecoderOnlyTransformer(nn.Module):
    def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None):
        super().__init__()
        self.config = config
        self.embedding = Embedding(config.vocab_size, config.dimension_model)
        self.positional_encoding = PositionalEncoding(config.context_size, config.dimension_model)
        self.decoder_list = nn.ModuleList(
            DecoderBlock(config, moe_flag, experts, active, shared, expert_hidden_dim) for _ in range(config.decoder_blocks)
        )
        self.unembedding = UnEmbedding(config.vocab_size, config.dimension_model)

    def forward(self, input_ids, padding_mask=None):

        hidden_states = self.positional_encoding(self.embedding(input_ids))
        for decoder in self.decoder_list:
            hidden_states = decoder(hidden_states, padding_mask)
        return self.unembedding(hidden_states)

    @torch.no_grad()
    def generate(self, input_ids, max_new_tokens=10, eos_token_id=None):
        was_training = self.training
        self.eval()
        try:
            generated = input_ids.clone()
            finished = torch.zeros(input_ids.size(0), dtype=torch.bool, device=input_ids.device)
            for _ in range(max_new_tokens):
                logits = self(generated[:, -self.config.context_size:])
                next_token = logits[:, -1, :].argmax(dim=-1)
                if eos_token_id is not None:
                    next_token = next_token.masked_fill(finished, eos_token_id)
                    finished = finished | (next_token == eos_token_id)
                generated = torch.cat((generated, next_token[:, None]), dim=1)
                if eos_token_id is not None and finished.all():
                    break
            return generated
        finally:
            self.train(was_training)