Download source/part2/model.py from walke007/anlp-assignment2-part2: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/model.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part2/source/part2/model.py
-
curl -L -o model.py https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/model.py
3.65 kB
| from dataclasses import dataclass | |
| import torch | |
| from torch import nn | |
| from .utils import Embedding, FFN, LayerNorm, MHA, PositionalEncoding, UnEmbedding, MOE | |
| class TransformerConfig: | |
| vocab_size: int | |
| context_size: int = 256 | |
| dimension_model: int = 256 | |
| attention_heads: int = 8 | |
| decoder_blocks: int = 4 | |
| def __post_init__(self): | |
| for name in ("vocab_size", "context_size", "dimension_model", "attention_heads", "decoder_blocks"): | |
| value = getattr(self, name) | |
| if isinstance(value, bool) or not isinstance(value, int) or value <= 0: | |
| raise ValueError(f"{name} must be a positive integer") | |
| if self.dimension_model % self.attention_heads: | |
| raise ValueError("dimension_model must be divisible by attention_heads") | |
| class DecoderBlock(nn.Module): | |
| def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None): | |
| super().__init__() | |
| self.mha = MHA(config.dimension_model, config.attention_heads) | |
| self.layernorm1 = LayerNorm(config.dimension_model) | |
| if moe_flag == True: | |
| self.ffn = MOE(config, experts, active, shared, hidden_dim=expert_hidden_dim) | |
| else: | |
| self.ffn = FFN(config.dimension_model) | |
| self.layernorm2 = LayerNorm(config.dimension_model) | |
| def forward(self, hidden_states, padding_mask=None): | |
| hidden_states = self.layernorm1( | |
| hidden_states + self.mha(hidden_states, padding_mask) | |
| ) | |
| hidden_states = self.layernorm2(hidden_states + self.ffn(hidden_states)) | |
| if padding_mask is not None: | |
| hidden_states = hidden_states.masked_fill(padding_mask[:, :, None], 0.0) | |
| return hidden_states | |
| class DecoderOnlyTransformer(nn.Module): | |
| def __init__(self, config: TransformerConfig, moe_flag : bool, experts, active, shared, expert_hidden_dim=None): | |
| super().__init__() | |
| self.config = config | |
| self.embedding = Embedding(config.vocab_size, config.dimension_model) | |
| self.positional_encoding = PositionalEncoding(config.context_size, config.dimension_model) | |
| self.decoder_list = nn.ModuleList( | |
| DecoderBlock(config, moe_flag, experts, active, shared, expert_hidden_dim) for _ in range(config.decoder_blocks) | |
| ) | |
| self.unembedding = UnEmbedding(config.vocab_size, config.dimension_model) | |
| def forward(self, input_ids, padding_mask=None): | |
| hidden_states = self.positional_encoding(self.embedding(input_ids)) | |
| for decoder in self.decoder_list: | |
| hidden_states = decoder(hidden_states, padding_mask) | |
| return self.unembedding(hidden_states) | |
| def generate(self, input_ids, max_new_tokens=10, eos_token_id=None): | |
| was_training = self.training | |
| self.eval() | |
| try: | |
| generated = input_ids.clone() | |
| finished = torch.zeros(input_ids.size(0), dtype=torch.bool, device=input_ids.device) | |
| for _ in range(max_new_tokens): | |
| logits = self(generated[:, -self.config.context_size:]) | |
| next_token = logits[:, -1, :].argmax(dim=-1) | |
| if eos_token_id is not None: | |
| next_token = next_token.masked_fill(finished, eos_token_id) | |
| finished = finished | (next_token == eos_token_id) | |
| generated = torch.cat((generated, next_token[:, None]), dim=1) | |
| if eos_token_id is not None and finished.all(): | |
| break | |
| return generated | |
| finally: | |
| self.train(was_training) | |