mahwizzzz commited on
Commit
2dcd089
·
verified ·
1 Parent(s): 3728182

Add Transformers AutoModelForMaskedLM remote code

Browse files
Files changed (1) hide show
  1. modeling_pakmosaic.py +238 -0
modeling_pakmosaic.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Transformers wrapper for PakMosaic. Same weights as PakEncoderForMLM."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ import torch
8
+ import torch.nn.functional as F
9
+ from torch import nn
10
+ from transformers import PreTrainedModel
11
+ from transformers.modeling_outputs import BaseModelOutput, MaskedLMOutput
12
+
13
+ try:
14
+ from .configuration_pakmosaic import PakMosaicConfig
15
+ except ImportError:
16
+ from configuration_pakmosaic import PakMosaicConfig
17
+
18
+
19
+ def rotate_half(x: torch.Tensor) -> torch.Tensor:
20
+ x1, x2 = x.chunk(2, dim=-1)
21
+ return torch.cat((-x2, x1), dim=-1)
22
+
23
+
24
+ class RotaryEmbedding(nn.Module):
25
+ def __init__(self, dim: int, max_seq_len: int = 512, base: float = 10000.0):
26
+ super().__init__()
27
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
28
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
29
+ self.max_seq_len = max_seq_len
30
+
31
+ def forward(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
32
+ positions = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
33
+ freqs = torch.outer(positions, self.inv_freq)
34
+ emb = torch.cat((freqs, freqs), dim=-1)
35
+ return emb.cos().to(dtype=dtype), emb.sin().to(dtype=dtype)
36
+
37
+
38
+ def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
39
+ cos = cos.unsqueeze(0).unsqueeze(0)
40
+ sin = sin.unsqueeze(0).unsqueeze(0)
41
+ return (x * cos) + (rotate_half(x) * sin)
42
+
43
+
44
+ def sliding_window_keep(seq_len: int, window: int, device: torch.device) -> torch.Tensor:
45
+ idx = torch.arange(seq_len, device=device)
46
+ return (idx[:, None] - idx[None, :]).abs() <= window
47
+
48
+
49
+ class EncoderAttention(nn.Module):
50
+ def __init__(self, config: PakMosaicConfig, *, layer_index: int):
51
+ super().__init__()
52
+ self.config = config
53
+ self.num_heads = config.num_attention_heads
54
+ self.head_dim = config.head_dim
55
+ self.qkv = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=config.use_bias)
56
+ self.out = nn.Linear(config.hidden_size, config.hidden_size, bias=config.use_bias)
57
+ self.dropout = config.attention_dropout
58
+ self.rope = RotaryEmbedding(self.head_dim, config.max_position_embeddings, config.rope_theta)
59
+ self.is_global = config.attention_kind == "full" or (
60
+ config.attention_kind == "hybrid" and ((layer_index + 1) % max(1, config.global_every) == 0)
61
+ )
62
+
63
+ def _mask(self, attention_mask: torch.Tensor | None, seq_len: int) -> torch.Tensor | None:
64
+ keep = None
65
+ if attention_mask is not None:
66
+ keep = attention_mask.bool()[:, None, None, :]
67
+ if not self.is_global:
68
+ window = sliding_window_keep(seq_len, self.config.local_window, self.qkv.weight.device)[None, None, :, :]
69
+ keep = window if keep is None else keep & window
70
+ return keep
71
+
72
+ def forward(self, hidden: torch.Tensor, attention_mask: torch.Tensor | None = None) -> torch.Tensor:
73
+ batch, seq_len, _ = hidden.shape
74
+ qkv = self.qkv(hidden).view(batch, seq_len, 3, self.num_heads, self.head_dim)
75
+ qkv = qkv.permute(2, 0, 3, 1, 4)
76
+ query, key, value = qkv[0], qkv[1], qkv[2]
77
+ cos, sin = self.rope(seq_len, hidden.device, query.dtype)
78
+ query = apply_rope(query, cos, sin)
79
+ key = apply_rope(key, cos, sin)
80
+ context = F.scaled_dot_product_attention(
81
+ query,
82
+ key,
83
+ value,
84
+ attn_mask=self._mask(attention_mask, seq_len),
85
+ dropout_p=self.dropout if self.training else 0.0,
86
+ )
87
+ context = context.transpose(1, 2).contiguous().view(batch, seq_len, self.config.hidden_size)
88
+ return self.out(context)
89
+
90
+
91
+ class EncoderFFN(nn.Module):
92
+ def __init__(self, config: PakMosaicConfig):
93
+ super().__init__()
94
+ self.kind = config.ffn_activation
95
+ bias = config.use_bias
96
+ if self.kind in {"geglu", "swiglu"}:
97
+ self.gate_up = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=bias)
98
+ self.down = nn.Linear(config.intermediate_size, config.hidden_size, bias=bias)
99
+ else:
100
+ self.up = nn.Linear(config.hidden_size, config.intermediate_size, bias=bias)
101
+ self.down = nn.Linear(config.intermediate_size, config.hidden_size, bias=bias)
102
+ self.act = nn.GELU()
103
+ self.dropout = nn.Dropout(config.hidden_dropout)
104
+
105
+ def forward(self, hidden: torch.Tensor) -> torch.Tensor:
106
+ if self.kind in {"geglu", "swiglu"}:
107
+ gate, up = self.gate_up(hidden).chunk(2, dim=-1)
108
+ hidden = (F.silu(gate) if self.kind == "swiglu" else F.gelu(gate)) * up
109
+ else:
110
+ hidden = self.act(self.up(hidden))
111
+ return self.dropout(self.down(hidden))
112
+
113
+
114
+ class TokenEmbeddings(nn.Module):
115
+ def __init__(self, config: PakMosaicConfig):
116
+ super().__init__()
117
+ self.word = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
118
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
119
+ self.dropout = nn.Dropout(config.hidden_dropout)
120
+
121
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
122
+ return self.dropout(self.norm(self.word(input_ids)))
123
+
124
+
125
+ class EncoderLayer(nn.Module):
126
+ def __init__(self, config: PakMosaicConfig, layer_index: int):
127
+ super().__init__()
128
+ self.attn_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
129
+ self.ffn_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
130
+ self.attn = EncoderAttention(config, layer_index=layer_index)
131
+ self.ffn = EncoderFFN(config)
132
+
133
+ def forward(self, hidden: torch.Tensor, attention_mask: torch.Tensor | None = None) -> torch.Tensor:
134
+ hidden = hidden + self.attn(self.attn_norm(hidden), attention_mask=attention_mask)
135
+ return hidden + self.ffn(self.ffn_norm(hidden))
136
+
137
+
138
+ class PakMosaicPreTrainedModel(PreTrainedModel):
139
+ config_class = PakMosaicConfig
140
+ base_model_prefix = "encoder"
141
+ supports_gradient_checkpointing = True
142
+ _no_split_modules = ["EncoderLayer"]
143
+
144
+ def _init_weights(self, module: nn.Module) -> None:
145
+ std = self.config.initializer_range
146
+ if isinstance(module, nn.Linear):
147
+ module.weight.data.normal_(mean=0.0, std=std)
148
+ if module.bias is not None:
149
+ module.bias.data.zero_()
150
+ elif isinstance(module, nn.Embedding):
151
+ module.weight.data.normal_(mean=0.0, std=std)
152
+ if module.padding_idx is not None:
153
+ module.weight.data[module.padding_idx].zero_()
154
+ elif isinstance(module, nn.LayerNorm):
155
+ module.weight.data.fill_(1.0)
156
+ module.bias.data.zero_()
157
+
158
+
159
+ class PakMosaicModel(PakMosaicPreTrainedModel):
160
+ def __init__(self, config: PakMosaicConfig):
161
+ super().__init__(config)
162
+ self.embeddings = TokenEmbeddings(config)
163
+ self.layers = nn.ModuleList(EncoderLayer(config, i) for i in range(config.num_hidden_layers))
164
+ self.final_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
165
+ self.gradient_checkpointing = False
166
+ self.post_init()
167
+
168
+ def get_input_embeddings(self) -> nn.Embedding:
169
+ return self.embeddings.word
170
+
171
+ def set_input_embeddings(self, value: nn.Embedding) -> None:
172
+ self.embeddings.word = value
173
+
174
+ def forward(
175
+ self,
176
+ input_ids: torch.Tensor | None = None,
177
+ attention_mask: torch.Tensor | None = None,
178
+ **kwargs: Any,
179
+ ) -> BaseModelOutput:
180
+ hidden = self.embeddings(input_ids)
181
+ for layer in self.layers:
182
+ if self.gradient_checkpointing and self.training:
183
+ hidden = self._gradient_checkpointing_func(layer, hidden, attention_mask)
184
+ else:
185
+ hidden = layer(hidden, attention_mask=attention_mask)
186
+ hidden = self.final_norm(hidden)
187
+ return BaseModelOutput(last_hidden_state=hidden)
188
+
189
+
190
+ class MLMHead(nn.Module):
191
+ def __init__(self, config: PakMosaicConfig, embedding: nn.Embedding):
192
+ super().__init__()
193
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
194
+ self.act = nn.GELU()
195
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
196
+ self.tie = config.tie_word_embeddings
197
+ if config.tie_word_embeddings:
198
+ self.weight = embedding.weight
199
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
200
+ else:
201
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=True)
202
+
203
+ def forward(self, hidden: torch.Tensor) -> torch.Tensor:
204
+ hidden = self.norm(self.act(self.dense(hidden)))
205
+ if self.tie:
206
+ return F.linear(hidden, self.weight, self.bias)
207
+ return self.decoder(hidden)
208
+
209
+
210
+ class PakMosaicForMaskedLM(PakMosaicPreTrainedModel):
211
+ _tied_weights_keys = {"mlm.weight": "encoder.embeddings.word.weight"}
212
+
213
+ def __init__(self, config: PakMosaicConfig):
214
+ super().__init__(config)
215
+ self.encoder = PakMosaicModel(config)
216
+ self.mlm = MLMHead(config, self.encoder.embeddings.word)
217
+ self.post_init()
218
+
219
+ def get_input_embeddings(self) -> nn.Embedding:
220
+ return self.encoder.embeddings.word
221
+
222
+ def forward(
223
+ self,
224
+ input_ids: torch.Tensor | None = None,
225
+ attention_mask: torch.Tensor | None = None,
226
+ labels: torch.Tensor | None = None,
227
+ **kwargs: Any,
228
+ ) -> MaskedLMOutput:
229
+ hidden = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
230
+ logits = self.mlm(hidden)
231
+ loss = None
232
+ if labels is not None:
233
+ loss = F.cross_entropy(logits.view(-1, self.config.vocab_size), labels.view(-1), ignore_index=-100)
234
+ return MaskedLMOutput(loss=loss, logits=logits, hidden_states=(hidden,))
235
+
236
+
237
+ PakMosaicModel.register_for_auto_class("AutoModel")
238
+ PakMosaicForMaskedLM.register_for_auto_class("AutoModelForMaskedLM")