mahwizzzz commited on
Commit
3728182
·
verified ·
1 Parent(s): 2f55be5

Add Transformers AutoModelForMaskedLM remote code

Browse files
Files changed (1) hide show
  1. configuration_pakmosaic.py +71 -0
configuration_pakmosaic.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hugging Face config for PakMosaic. Custom architecture, not BERT."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from transformers import PretrainedConfig
6
+
7
+
8
+ class PakMosaicConfig(PretrainedConfig):
9
+ model_type = "pakmosaic"
10
+
11
+ def __init__(
12
+ self,
13
+ vocab_size: int = 32000,
14
+ hidden_size: int = 512,
15
+ num_hidden_layers: int = 12,
16
+ num_attention_heads: int = 8,
17
+ intermediate_size: int = 2048,
18
+ max_position_embeddings: int = 512,
19
+ rope_theta: float = 10000.0,
20
+ attention_dropout: float = 0.0,
21
+ hidden_dropout: float = 0.0,
22
+ ffn_activation: str = "geglu",
23
+ attention_kind: str = "full",
24
+ local_window: int = 128,
25
+ global_every: int = 3,
26
+ use_bias: bool = False,
27
+ tie_word_embeddings: bool = True,
28
+ pad_token_id: int = 0,
29
+ bos_token_id: int = 1,
30
+ eos_token_id: int = 2,
31
+ unk_token_id: int = 3,
32
+ mask_token_id: int = 4,
33
+ layer_norm_eps: float = 1e-5,
34
+ initializer_range: float = 0.02,
35
+ script_signal: str = "none",
36
+ **kwargs,
37
+ ) -> None:
38
+ kwargs.pop("algebraic_parameters", None)
39
+ super().__init__(
40
+ pad_token_id=pad_token_id,
41
+ bos_token_id=bos_token_id,
42
+ eos_token_id=eos_token_id,
43
+ **kwargs,
44
+ )
45
+ self.vocab_size = vocab_size
46
+ self.hidden_size = hidden_size
47
+ self.num_hidden_layers = num_hidden_layers
48
+ self.num_attention_heads = num_attention_heads
49
+ self.intermediate_size = intermediate_size
50
+ self.max_position_embeddings = max_position_embeddings
51
+ self.rope_theta = rope_theta
52
+ self.attention_dropout = attention_dropout
53
+ self.hidden_dropout = hidden_dropout
54
+ self.ffn_activation = ffn_activation
55
+ self.attention_kind = attention_kind
56
+ self.local_window = local_window
57
+ self.global_every = global_every
58
+ self.use_bias = use_bias
59
+ self.tie_word_embeddings = tie_word_embeddings
60
+ self.unk_token_id = unk_token_id
61
+ self.mask_token_id = mask_token_id
62
+ self.layer_norm_eps = layer_norm_eps
63
+ self.initializer_range = initializer_range
64
+ self.script_signal = script_signal
65
+
66
+ @property
67
+ def head_dim(self) -> int:
68
+ return self.hidden_size // self.num_attention_heads
69
+
70
+
71
+ PakMosaicConfig.register_for_auto_class()