File size: 1,138 Bytes
ffe4b06 886bcaa ffe4b06 886bcaa ffe4b06 886bcaa ffe4b06 886bcaa ffe4b06 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | # configuration_easyformer.py
from transformers import PretrainedConfig
class EasyFormerConfig(PretrainedConfig):
model_type = "transformer"
def __init__(
self,
vocab_size=8000,
d_model=192,
n_layer=5,
n_head=4,
ctx=128,
dropout=0.1,
**kwargs,
):
super().__init__(**kwargs)
self.vocab_size = vocab_size
self.d_model = d_model
self.n_layer = n_layer
self.n_head = n_head
self.ctx = ctx
self.dropout = dropout
# --- HF-совместимые property ---
@property
def num_hidden_layers(self):
return self.n_layer
@property
def hidden_size(self):
return self.d_model
@property
def num_attention_heads(self):
return self.n_head
@property
def intermediate_size(self):
return 2 * self.d_model
@property
def max_position_embeddings(self):
return self.ctx
@property
def num_key_value_heads(self):
return self.n_head
@property
def head_dim(self):
return self.d_model // max(1, self.n_head) |