File size: 1,401 Bytes
9cb442b 8e8ba17 9cb442b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | from transformers import PreTrainedConfig
class CheddarConfig(PreTrainedConfig):
model_type = "gouda-cheddar"
attribute_map = {
"num_hidden_layers": "n_layer",
"hidden_size": "n_embd",
"num_attention_heads": "n_head",
"num_key_value_heads": "n_kv_head",
"intermediate_size": "mlp_hidden",
"max_position_embeddings": "max_seq_len",
}
def __init__(
self,
vocab_size=32768,
max_seq_len=4096,
n_layer=44,
n_embd=768,
n_head=12,
n_kv_head=4,
mlp_hidden=2048,
rope_base=10000.0,
rms_eps=1e-6,
qk_norm=True,
z_loss=0.0,
**kwargs,
):
super().__init__(**kwargs)
self.vocab_size = vocab_size
self.max_seq_len = max_seq_len
self.n_layer = n_layer
self.n_embd = n_embd
self.n_head = n_head
self.n_kv_head = n_kv_head
self.mlp_hidden = mlp_hidden
self.rope_base = rope_base
self.rms_eps = rms_eps
self.qk_norm = qk_norm
self.z_loss = z_loss
if self.n_embd % self.n_head:
raise ValueError("n_embd must be divisible by n_head")
if self.n_head % self.n_kv_head:
raise ValueError("n_head must be divisible by n_kv_head")
@property
def head_dim(self):
return self.n_embd // self.n_head |