abyss-api / ml /model /config.py
KyleEastwood's picture
Upload folder using huggingface_hub
cc15101 verified
Raw History Blame Contribute Delete
916 Bytes
"""
AbyssConfig: every hyperparameter that defines the Abyss-1 model, in
tne place so the model, trainer, server and agent later all agree.
"""
from dataclasses import dataclass
@dataclass
class AbyssConfig:
# --- size ---
vocab_size: int = 16384
block_size: int = 512
n_layer: int = 8
n_head: int = 8
n_embd: int = 512
# --- options ---
dropout: float = 0.0
bias: bool = False
rope_theta: float = 10000.0
norm_eps: float = 1e-5
tie_embeddings: bool = False
@property
def head_dim(self):
# width of one attention head; must be even so RoPE can pair dims
assert self.n_embd % self.n_head == 0, "n_embd must divide n_head"
return self.n_embd // self.n_head
@property
def n_inner(self):
# SwiGLU hidden width: plain 4x the model width (decision #14)
return 4 * self.n_embd