""" AbyssConfig: every hyperparameter that defines the Abyss-1 model, in tne place so the model, trainer, server and agent later all agree. """ from dataclasses import dataclass @dataclass class AbyssConfig: # --- size --- vocab_size: int = 16384 block_size: int = 512 n_layer: int = 8 n_head: int = 8 n_embd: int = 512 # --- options --- dropout: float = 0.0 bias: bool = False rope_theta: float = 10000.0 norm_eps: float = 1e-5 tie_embeddings: bool = False @property def head_dim(self): # width of one attention head; must be even so RoPE can pair dims assert self.n_embd % self.n_head == 0, "n_embd must divide n_head" return self.n_embd // self.n_head @property def n_inner(self): # SwiGLU hidden width: plain 4x the model width (decision #14) return 4 * self.n_embd