Spaces:
Runtime error
Runtime error
Download ml/model/config.py from TheAbyss99/abyss-api: direct link, hf CLI and curl.
- Browser
- Download file 916 Bytes
-
https://huggingface.co/spaces/TheAbyss99/abyss-api/resolve/main/ml/model/config.py
- Command line
-
hf download hf://spaces/TheAbyss99/abyss-api/ml/model/config.py
-
curl -L -o config.py https://huggingface.co/spaces/TheAbyss99/abyss-api/resolve/main/ml/model/config.py
916 Bytes
| """ | |
| AbyssConfig: every hyperparameter that defines the Abyss-1 model, in | |
| tne place so the model, trainer, server and agent later all agree. | |
| """ | |
| from dataclasses import dataclass | |
| class AbyssConfig: | |
| # --- size --- | |
| vocab_size: int = 16384 | |
| block_size: int = 512 | |
| n_layer: int = 8 | |
| n_head: int = 8 | |
| n_embd: int = 512 | |
| # --- options --- | |
| dropout: float = 0.0 | |
| bias: bool = False | |
| rope_theta: float = 10000.0 | |
| norm_eps: float = 1e-5 | |
| tie_embeddings: bool = False | |
| def head_dim(self): | |
| # width of one attention head; must be even so RoPE can pair dims | |
| assert self.n_embd % self.n_head == 0, "n_embd must divide n_head" | |
| return self.n_embd // self.n_head | |
| def n_inner(self): | |
| # SwiGLU hidden width: plain 4x the model width (decision #14) | |
| return 4 * self.n_embd |