| { | |
| "vocab_size": 16384, | |
| "context_length": 128, | |
| "d_model": 320, | |
| "n_heads": 5, | |
| "n_layers": 6, | |
| "d_ff": 832, | |
| "dropout": 0.0, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-05, | |
| "activation": "swiglu", | |
| "bias": false, | |
| "tie_embeddings": true, | |
| "position_embedding": "learned" | |
| } | |