| { | |
| "vocab_size": 16000, | |
| "n_ctx": 256, | |
| "rope_base": 10000.0, | |
| "d_model": 256, | |
| "n_heads": 8, | |
| "n_layers": 6, | |
| "d_ff": 1024, | |
| "dropout": 0.1, | |
| "tie_embeddings": true, | |
| "use_moe": true, | |
| "n_experts": 4, | |
| "n_active": 2, | |
| "n_shared": 0, | |
| "match_active": true, | |
| "moe_aux_weight": 0.01 | |
| } |