File size: 299 Bytes
42c70da | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | {
"vocab_size": 16000,
"n_ctx": 256,
"rope_base": 10000.0,
"d_model": 256,
"n_heads": 8,
"n_layers": 8,
"d_ff": 1024,
"dropout": 0.1,
"tie_embeddings": true,
"use_moe": false,
"n_experts": 4,
"n_active": 1,
"n_shared": 0,
"match_active": false,
"moe_aux_weight": 0.01
} |