config.json

{
    "vocab_size": 50257,
    "block_size": 512,
    "d_model": 512,
    "n_layers": 8,
    "n_heads": 8,
    "use_moe": true,
    "num_experts": 8,
    "top_k": 2,
    "dense_ffn_dim": 1536,
    "expert_ffn_dim": 768,
    "dropout": 0.0
}