config.json

{
    "architectures": [
        "MiniMoE50M"
    ],
    "model_type": "mini-moe",
    "vocab_size": 16000,
    "d_model": 512,
    "n_heads": 8,
    "d_ff": 1536,
    "n_layers": 5,
    "num_experts": 4,
    "task_specific_params": {
        "text-generation": {
            "do_sample": true,
            "max_length": 50
        }
    },
    "auto_map": {
        "AutoConfig": "model.MiniMoEConfig",
        "AutoModel": "model.MiniMoE50M"
    }
}