MoEPPlus / config.json
Valdemar123's picture
Upload folder using huggingface_hub
29aaea0 verified
Raw History Blame Contribute Delete
959 Bytes
{
"architectures": [
"MoEPForCausalLM"
],
"attn_pdrop": 0.1,
"auto_map": {
"AutoConfig": "moep_different_experts.MoEPLMConfig",
"AutoModel": "moep_different_experts.MoEPModel",
"AutoModelForCausalLM": "moep_different_experts.MoEPForCausalLM"
},
"bos_token_id": 1,
"dim": 384,
"dtype": "float32",
"dual_feedforward": false,
"embd_pdrop": 0.1,
"eos_token_id": 2,
"ffn_multp": 2.6666666666666665,
"identity_experts": false,
"layer_norm_epsilon": 1e-08,
"mask_token_id": 4,
"max_position_embeddings": 512,
"mlp_experts": null,
"model_type": "moep_lm",
"moe_k": 2,
"n_experts": 4,
"n_heads": 6,
"n_layers": 12,
"n_null_experts": 0,
"n_null_parallel": 0,
"n_parallel_blocks": 4,
"null_experts": false,
"pad_token_id": 3,
"parallel_dim": 192,
"parallel_k": 2,
"parallel_n_heads": 3,
"resid_pdrop": 0.1,
"sliding_window": null,
"transformers_version": "4.57.1",
"vocab_size": 16384
}