{ "architectures": [ "MoEPForCausalLM" ], "attn_pdrop": 0.1, "auto_map": { "AutoConfig": "moep_different_experts.MoEPLMConfig", "AutoModel": "moep_different_experts.MoEPModel", "AutoModelForCausalLM": "moep_different_experts.MoEPForCausalLM" }, "bos_token_id": 1, "dim": 384, "dtype": "float32", "dual_feedforward": false, "embd_pdrop": 0.1, "eos_token_id": 2, "ffn_multp": 2.6666666666666665, "identity_experts": false, "layer_norm_epsilon": 1e-08, "mask_token_id": 4, "max_position_embeddings": 512, "mlp_experts": null, "model_type": "moep_lm", "moe_k": 2, "n_experts": 4, "n_heads": 6, "n_layers": 12, "n_null_experts": 0, "n_null_parallel": 0, "n_parallel_blocks": 4, "null_experts": false, "pad_token_id": 3, "parallel_dim": 192, "parallel_k": 2, "parallel_n_heads": 3, "resid_pdrop": 0.1, "sliding_window": null, "transformers_version": "4.57.1", "vocab_size": 16384 }