HeteroMoEProjSpace-v2 / config.json
redmingwei's picture
update config.json
bbc6577 verified
Raw History Blame Contribute Delete
592 Bytes
{
"architectures": [
"HeteroMoEProjSpaceModel"
],
"auto_map": {
"AutoConfig": "modeling_hetero_moe_projspace.HeteroMoEProjSpaceConfig",
"AutoModel": "modeling_hetero_moe_projspace.HeteroMoEProjSpaceModel"
},
"bert_id": "google/bert_uncased_L-4_H-256_A-4",
"d_shared": 256,
"d_space": 1024,
"dtype": "float32",
"modal_dims": [
896,
256,
768
],
"model_type": "hetero_moe_projspace",
"moe_top_k": 2,
"n_experts": 4,
"qwen_id": "Qwen/Qwen2.5-0.5B-Instruct",
"transformers_version": "5.21.0",
"vit_id": "google/vit-base-patch16-224-in21k"
}