Maxwell-V1-Flash / config.json
Maxwell
Initial commit
5a278bd
Raw History Blame Contribute Delete
2.12 kB
{
"architectures": [
"MaxwellMoeForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_maxwell_moe.MaxwellMoeConfig",
"AutoModelForCausalLM": "modeling_maxwell_moe.MaxwellMoeForCausalLM"
},
"attention_variant": "block_sparse_gqa",
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 3,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.008,
"intermediate_size": 6144,
"max_position_embeddings": 524288,
"model_type": "maxwell_moe",
"moe_intermediate_size": 512,
"shared_expert_intermediate_size": 1024,
"num_attention_heads": 32,
"num_experts": 256,
"num_experts_per_tok": 12,
"num_hidden_layers": 45,
"num_key_value_heads": 4,
"attention_head_size": 128,
"output_router_logits": false,
"rms_norm_eps": 1e-06,
"rope_theta": 20000000,
"router_aux_loss_coef": 0.0,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.0",
"use_cache": true,
"block_sparse_gqa_indexer_n_heads": 4,
"block_sparse_gqa_indexer_head_dim": 128,
"block_sparse_gqa_block_size": 128,
"block_sparse_gqa_topk_blocks": 16,
"block_sparse_gqa_total_support_blocks": 32,
"block_sparse_gqa_group_reduction": "per_group_max",
"block_sparse_gqa_prefix_window_size": 1024,
"block_sparse_gqa_local_window_size": 1024,
"block_sparse_gqa_num_remote_blocks": 16,
"block_sparse_gqa_layer_freq": [
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1
],
"block_sparse_gqa_query_chunk_size": 64,
"vocab_size": 200704,
"mlp_only_layers": [
0,
1,
2
],
"qk_layernorm": true,
"enable_hyper_connections": true,
"num_residual_streams": 4,
"mhc_sinkhorn_iterations": 20,
"mhc_init_gating_factor": 0.01,
"moe_router_enable_expert_bias": true,
"router_score_function": "sigmoid",
"mtp_num_layers": 1
}