File size: 1,482 Bytes
4d74557 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | {
"vocab_size": 206464,
"d_model": 1024,
"n_layers": 18,
"max_seq_len": 4096,
"mha_layer_indices": [
5,
11,
17
],
"dense_ffn_layer_indices": [
1,
3,
5,
7,
9,
11,
13,
15,
17
],
"hse_ffn_layer_indices": [
0,
2,
4,
6,
8,
10,
12,
14,
16
],
"arg_d_inner": 1024,
"arg_d_state": 48,
"arg_d_rank": 48,
"arg_conv_kernel": 4,
"arg_local_window": 64,
"arg_local_n_heads": 8,
"arg_local_n_kv_heads": 2,
"arg_head_dim": 64,
"arg_scalar_gate": true,
"mha_n_heads": 16,
"mha_n_kv_heads": 4,
"mha_head_dim": 64,
"rope_theta": 500000.0,
"rope_scaling_type": null,
"rope_scaling_factor": 1.0,
"dense_d_ff": 2560,
"hse_n_domains": 4,
"hse_n_experts_per_domain": 6,
"hse_top_k": 2,
"hse_d_ff_expert": 210,
"hse_capacity_factor": 1.25,
"hse_expert_dropout": 0.1,
"hse_aux_loss_alpha": 0.01,
"hse_gumbel_tau_init": 1.0,
"hse_gumbel_tau_final": 0.1,
"hse_gumbel_tau_warmup_tokens": 10000000000,
"norm_type": "dgn",
"dgn_n_groups": 16,
"norm_eps": 1e-06,
"init_std": 0.02,
"init_std_residual_scale": true,
"dropout": 0.0,
"tie_word_embeddings": true,
"use_cache": true,
"pad_token_id": 0,
"bos_token_id": 1,
"eos_token_id": 2,
"use_flash_attention": true,
"use_triton_kernels": true,
"use_torch_compile": true,
"compile_mode": "max-autotune",
"use_gradient_checkpointing": true,
"gradient_checkpointing_ratio": 1.0
} |