| { | |
| "vocab_size": 8192, | |
| "max_seq_len": 1024, | |
| "dim": 192, | |
| "n_layers": 8, | |
| "n_dense_layers": 1, | |
| "dense_hidden_dim": 512, | |
| "norm_eps": 1e-06, | |
| "init_std": 0.02, | |
| "n_heads": 6, | |
| "q_lora_rank": 0, | |
| "kv_lora_rank": 48, | |
| "qk_nope_head_dim": 24, | |
| "qk_rope_head_dim": 16, | |
| "v_head_dim": 24, | |
| "rope_theta": 10000.0, | |
| "n_routed_experts": 12, | |
| "n_shared_experts": 1, | |
| "n_activated_experts": 3, | |
| "moe_hidden_dim": 128, | |
| "n_expert_groups": 3, | |
| "n_limited_groups": 2, | |
| "route_scale": 1.0, | |
| "bias_update_speed": 0.001, | |
| "seq_aux_loss_alpha": 0.0001, | |
| "n_mtp_modules": 1, | |
| "mtp_loss_weight": 0.3, | |
| "use_fp8": false, | |
| "fp8_block_size": 128 | |
| } |