{
    "activation_situ_beta": 4.0,
    "activation_situ_linear_beta": 25.0,
    "add_full_attention_sink_bias": true,
    "architectures": [
        "Ernie5_5_MoeForCausalLM"
    ],
    "attention_bias": false,
    "bos_token_id": 1,
    "compress_ratios": [
        128,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        128,
        128,
        128,
        128,
        128,
        128,
        -2,
        -2
    ],
    "compress_rope_theta": 160000.0,
    "csa_dense_mode": true,
    "enable_hyper_connections": true,
    "eos_token_id": 2,
    "experimental_attention_variant": "dsv4_hybrid",
    "first_k_dense_replace": 1,
    "gated_attn_act": "sigmoid",
    "gated_attn_use_q_lora": true,
    "gated_linear_unit": true,
    "hc_eps": 1e-06,
    "hc_mult": 4,
    "hc_sinkhorn_iters": 20,
    "hca_rope_type": "yarn",
    "head_dim": 512,
    "hidden_act": "situ",
    "hidden_size": 4096,
    "hybrid_mla_attention": "mqa_dsa",
    "hybrid_mla_kv_lora_rank": 512,
    "hybrid_mla_num_attention_heads": 64,
    "hybrid_mla_num_key_value_heads": 64,
    "hybrid_mla_q_lora_rank": 1024,
    "hybrid_mla_qk_nope_head_dim": 192,
    "hybrid_mla_qk_rope_head_dim": 64,
    "hybrid_mla_v_head_dim": 256,
    "index_head_dim": 128,
    "index_n_heads": 64,
    "index_topk": 2048,
    "intermediate_size": 14336,
    "kv_lora_rank": 512,
    "latent_moe_use_norm": true,
    "max_position_embeddings": 1048576,
    "model_type": "ernie5_5_moe",
    "moe_group_experts": false,
    "moe_intermediate_size": 2048,
    "moe_latent_size": 2048,
    "moe_layer_freq": 1,
    "moe_shared_expert_gate": false,
    "moe_split_feature_routing": true,
    "moe_token_dispatcher_type": "deepep",
    "mqa_split_kv_b_proj": true,
    "mtp_shared_last_layer": true,
    "multimax": "lm_head",
    "n_group": 1,
    "n_routed_experts": 512,
    "n_shared_experts": 1,
    "norm_topk_prob": true,
    "num_attention_heads": 64,
    "num_experts_per_tok": 10,
    "num_hidden_layers": 43,
    "num_key_value_heads": 1,
    "num_nextn_predict_layers": 1,
    "o_groups": 8,
    "o_lora_rank": 1024,
    "pad_token_id": 0,
    "q_lora_rank": 1024,
    "qk_nope_head_dim": 448,
    "qk_norm_type": "per_head",
    "qk_pos_emb_head_dim": 64,
    "qk_rope_head_dim": 64,
    "rms_norm_eps": 1e-06,
    "rope_interleave": true,
    "rope_scaling": {
        "beta_fast": 32,
        "beta_slow": 1,
        "factor": 16,
        "original_max_position_embeddings": 65536,
        "type": "yarn"
    },
    "rope_theta": 8000000,
    "rope_type": "rope",
    "rotary_base": 8000000,
    "rotary_percent": 1.0,
    "routed_scaling_factor": 3.0,
    "routed_scaling_factor_learnable": true,
    "scoring_func": "sigmoid",
    "sliding_window": 128,
    "tie_word_embeddings": false,
    "topk_group": 1,
    "topk_method": "noaux_tc",
    "torch_dtype": "bfloat16",
    "transformers_version": "4.46.1",
    "use_attn_merger": true,
    "use_bias": false,
    "use_cache": true,
    "use_dense_mtp": false,
    "use_fast_hadamard": true,
    "use_gated_attn": true,
    "use_mla": false,
    "use_qk_norm": true,
    "use_vha_attention": true,
    "v_head_dim": 512,
    "vocab_size": 201216,
    "quantization_config": {
        "activation_scheme": "dynamic",
        "fmt": "e4m3",
        "quant_method": "fp8",
        "scale_fmt": "ue8m0",
        "weight_block_size": [
            128,
            128
        ]
    }
}