{
  "architectures": [
    "KamboForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_kambo.KamboConfig",
    "AutoModel": "modeling_kambo.KamboModel",
    "AutoModelForCausalLM": "modeling_kambo.KamboForCausalLM"
  },
  "bos_token_id": 151643,
  "conv_kernel": 3,
  "d_ff": 1152,
  "dtype": "bfloat16",
  "eos_token_id": 151645,
  "gqa_layers": [
    3,
    7,
    11,
    15,
    19,
    23
  ],
  "head_dim": 64,
  "hidden_size": 1024,
  "intermediate_size": 1152,
  "max_position_embeddings": 16384,
  "model_type": "kambo",
  "n_experts": 16,
  "num_attention_heads": 16,
  "num_experts": 16,
  "num_experts_per_tok": 2,
  "num_hidden_layers": 24,
  "num_key_value_heads": 4,
  "pad_token_id": 151643,
  "quantization_config": {
    "checkpoint_format": "dynquant-packed",
    "group_size": 128,
    "lm_head_quantized": false,
    "modules": {
      "model.embed_tokens": {
        "bits": 8,
        "out_features": 151936
      },
      "model.layers.0.conv.in_proj": {
        "bits": 8,
        "out_features": 3072
      },
      "model.layers.0.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.0.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.0.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.0.moe.sw3": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.0.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.0.moe.w2": {
        "bits": 2,
        "out_features": 18432
      },
      "model.layers.0.moe.w3": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.1.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.1.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.1.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.1.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.1.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.1.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.1.moe.w2": {
        "bits": 2,
        "out_features": 18432
      },
      "model.layers.1.moe.w3": {
        "bits": 2,
        "out_features": 16384
      },
      "model.layers.10.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.10.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.10.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.10.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.10.moe.sw3": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.10.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.10.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.10.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.11.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.11.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.11.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.11.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.11.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.11.moe.w3": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.11.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.11.self_attn.o_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.11.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.11.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.12.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.12.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.12.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.12.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.12.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.12.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.12.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.12.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.13.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.13.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.13.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.13.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.13.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.13.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.13.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.13.moe.w3": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.14.conv.in_proj": {
        "bits": 8,
        "out_features": 3072
      },
      "model.layers.14.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.14.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.14.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.14.moe.sw3": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.14.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.14.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.14.moe.w3": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.15.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.15.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.15.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.15.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.15.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.15.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.15.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.15.self_attn.o_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.15.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.15.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.16.conv.in_proj": {
        "bits": 8,
        "out_features": 3072
      },
      "model.layers.16.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.16.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.16.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.16.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.16.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.16.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.16.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.17.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.17.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.17.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.17.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.17.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.17.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.17.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.17.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.18.conv.in_proj": {
        "bits": 8,
        "out_features": 3072
      },
      "model.layers.18.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.18.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.18.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.18.moe.sw3": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.18.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.18.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.18.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.19.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.19.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.19.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.19.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.19.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.19.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.19.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.19.self_attn.o_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.19.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.19.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.2.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.2.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.2.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.2.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.2.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.2.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.2.moe.w2": {
        "bits": 2,
        "out_features": 18432
      },
      "model.layers.2.moe.w3": {
        "bits": 2,
        "out_features": 16384
      },
      "model.layers.20.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.20.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.20.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.20.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.20.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.20.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.20.moe.w2": {
        "bits": 2,
        "out_features": 18432
      },
      "model.layers.20.moe.w3": {
        "bits": 2,
        "out_features": 16384
      },
      "model.layers.21.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.21.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.21.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.21.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.21.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.21.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.21.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.21.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.22.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.22.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.22.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.22.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.22.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.22.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.22.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.22.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.23.moe.sw1": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.23.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.23.moe.sw3": {
        "bits": 8,
        "out_features": 1152
      },
      "model.layers.23.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.23.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.23.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.23.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.23.self_attn.o_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.23.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.23.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.3.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.3.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.3.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.3.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.3.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.3.moe.w3": {
        "bits": 2,
        "out_features": 16384
      },
      "model.layers.3.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.3.self_attn.o_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.3.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.3.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.4.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.4.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.4.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.4.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.4.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.4.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.4.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.4.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.5.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.5.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.5.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.5.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.5.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.5.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.5.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.5.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.6.conv.in_proj": {
        "bits": 8,
        "out_features": 3072
      },
      "model.layers.6.conv.out_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.6.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.6.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.6.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.6.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.6.moe.w2": {
        "bits": 2,
        "out_features": 18432
      },
      "model.layers.6.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.7.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.7.moe.sw2": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.7.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.7.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.7.moe.w2": {
        "bits": 3,
        "out_features": 18432
      },
      "model.layers.7.moe.w3": {
        "bits": 3,
        "out_features": 16384
      },
      "model.layers.7.self_attn.k_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.7.self_attn.o_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.7.self_attn.q_proj": {
        "bits": 8,
        "out_features": 1024
      },
      "model.layers.7.self_attn.v_proj": {
        "bits": 8,
        "out_features": 256
      },
      "model.layers.8.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.8.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.8.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.8.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.8.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.8.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.8.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.8.moe.w3": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.9.conv.in_proj": {
        "bits": 4,
        "out_features": 3072
      },
      "model.layers.9.conv.out_proj": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.9.moe.sw1": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.9.moe.sw2": {
        "bits": 4,
        "out_features": 1024
      },
      "model.layers.9.moe.sw3": {
        "bits": 4,
        "out_features": 1152
      },
      "model.layers.9.moe.w1": {
        "bits": 4,
        "out_features": 16384
      },
      "model.layers.9.moe.w2": {
        "bits": 4,
        "out_features": 18432
      },
      "model.layers.9.moe.w3": {
        "bits": 4,
        "out_features": 16384
      }
    },
    "modules_to_not_convert": [],
    "quant_method": "dynquant",
    "schema_version": 1,
    "symmetric": false,
    "version": "0.5.3"
  },
  "rms_norm_eps": 1e-06,
  "rope_theta": 40000.0,
  "tie_word_embeddings": true,
  "top_k": 2,
  "transformers_version": "5.14.1",
  "use_cache": true,
  "vocab_size": 151936
}