{ "architectures": [ "Inkling2LoopForConditionalGeneration" ], "dtype": "bfloat16", "eos_token_id": 200006, "model_type": "inkling2_loop", "mtp_config": { "hidden_size": 2304, "num_hidden_layers": 3, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "intermediate_size": 9216, "max_position_embeddings": 1048576, "torch_dtype": "bfloat16", "q_bias": false, "o_bias": false, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "local_layer_ids": [ 0, 2 ], "sliding_window_size": 1024, "block_size": 8, "markov_rank": 256, "markov_head_type": "rnn_hidden", "target_layer_ids": [ 5, 23, 41, 48 ] }, "quantization": { "group_size": 64, "bits": 8, "mode": "affine", "language_model.lm_head": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.embed_tokens": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.proj_in": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.proj_out": { "group_size": 64, "bits": 8, "mode": "affine" } }, "quantization_config": { "group_size": 64, "bits": 8, "mode": "affine", "language_model.lm_head": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.embed_tokens": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.0.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.1.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.10.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.11.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.12.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.13.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.14.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.15.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.16.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.17.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.18.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.19.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.2.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.20.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.21.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.22.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.23.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.23.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.3.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.4.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.5.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.6.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.7.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.8.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.down_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.gate_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.mlp.switch_mlp.up_proj": { "group_size": 64, "bits": 2, "mode": "affine" }, "language_model.model.layers.9.self_attn.gate_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.o_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.r_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "language_model.model.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.0.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.1.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.10.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.11.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.12.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.13.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.14.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.15.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.16.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.17.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.18.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.19.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.2.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.20.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.21.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.22.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.3.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.4.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.5.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.6.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.7.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.8.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.mlp.fc1": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.mlp.fc2": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.k_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.out_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.q_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.layers.9.self_attn.v_proj": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.proj_in": { "group_size": 64, "bits": 8, "mode": "affine" }, "vision_tower.proj_out": { "group_size": 64, "bits": 8, "mode": "affine" } }, "text_config": { "model_type": "inkling2_loop_text", "model_max_length": 1048576, "torch_dtype": "bfloat16", "hidden_size": 2304, "num_hidden_layers": 24, "vocab_size": 201024, "num_attention_heads": 24, "num_key_value_heads": 8, "head_dim": 128, "d_rel": 16, "rel_extent": 1024, "q_bias": false, "o_bias": false, "log_scaling_n_floor": 8192, "log_scaling_alpha": 0.1, "rms_norm_eps": 1e-06, "use_value_norm_no_gamma": true, "qk_l2_clip_threshold": 22.627416997969522, "rel_l2_clip_threshold": 8.0, "local_layer_ids": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22 ], "sconv_kernel_size": 4, "unpadded_vocab_size": 200058, "final_logit_softcapping": 30.0, "swa_head_dim": 128, "swa_num_attention_heads": 24, "swa_num_key_value_heads": 8, "sliding_window_size": 1024, "n_layers_per_attn_res_block": 6, "attn_res_normalize_values": true, "n_routed_experts": 128, "num_experts_per_tok": 8, "n_shared_experts": 0, "intermediate_size": 1152, "mlp_gate_softcap": 4.0, "mlp_up_softcap": 25.0, "route_scale": 8.0, "use_gate_bias": true, "loop_config": { "core_start": 6, "core_end": 18, "num_passes": 3, "gate_rank": 64, "is_varloop": false } }, "vision_config": { "model_type": "inkling2_vision", "text_hidden_size": 2304, "temporal_patch_size": 2, "native_patch_size": 20, "adapter_merge_size": 2, "projection_hidden_dim": 4096, "projector_bias": false, "hidden_size": 1024, "num_hidden_layers": 23, "num_attention_heads": 16, "mlp_ratio": 4.0, "posemb_grid_size": 32, "layer_norm_eps": 1e-05 } }