samai-4b SFT merged full weights (rent-math sft, 831 rows x2ep)

config.json

@@ -2,35 +2,26 @@
   "architectures": [
     "Samai4bForCausalLM"
   ],
-  "model_type": "samai_4b",
+  "attention_bias": false,
+  "attention_dropout": 0.0,
   "auto_map": {
     "AutoConfig": "modeling_samai_4b.Samai4bConfig",
     "AutoModelForCausalLM": "modeling_samai_4b.Samai4bForCausalLM"
   },
-  "vocab_size": 131072,
-  "hidden_size": 2560,
-  "intermediate_size": 10240,
-  "num_hidden_layers": 36,
-  "num_attention_heads": 16,
-  "num_key_value_heads": 4,
-  "head_dim": 256,
-  "hidden_act": "gelu",
-  "max_position_embeddings": 131072,
-  "initializer_range": 0.01976,
-  "rms_norm_eps": 1e-06,
-  "use_cache": true,
-  "pad_token_id": 2,
   "bos_token_id": 0,
+  "commit_kv": true,
+  "drift_threshold": 0.98,
+  "dtype": "float16",
   "eos_token_id": [
     1
   ],
-  "tie_word_embeddings": true,
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "mlp_bias": false,
-  "headwise_attn_output_gate": true,
   "gate_attn_act_mode": "sigmoid",
-  "sliding_window": 512,
+  "head_dim": 256,
+  "headwise_attn_output_gate": true,
+  "hidden_act": "gelu",
+  "hidden_size": 2560,
+  "initializer_range": 0.01976,
+  "intermediate_size": 10240,
   "layer_types": [
     "sliding_attention",
     "sliding_attention",
@@ -69,36 +60,47 @@
     "sliding_attention",
     "full_attention"
   ],
+  "max_ponder_steps": 8,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "samai_4b",
+  "moe_aux_weight": 0.01,
+  "moe_num_experts": 8,
+  "moe_route_tau": null,
+  "moe_scope": "ponder",
+  "moe_tau": 0.125,
+  "mtp_loss_weight": 1.0,
+  "mtp_num_tokens": 2,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 4,
+  "pad_token_id": 2,
+  "ponder_all_positions": false,
+  "ponder_epsilon": 0.01,
+  "ponder_forced_steps": 0,
+  "ponder_head_hidden": 0,
+  "ponder_loss_beta": 0.01,
+  "ponder_prior_p": 0.5,
+  "ponder_signal": "learned",
+  "ponder_start_layer": 28,
+  "ponder_step_ce_weight": 0.0,
+  "ponder_window": 1,
+  "rms_norm_eps": 1e-06,
   "rope_parameters": {
     "full_attention": {
       "partial_rotary_factor": 0.25,
       "rope_theta": 5000000.0
     },
+    "rope_theta": 10000.0,
+    "rope_type": "default",
     "sliding_attention": {
       "partial_rotary_factor": 1.0,
       "rope_theta": 10000.0
     }
   },
-  "dtype": "float16",
-  "ponder_start_layer": 28,
-  "ponder_window": 1,
-  "max_ponder_steps": 8,
-  "ponder_epsilon": 0.01,
-  "ponder_signal": "learned",
-  "ponder_all_positions": false,
-  "commit_kv": true,
-  "ponder_prior_p": 0.5,
-  "ponder_loss_beta": 0.01,
-  "drift_threshold": 0.98,
-  "ponder_head_hidden": 0,
-  "ponder_step_ce_weight": 0.0,
-  "ponder_forced_steps": 0,
-  "moe_num_experts": 8,
-  "moe_route_tau": null,
-  "moe_aux_weight": 0.01,
-  "moe_scope": "ponder",
-  "moe_tau": 0.125,
-  "mtp_num_tokens": 2,
-  "mtp_loss_weight": 1.0,
-  "transformers_version": "5.16.1"
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.16.1",
+  "use_cache": true,
+  "vocab_size": 131072
 }

generation_config.json

@@ -1,11 +1,12 @@
 {
   "bos_token_id": 0,
+  "do_sample": true,
   "eos_token_id": [
     1
   ],
+  "max_new_tokens": 512,
   "pad_token_id": 2,
-  "do_sample": true,
   "temperature": 0.8,
   "top_p": 0.95,
-  "max_new_tokens": 512
-}
+  "transformers_version": "5.16.1"
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ae082004604db866e5b975046a98d2e038ee93e126b24f22afa877dccc596c2
-size 3503449008
+oid sha256:57a65a39e0220c1fc60dec34d53579a42e3af7d7bd3e9159ff12dacd6e934ad7
+size 3996563896

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6a9998f365759ed036372a17603993ea0430fef5660eee0fb99c8893bcd6f03
-size 3501303392
+oid sha256:c6640d211698877759990d393be3b60d1fd720d7385b2047d994654f55dacbda
+size 3986664064

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3599c865620a727f6ceecbf82d09285ba7eacadf5a0c4af91f6824aa4d998dba
-size 1298470074
+oid sha256:c321167ec4db2314fc90d8c3eba15d077084326901722d9fd4824bc5ef55731f
+size 319994842

model.safetensors.index.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "total_size": 8303821186
+    "total_parameters": 4151582785,
+    "total_size": 8303165570
   },
   "weight_map": {
     "model.embedding.weight": "model-00001-of-00003.safetensors",
@@ -60,58 +61,14 @@
     "model.layers.14.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.14.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.14.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
     "model.layers.15.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.15.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.15.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.2.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.3.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.4.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.5.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.6.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -144,6 +101,14 @@
     "model.layers.19.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.19.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.19.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -270,180 +235,168 @@
     "model.layers.29.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.29.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.29.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.0.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.1.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.1.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.2.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.2.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.2.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.3.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.3.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.3.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.4.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.4.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.4.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.30.mlp.experts.5.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.8.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.9.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.30.mlp.experts.0.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.0.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.1.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.1.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.2.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.2.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.3.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.3.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.4.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.4.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.5.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.5.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.6.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.6.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.6.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.7.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.7.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.experts.7.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.router.bias": "model-00003-of-00003.safetensors",
-    "model.layers.30.mlp.router.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.30.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.0.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.0.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.0.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.1.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.1.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.1.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.2.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.2.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.2.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.3.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.3.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.3.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.4.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.4.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.4.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.5.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.5.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.5.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.6.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.6.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.6.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.7.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.7.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.experts.7.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.router.bias": "model-00003-of-00003.safetensors",
-    "model.layers.31.mlp.router.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.31.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.0.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.0.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.0.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.1.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.1.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.1.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.2.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.2.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.2.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.3.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.3.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.3.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.4.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.4.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.4.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.5.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.5.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.5.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.6.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.6.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.6.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.7.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.7.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.experts.7.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.router.bias": "model-00003-of-00003.safetensors",
-    "model.layers.32.mlp.router.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.32.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.0.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.0.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.0.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.1.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.1.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.1.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.2.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.2.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.2.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.3.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.3.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.3.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.4.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.4.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.4.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.5.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.5.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.5.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.6.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.6.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.6.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.7.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.7.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.experts.7.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.router.bias": "model-00003-of-00003.safetensors",
-    "model.layers.33.mlp.router.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.33.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.0.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.0.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.0.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.1.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.1.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.1.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.2.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.2.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.2.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.3.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.3.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.3.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.4.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.4.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.4.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.5.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.5.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.5.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.6.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.6.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.6.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.7.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.7.gate_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.experts.7.up_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.router.bias": "model-00003-of-00003.safetensors",
-    "model.layers.34.mlp.router.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.34.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.experts.5.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.5.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.6.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.6.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.6.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.7.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.7.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.experts.7.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.router.bias": "model-00002-of-00003.safetensors",
+    "model.layers.30.mlp.router.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.30.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.0.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.1.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.1.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.2.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.2.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.2.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.3.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.3.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.3.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.4.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.4.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.4.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.5.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.5.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.5.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.6.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.6.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.6.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.7.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.7.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.experts.7.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.router.bias": "model-00002-of-00003.safetensors",
+    "model.layers.31.mlp.router.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.31.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.0.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.1.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.1.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.2.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.2.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.2.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.3.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.3.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.3.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.4.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.4.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.4.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.5.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.5.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.5.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.6.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.6.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.6.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.7.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.7.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.experts.7.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.router.bias": "model-00002-of-00003.safetensors",
+    "model.layers.32.mlp.router.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.32.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.0.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.1.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.1.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.2.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.2.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.2.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.3.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.3.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.3.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.4.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.4.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.4.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.5.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.5.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.5.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.6.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.6.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.6.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.7.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.7.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.experts.7.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.router.bias": "model-00002-of-00003.safetensors",
+    "model.layers.33.mlp.router.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.33.self_attn.q_k_v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.0.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.0.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.0.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.1.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.1.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.1.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.2.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.2.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.2.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.3.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.3.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.3.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.4.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.4.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.4.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.5.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.5.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.5.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.6.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.6.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.6.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.7.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.7.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.experts.7.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.router.bias": "model-00002-of-00003.safetensors",
+    "model.layers.34.mlp.router.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.34.self_attn.out_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.34.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.35.mlp.experts.0.down_proj.weight": "model-00003-of-00003.safetensors",
@@ -476,6 +429,54 @@
     "model.layers.35.self_attn.g_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.35.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.35.self_attn.q_k_v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_k_v_proj.weight": "model-00001-of-00003.safetensors",
     "model.norm.weight": "model-00003-of-00003.safetensors",
     "mtp_heads.0.norm_e.weight": "model-00003-of-00003.safetensors",
     "mtp_heads.0.norm_h.weight": "model-00003-of-00003.safetensors",
@@ -490,4 +491,4 @@
     "ponder_head.bias": "model-00003-of-00003.safetensors",
     "ponder_head.weight": "model-00003-of-00003.safetensors"
   }
-}
+}

tokenizer.json

@@ -55,7 +55,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 6,
@@ -1100,19 +1100,6 @@
       "<｜fim▁begin｜>": 11,
       "<｜fim▁hole｜>": 12,
       "<｜fim▁end｜>": 13,
-      "<|System|>": 130972,
-      "<|User|>": 130973,
-      "<|Tool|>": 130974,
-      "<|Developer|>": 130975,
-      "<|Bot|>": 130976,
-      "<tool_call>": 130977,
-      "<tool_response>": 130978,
-      "</tool_response>": 130979,
-      "<arg_key>": 130980,
-      "</arg_key>": 130981,
-      "<arg_value>": 130982,
-      "</arg_value>": 130983,
-      "</tool_call>": 130984,
       "!": 14,
       "\"": 15,
       "#": 16,
@@ -132071,6 +132058,19 @@
       "olone": 130969,
       "ĠLef": 130970,
       "Ġshines": 130971,
+      "<|System|>": 130972,
+      "<|User|>": 130973,
+      "<|Tool|>": 130974,
+      "<|Developer|>": 130975,
+      "<|Bot|>": 130976,
+      "<tool_call>": 130977,
+      "<tool_response>": 130978,
+      "</tool_response>": 130979,
+      "<arg_key>": 130980,
+      "</arg_key>": 130981,
+      "<arg_value>": 130982,
+      "</arg_value>": 130983,
+      "</tool_call>": 130984,
       "Ć": 130985,
       "<||unused2||>": 130986,
       "<||unused3||>": 130987,

tokenizer_config.json

@@ -1,42 +1,14 @@
 {
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "<｜start▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "backend": "tokenizers",
+  "bos_token": "<｜start▁of▁sentence｜>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "<｜end▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "is_local": true,
   "legacy": true,
+  "local_files_only": false,
   "model_max_length": 131072,
-  "pad_token": {
-    "__type": "AddedToken",
-    "content": "<｜▁pad▁｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "pad_token": "<｜▁pad▁｜>",
   "sp_model_kwargs": {},
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "chat_template": "{%- if not messages %}{{- raise_exception('No messages provided.') }}{%- endif %}{%- set enable_thinking = enable_thinking | default(true) %}{#- Render a string or a list of text blocks. -#}{%- macro render_content(content, context_name) %}{%- if content is string %}{{- content }}{%- elif content is none or content is undefined %}{{- '' }}{%- elif content is iterable and content is not mapping %}{%- for block in content %}{%- if block.type == 'text' %}{{- block.text }}{%- else %}{{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }}{%- endif %}{%- endfor %}{%- else %}{{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }}{%- endif %}{%- endmacro %}{#- Default system prompt -#}{%- set default_system = 'you are a helpful assistant.' %}{#- The first message-level system is placed in the initial system block. -#}{%- set ns = namespace(initial_system='') %}{%- if messages[0].role == 'system' %}{%- set ns.initial_system = render_content(messages[0].content, 'system') %}{%- endif %}{#- System block -#}{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }}{%- if tools %}{{- '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }}{%- for tool in tools %}{{- '\n' + tool.function | tojson}}{%- endfor %}{{- '\n' + '</tools>' }}{%- endif %}{%- if ns.initial_system %}{{- '\n\n' + ns.initial_system }}{%- endif %}{{- '<｜end▁of▁sentence｜>'}}{#- Conversation turns -#}{%- for message in messages %}{%- if message.role == 'system' %}{#- The first system message was consumed by the initial block. -#}{%- if not loop.first %}{{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }}{%- endif %}{%- elif message.role == 'user' %}{{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }}{%- elif message.role == 'assistant' %}{%- set assistant_content = render_content(message.content, 'assistant') %}{%- if message.reasoning_content is defined and message.reasoning_content %}{%- set reasoning_content = message.reasoning_content %}{%- else %}{%- set reasoning_content = '' %}{%- endif %}{{- '<｜start▁of▁sentence｜><|Bot|>'}}{%- if reasoning_content %}{{- '<think>' + reasoning_content + '</think>'}}{%- else %}{{- '</think>' }}{%- endif %}{%- if assistant_content %}{{- assistant_content }}{%- endif %}{%- if message.tool_calls is defined and message.tool_calls is not none %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function.arguments is not mapping %}{{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }}{%- endif %}{%- set args = tool_call.function.arguments %}{{- '<tool_call>' + tool_call.function.name }}{%- for k, v in args.items() %}{{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson) ~ '</arg_value>' }}{%- endfor %}{{- '</tool_call>' }}{%- endfor %}{%- endif %}{{- '<｜end▁of▁sentence｜>' }}{%- elif message.role == 'tool' %}{%- if loop.previtem is undefined or loop.previtem.role != 'tool' %}{{- '<｜start▁of▁sentence｜><|Tool|>' }}{%- endif %}{{- '<tool_response>' ~ message.content ~ '</tool_response>' }}{%- if loop.nextitem is undefined or loop.nextitem.role != 'tool' %}{{- '<｜end▁of▁sentence｜>' }}{%- endif %}{%- else %}{{- raise_exception('Unsupported message role: ' ~ message.role) }}{%- endif %}{%- endfor %}{#- Generation prompt -#}{%- if add_generation_prompt %}{{- '<｜start▁of▁sentence｜><|Bot|>' }}{%- if enable_thinking is defined and enable_thinking %}{{- '<think>' }}{%- endif %}{%- if enable_thinking is defined and not enable_thinking %}{{- '</think>' }}{%- endif %}{%- endif %}"
-}
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}