Checkpoint with the mu-gated readout: text faithful, chip emits when driven

chipformer.json

@@ -2,16 +2,16 @@
  "layout": "rv1",
  "chip_token_offset": 248320,
  "chip_vocab": 263124,
- "chip_layers": 36,
- "qwen_layers_from": 36,
+ "chip_layers": 37,
+ "qwen_layers_from": 37,
  "hidden": 4096,
  "chip_dims_offset": 2560,
  "ballast_dims": 64,
  "circuit": {
-  "layers": 36,
+  "layers": 37,
   "dims": 3259,
   "heads": 457,
-  "ffn_units": 46685,
+  "ffn_units": 46727,
   "max_units": 8648,
   "max_heads": 16
  },

config.json

@@ -49,6 +49,7 @@
     "linear_attention",
     "linear_attention",
     "linear_attention",
+    "linear_attention",
     "full_attention",
     "linear_attention",
     "linear_attention",
@@ -90,7 +91,7 @@
   "mtp_num_hidden_layers": 0,
   "mtp_use_dedicated_embeddings": false,
   "num_attention_heads": 16,
-  "num_hidden_layers": 68,
+  "num_hidden_layers": 69,
   "num_key_value_heads": 4,
   "rms_norm_eps": 1e-06,
   "tie_word_embeddings": true,

model-00006-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be2aaa196a820a45b01305692894eb171101e5567099d68ddd1531d282f4b37f
+oid sha256:299ea832791ccb5b948594040dcf70673847bddbece872b848edd72328e1f007
 size 2167910096

model-00007-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b1904cba863a3c1c4b74861b8e91065983c00120c04355b89090bbd835ea95e
-size 2150544520
+oid sha256:bc5bec740f4b68efd3f198ed3f7a3a3391e2506d9dc8b5e780be52ab5ccacc3c
+size 2150552528

model-00008-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7416decca464a6a7ef644d9a386a4199f1ee400281c50603fb70391d73e02510
-size 2150544824
+oid sha256:df581728e54a2a04ebca8cf67bea6fc0b7a2ed7e468f5bfdd976f739445cb5df
+size 2150536528

model-00009-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8dcbb8cf36a3bd5e5f854775a2d088e573794f530d1cf70f8b27d6edf06a7e6
-size 2200893680
+oid sha256:f500b948f11a6b878ecef6b7d417d871cd76608449447aed395e4d41cd19b98d
+size 2150544520

model-00010-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cf39e976f5998625a11ef7704a144189224e8813ad9c91e167eaa5c471f2745
-size 2150544880
+oid sha256:82edbc6f0171e372bbbad46dbf26b6caf9c9df908aa8cd5a593c115c95202e56
+size 2200893984

model-00011-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aaf17ed81e0fa3f7078917e87311ddd826d9023f4057639fdd22ecece276194
-size 2158342584
+oid sha256:6f2035dd1fa3c5c83d9877912dbe922703bad4a5390eeb48c1edffb61e6f3b4f
+size 2150544568

model-00012-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a96830273a2057804bab4a24f1b813dd4329a96986bcb84a5bcba5d4eaa02bf2
-size 2158933976
+oid sha256:662cd9ca16633e772ffb9cad268f3556f0356873affb006ee746289de0ddf1e7
+size 2166732312

model-00013-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd478de756d4fa3dbaf3c297ed7ca77adc13eec95d959b24e4ca3308c77f4bfd
-size 621382600
+oid sha256:56c4d7b608f6bec627e3458130cd674f538bd7d1a8d66d9b958ccce9c59e96c7
+size 982700928

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 28620724736
+    "total_size": 28982041472
   },
   "weight_map": {
     "model.embed_tokens.weight": "model-00001-of-00013.safetensors",
@@ -467,16 +467,19 @@
     "model.layers.38.mlp.up_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.38.post_attention_layernorm.weight": "model-00007-of-00013.safetensors",
     "model.layers.39.input_layernorm.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.A_log": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.conv1d.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.dt_bias": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.in_proj_a.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.in_proj_b.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.in_proj_qkv.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.in_proj_z.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.norm.weight": "model-00007-of-00013.safetensors",
+    "model.layers.39.linear_attn.out_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.39.mlp.down_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.39.mlp.gate_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.39.mlp.up_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.39.post_attention_layernorm.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.k_norm.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.k_proj.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.o_proj.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.q_norm.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.q_proj.weight": "model-00007-of-00013.safetensors",
-    "model.layers.39.self_attn.v_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00007-of-00013.safetensors",
     "model.layers.4.linear_attn.A_log": "model-00007-of-00013.safetensors",
     "model.layers.4.linear_attn.conv1d.weight": "model-00007-of-00013.safetensors",
@@ -492,19 +495,16 @@
     "model.layers.4.mlp.up_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00007-of-00013.safetensors",
     "model.layers.40.input_layernorm.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.A_log": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.conv1d.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.dt_bias": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.in_proj_a.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.in_proj_b.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.in_proj_qkv.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.in_proj_z.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.norm.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.linear_attn.out_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.40.mlp.down_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.40.mlp.gate_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.40.mlp.up_proj.weight": "model-00007-of-00013.safetensors",
-    "model.layers.40.post_attention_layernorm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.k_norm.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.q_norm.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00007-of-00013.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00007-of-00013.safetensors",
     "model.layers.41.input_layernorm.weight": "model-00008-of-00013.safetensors",
     "model.layers.41.linear_attn.A_log": "model-00008-of-00013.safetensors",
     "model.layers.41.linear_attn.conv1d.weight": "model-00008-of-00013.safetensors",
@@ -534,30 +534,30 @@
     "model.layers.42.mlp.up_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.42.post_attention_layernorm.weight": "model-00008-of-00013.safetensors",
     "model.layers.43.input_layernorm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.A_log": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.conv1d.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.dt_bias": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.in_proj_a.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.in_proj_b.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.in_proj_qkv.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.in_proj_z.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.norm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.43.linear_attn.out_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.43.mlp.down_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.43.mlp.gate_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.43.mlp.up_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.43.post_attention_layernorm.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.k_norm.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.k_proj.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.o_proj.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.q_norm.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.q_proj.weight": "model-00008-of-00013.safetensors",
-    "model.layers.43.self_attn.v_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.44.input_layernorm.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.A_log": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.conv1d.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.dt_bias": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.in_proj_a.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.in_proj_b.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.in_proj_qkv.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.in_proj_z.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.norm.weight": "model-00008-of-00013.safetensors",
-    "model.layers.44.linear_attn.out_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.44.mlp.down_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.44.mlp.gate_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.44.mlp.up_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.44.post_attention_layernorm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.k_norm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.q_norm.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00008-of-00013.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.45.input_layernorm.weight": "model-00008-of-00013.safetensors",
     "model.layers.45.linear_attn.A_log": "model-00008-of-00013.safetensors",
     "model.layers.45.linear_attn.conv1d.weight": "model-00008-of-00013.safetensors",
@@ -587,30 +587,30 @@
     "model.layers.46.mlp.up_proj.weight": "model-00008-of-00013.safetensors",
     "model.layers.46.post_attention_layernorm.weight": "model-00009-of-00013.safetensors",
     "model.layers.47.input_layernorm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.A_log": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.conv1d.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.dt_bias": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.in_proj_a.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.in_proj_b.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.in_proj_qkv.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.in_proj_z.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.norm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.47.linear_attn.out_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.47.mlp.down_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.47.mlp.gate_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.47.mlp.up_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.47.post_attention_layernorm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.k_norm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.k_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.o_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.q_norm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.q_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.47.self_attn.v_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.48.input_layernorm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.A_log": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.conv1d.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.dt_bias": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.in_proj_a.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.in_proj_b.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.in_proj_qkv.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.in_proj_z.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.norm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.48.linear_attn.out_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.48.mlp.down_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.48.mlp.gate_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.48.mlp.up_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.48.post_attention_layernorm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.k_norm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.k_proj.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.o_proj.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.q_norm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.q_proj.weight": "model-00009-of-00013.safetensors",
+    "model.layers.48.self_attn.v_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.49.input_layernorm.weight": "model-00009-of-00013.safetensors",
     "model.layers.49.linear_attn.A_log": "model-00009-of-00013.safetensors",
     "model.layers.49.linear_attn.conv1d.weight": "model-00009-of-00013.safetensors",
@@ -654,30 +654,30 @@
     "model.layers.50.mlp.up_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.50.post_attention_layernorm.weight": "model-00009-of-00013.safetensors",
     "model.layers.51.input_layernorm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.A_log": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.conv1d.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.dt_bias": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.in_proj_a.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.in_proj_b.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.in_proj_qkv.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.in_proj_z.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.norm.weight": "model-00009-of-00013.safetensors",
+    "model.layers.51.linear_attn.out_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.51.mlp.down_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.51.mlp.gate_proj.weight": "model-00009-of-00013.safetensors",
     "model.layers.51.mlp.up_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.post_attention_layernorm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.k_norm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.k_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.o_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.q_norm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.q_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.51.self_attn.v_proj.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.input_layernorm.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.A_log": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.conv1d.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.dt_bias": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.in_proj_a.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.in_proj_b.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.in_proj_qkv.weight": "model-00009-of-00013.safetensors",
-    "model.layers.52.linear_attn.in_proj_z.weight": "model-00010-of-00013.safetensors",
-    "model.layers.52.linear_attn.norm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.52.linear_attn.out_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.51.post_attention_layernorm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.input_layernorm.weight": "model-00010-of-00013.safetensors",
     "model.layers.52.mlp.down_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.52.mlp.gate_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.52.mlp.up_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.52.post_attention_layernorm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.k_norm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.k_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.o_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.q_norm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.q_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.52.self_attn.v_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.53.input_layernorm.weight": "model-00010-of-00013.safetensors",
     "model.layers.53.linear_attn.A_log": "model-00010-of-00013.safetensors",
     "model.layers.53.linear_attn.conv1d.weight": "model-00010-of-00013.safetensors",
@@ -707,30 +707,30 @@
     "model.layers.54.mlp.up_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.54.post_attention_layernorm.weight": "model-00010-of-00013.safetensors",
     "model.layers.55.input_layernorm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.A_log": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.conv1d.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.dt_bias": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.in_proj_a.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.in_proj_b.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.in_proj_qkv.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.in_proj_z.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.norm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.55.linear_attn.out_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.55.mlp.down_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.55.mlp.gate_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.55.mlp.up_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.55.post_attention_layernorm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.k_norm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.k_proj.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.o_proj.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.q_norm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.q_proj.weight": "model-00010-of-00013.safetensors",
-    "model.layers.55.self_attn.v_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.56.input_layernorm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.A_log": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.conv1d.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.dt_bias": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.in_proj_a.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.in_proj_b.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.in_proj_qkv.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.in_proj_z.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.norm.weight": "model-00010-of-00013.safetensors",
-    "model.layers.56.linear_attn.out_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.56.mlp.down_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.56.mlp.gate_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.56.mlp.up_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.56.post_attention_layernorm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.k_norm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.k_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.o_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.q_norm.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.q_proj.weight": "model-00010-of-00013.safetensors",
+    "model.layers.56.self_attn.v_proj.weight": "model-00010-of-00013.safetensors",
     "model.layers.57.input_layernorm.weight": "model-00010-of-00013.safetensors",
     "model.layers.57.linear_attn.A_log": "model-00010-of-00013.safetensors",
     "model.layers.57.linear_attn.conv1d.weight": "model-00010-of-00013.safetensors",
@@ -760,16 +760,19 @@
     "model.layers.58.mlp.up_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.58.post_attention_layernorm.weight": "model-00011-of-00013.safetensors",
     "model.layers.59.input_layernorm.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.A_log": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.conv1d.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.dt_bias": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.in_proj_a.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.in_proj_b.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.in_proj_qkv.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.in_proj_z.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.norm.weight": "model-00011-of-00013.safetensors",
+    "model.layers.59.linear_attn.out_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.59.mlp.down_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.59.mlp.gate_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.59.mlp.up_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.59.post_attention_layernorm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.k_norm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.k_proj.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.o_proj.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.q_norm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.q_proj.weight": "model-00011-of-00013.safetensors",
-    "model.layers.59.self_attn.v_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00011-of-00013.safetensors",
     "model.layers.6.linear_attn.A_log": "model-00011-of-00013.safetensors",
     "model.layers.6.linear_attn.conv1d.weight": "model-00011-of-00013.safetensors",
@@ -785,19 +788,16 @@
     "model.layers.6.mlp.up_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00011-of-00013.safetensors",
     "model.layers.60.input_layernorm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.A_log": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.conv1d.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.dt_bias": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.in_proj_a.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.in_proj_b.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.in_proj_qkv.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.in_proj_z.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.norm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.60.linear_attn.out_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.60.mlp.down_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.60.mlp.gate_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.60.mlp.up_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.60.post_attention_layernorm.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.k_norm.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.k_proj.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.o_proj.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.q_norm.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.q_proj.weight": "model-00011-of-00013.safetensors",
+    "model.layers.60.self_attn.v_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.61.input_layernorm.weight": "model-00011-of-00013.safetensors",
     "model.layers.61.linear_attn.A_log": "model-00011-of-00013.safetensors",
     "model.layers.61.linear_attn.conv1d.weight": "model-00011-of-00013.safetensors",
@@ -827,30 +827,30 @@
     "model.layers.62.mlp.up_proj.weight": "model-00011-of-00013.safetensors",
     "model.layers.62.post_attention_layernorm.weight": "model-00011-of-00013.safetensors",
     "model.layers.63.input_layernorm.weight": "model-00011-of-00013.safetensors",
-    "model.layers.63.mlp.down_proj.weight": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.A_log": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.conv1d.weight": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.dt_bias": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.in_proj_a.weight": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.in_proj_b.weight": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.in_proj_qkv.weight": "model-00011-of-00013.safetensors",
+    "model.layers.63.linear_attn.in_proj_z.weight": "model-00012-of-00013.safetensors",
+    "model.layers.63.linear_attn.norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.63.linear_attn.out_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.63.mlp.down_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.63.mlp.gate_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.63.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.63.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.k_norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.k_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.o_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.q_norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.q_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.63.self_attn.v_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.64.input_layernorm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.A_log": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.conv1d.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.dt_bias": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.in_proj_a.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.in_proj_b.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.in_proj_qkv.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.in_proj_z.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.64.linear_attn.out_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.64.mlp.down_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.64.mlp.gate_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.64.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.64.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.k_norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.k_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.o_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.q_norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.q_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.64.self_attn.v_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.65.input_layernorm.weight": "model-00012-of-00013.safetensors",
     "model.layers.65.linear_attn.A_log": "model-00012-of-00013.safetensors",
     "model.layers.65.linear_attn.conv1d.weight": "model-00012-of-00013.safetensors",
@@ -880,16 +880,30 @@
     "model.layers.66.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.66.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
     "model.layers.67.input_layernorm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.A_log": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.conv1d.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.dt_bias": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.in_proj_a.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.in_proj_b.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.in_proj_qkv.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.in_proj_z.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.67.linear_attn.out_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.67.mlp.down_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.67.mlp.gate_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.67.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.67.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.k_norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.k_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.o_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.q_norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.q_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.67.self_attn.v_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.input_layernorm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.mlp.down_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.mlp.gate_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.k_norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.k_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.o_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.q_norm.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.q_proj.weight": "model-00012-of-00013.safetensors",
+    "model.layers.68.self_attn.v_proj.weight": "model-00012-of-00013.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00012-of-00013.safetensors",
     "model.layers.7.linear_attn.A_log": "model-00012-of-00013.safetensors",
     "model.layers.7.linear_attn.conv1d.weight": "model-00012-of-00013.safetensors",
@@ -898,20 +912,20 @@
     "model.layers.7.linear_attn.in_proj_b.weight": "model-00012-of-00013.safetensors",
     "model.layers.7.linear_attn.in_proj_qkv.weight": "model-00012-of-00013.safetensors",
     "model.layers.7.linear_attn.in_proj_z.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.linear_attn.norm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.linear_attn.out_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00012-of-00013.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.A_log": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.conv1d.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.dt_bias": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.in_proj_a.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.in_proj_b.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.in_proj_qkv.weight": "model-00012-of-00013.safetensors",
-    "model.layers.8.linear_attn.in_proj_z.weight": "model-00012-of-00013.safetensors",
+    "model.layers.7.linear_attn.norm.weight": "model-00013-of-00013.safetensors",
+    "model.layers.7.linear_attn.out_proj.weight": "model-00013-of-00013.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00013-of-00013.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00013-of-00013.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00013-of-00013.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.A_log": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.conv1d.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.dt_bias": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.in_proj_a.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.in_proj_b.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.in_proj_qkv.weight": "model-00013-of-00013.safetensors",
+    "model.layers.8.linear_attn.in_proj_z.weight": "model-00013-of-00013.safetensors",
     "model.layers.8.linear_attn.norm.weight": "model-00013-of-00013.safetensors",
     "model.layers.8.linear_attn.out_proj.weight": "model-00013-of-00013.safetensors",
     "model.layers.8.mlp.down_proj.weight": "model-00013-of-00013.safetensors",