diff --git a/.gitattributes b/.gitattributes
index a07163b8cc0cf3da1f4f511459105dd1983e1b60..3cc96fd34f17b802e16e07324be5b533bc7d9be7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -18715,3 +18715,11 @@ model/model/layers/6/mlp/experts/gate_proj/weight/4.0.0 filter=lfs diff=lfs merg
 model/model/layers/6/mlp/experts/gate_proj/weight/5.0.0 filter=lfs diff=lfs merge=lfs -text
 model/model/layers/6/mlp/experts/gate_proj/weight/6.0.0 filter=lfs diff=lfs merge=lfs -text
 model/model/layers/6/mlp/experts/gate_proj/weight/7.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/0.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/1.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/2.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/3.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/4.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/5.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/6.0.0 filter=lfs diff=lfs merge=lfs -text
+model/model/layers/6/mlp/experts/up_proj/weight/7.0.0 filter=lfs diff=lfs merge=lfs -text
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/.zarray b/model/model/layers/6/mlp/experts/up_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..2991ac5cdf07c55ed82a809f851193ec646192b7
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[32,4096,2048],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[256,4096,2048],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/0.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/0.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..f522350be53cf54a13d7034b9dc54cbb09d464c7
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/0.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e79f3fc08a8bd8078f907a44e4fed19a31f22d85f64619c5bd6976dafb0dda5f
+size 173665988
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/1.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/1.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..686f6422516041bac9e37ef2e3be8f0a428970e2
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/1.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:771afd50e9d17787bb50f0419b880c2e48430d5590192ae08959f174f1ab4cbf
+size 173675493
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/2.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/2.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..428d4df350c1a3c64e47569e03dd97c8a2787ca8
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/2.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62555a553e42c21bb3bad8baf26edd884c7a38aa7bb43ea2912a427e5042e6a2
+size 173648180
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/3.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/3.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..fa4acc78e9a70feff6d78cc1dd2dbf834f47e697
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/3.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afee660871bc04cac09a4d1f9ab3d7ce1be04818f3741b28873278f53ce93cb2
+size 173601598
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/4.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/4.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..287003693ee8bd79d650e7d8305943a63245fa2f
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/4.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f113e40d37a92b8623422cd0e5fc7fb7c5c94dcde8b6d46feb5bdae2c91738dd
+size 173663664
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/5.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/5.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..e76f2f77c1900835d3e51343425f1cabb4a1d1cb
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/5.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5c24ec0e693122753e827d76e97d3b9493627654980cf722b3d0ab598f92436
+size 173621991
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/6.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/6.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..5471ba9dad050926e9489c7960693c43f0e29dd3
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/6.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c23b8cb5942dba9a56831fd5e26347781987d12b98ccc368042efbbacffb5ae8
+size 173589492
diff --git a/model/model/layers/6/mlp/experts/up_proj/weight/7.0.0 b/model/model/layers/6/mlp/experts/up_proj/weight/7.0.0
new file mode 100644
index 0000000000000000000000000000000000000000..edd08cd9041dd50432ae08c8af16f55bff26eba3
--- /dev/null
+++ b/model/model/layers/6/mlp/experts/up_proj/weight/7.0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:140c6bd83ec9a2c0b7a25259c5121f1ea2cd9e5bf5171402b95a1a104ea1ad07
+size 173626841
diff --git a/model/model/layers/6/mlp/gate/e_score_correction_bias/.zarray b/model/model/layers/6/mlp/gate/e_score_correction_bias/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..653c877efa83da0ab518995885715f7e8b4275d7
--- /dev/null
+++ b/model/model/layers/6/mlp/gate/e_score_correction_bias/.zarray
@@ -0,0 +1 @@
+{"chunks":[256],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[256],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/mlp/gate/e_score_correction_bias/0 b/model/model/layers/6/mlp/gate/e_score_correction_bias/0
new file mode 100644
index 0000000000000000000000000000000000000000..2bdb8e1afd97383770519f8093764cede566d4f3
Binary files /dev/null and b/model/model/layers/6/mlp/gate/e_score_correction_bias/0 differ
diff --git a/model/model/layers/6/mlp/gate/weight/.zarray b/model/model/layers/6/mlp/gate/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6c4a8811f118509288d39f31de2f8d252f44a265
--- /dev/null
+++ b/model/model/layers/6/mlp/gate/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,256],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,256],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/mlp/gate/weight/0.0 b/model/model/layers/6/mlp/gate/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..7411f1b11abeeaba73a2bf3b311417e5e4912acc
--- /dev/null
+++ b/model/model/layers/6/mlp/gate/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:327bfb935e1b670154d483af3d86f984e365319532670c1f0333e30cbc9c450e
+size 1636633
diff --git a/model/model/layers/6/mlp/shared_experts/down_proj/weight/.zarray b/model/model/layers/6/mlp/shared_experts/down_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..8d1cf35badea95c84559f98ec0f56273a52e3e93
--- /dev/null
+++ b/model/model/layers/6/mlp/shared_experts/down_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[2048,4096],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[2048,4096],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/mlp/shared_experts/down_proj/weight/0.0 b/model/model/layers/6/mlp/shared_experts/down_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..1dd9d8ab37681f589274e43db3e631a5f87d072d
--- /dev/null
+++ b/model/model/layers/6/mlp/shared_experts/down_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53edcfd883c4a8d93914ab4b897a31030e2e027c2f0e99809781fca973ba7f2e
+size 9128269
diff --git a/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/.zarray b/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..ee0e127c84bcfbc4a602467da0335ad2bdcebcc7
--- /dev/null
+++ b/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,4096],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,4096],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/0.0 b/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..fa7bc89dea979723d8b26e2fe27588e56f4713f5
--- /dev/null
+++ b/model/model/layers/6/mlp/shared_experts/gate_up_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e381f7481178fc91185980d7f37f0572decdf98395f96fb331222b349bf4bf52
+size 18263722
diff --git a/model/model/layers/6/post_attention_layernorm/weight/.zarray b/model/model/layers/6/post_attention_layernorm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6cb36eccfd612cd6d39e72807aaa8527aa28075d
--- /dev/null
+++ b/model/model/layers/6/post_attention_layernorm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/post_attention_layernorm/weight/0 b/model/model/layers/6/post_attention_layernorm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..90df2ada432a6b37035265f058cb199347b918a0
Binary files /dev/null and b/model/model/layers/6/post_attention_layernorm/weight/0 differ
diff --git a/model/model/layers/6/self_attn/compressor/gate_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/gate_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..596ccedf2526077709306fcdd4608b3771524f10
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/gate_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,1024],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,1024],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/gate_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/gate_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..607efa5d1ea13c1db299623bf173bc82e20ebc2d
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/gate_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c03375b3304c4e434f99d21afbbc02c555032bec952290e7a78a7498b1a22f6
+size 6585521
diff --git a/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6c4a8811f118509288d39f31de2f8d252f44a265
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,256],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,256],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..c0eb16d4f51ad5a6fde1b0917a7d8dfab714e29a
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/gate_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d50dbfe746188741df0104e98754f61f9fdc0548cd6ca1efe902a7238919866
+size 1652612
diff --git a/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/.zarray b/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..3eeba2a45304285824238b9edd4f261c3d5d6f01
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[128],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[128],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/0 b/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..2c12c59b5002921d9f35cd46d05ab4f4b735d9da
Binary files /dev/null and b/model/model/layers/6/self_attn/compressor/indexer/kv_norm/weight/0 differ
diff --git a/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6c4a8811f118509288d39f31de2f8d252f44a265
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,256],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,256],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..c66aa3453c0a564f9b99efecdce120233aad312a
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/kv_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80e8f3a5cf013588ba487dcfef1c4a3a526d13dd0b446a5a75d6249b13e67edc
+size 1644920
diff --git a/model/model/layers/6/self_attn/compressor/indexer/position_bias/.zarray b/model/model/layers/6/self_attn/compressor/indexer/position_bias/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..021a5f3ea06ae5db9b04d668f3bf6295a7a9da08
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/position_bias/.zarray
@@ -0,0 +1 @@
+{"chunks":[4,256],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4,256],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/position_bias/0.0 b/model/model/layers/6/self_attn/compressor/indexer/position_bias/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..845622f1cfcf45fe24a62b042f03bc52f27727bb
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/position_bias/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c8f9e9e5b5f81096b7b189eb6370e0ffb345a6ab8f8f8fab793bf02ab974b1b
+size 1672
diff --git a/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..4befbe8d1a20c1ba183f08f3152932e44abe7e21
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[1024,8192],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[1024,8192],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..341fe362daa58ab4109338eaba68f5929c5f3acb
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/q_b_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb7dec128d6495cd679fb448b395e562dd762cac8513f3af1506935e021d74f
+size 9349553
diff --git a/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6f3804ff38ae99b35c071f7e568a71fc444b112d
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,64],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,64],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..8dde1c76ee75d21d5870c77a6f6253c975cf0bec
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/indexer/scorer/weights_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63df42eefac12ea8a32f2616cfd46e03888dd2187b98f23f8a2276b8a462349b
+size 411523
diff --git a/model/model/layers/6/self_attn/compressor/kv_norm/weight/.zarray b/model/model/layers/6/self_attn/compressor/kv_norm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..82cda9219c678aba952afd8b130fd0430cff4770
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/kv_norm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[512],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[512],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/kv_norm/weight/0 b/model/model/layers/6/self_attn/compressor/kv_norm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..106f942bc93d7b8c979d797f1d29aed51b49b3a2
Binary files /dev/null and b/model/model/layers/6/self_attn/compressor/kv_norm/weight/0 differ
diff --git a/model/model/layers/6/self_attn/compressor/kv_proj/weight/.zarray b/model/model/layers/6/self_attn/compressor/kv_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..596ccedf2526077709306fcdd4608b3771524f10
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/kv_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,1024],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,1024],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/kv_proj/weight/0.0 b/model/model/layers/6/self_attn/compressor/kv_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..71a96afc6db27394e1d5560b94c369e1199012b9
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/kv_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d902f0ca5bb6b600ef8657ace2f873de250f2266ae915e92f12d73b5724e750a
+size 6573121
diff --git a/model/model/layers/6/self_attn/compressor/position_bias/.zarray b/model/model/layers/6/self_attn/compressor/position_bias/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..a3b983d5097485d689093b486c72eb1d8a6dfc3b
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/position_bias/.zarray
@@ -0,0 +1 @@
+{"chunks":[4,1024],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4,1024],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/compressor/position_bias/0.0 b/model/model/layers/6/self_attn/compressor/position_bias/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..fc4cfb263b5e00417b1c02930c35cd28414af692
--- /dev/null
+++ b/model/model/layers/6/self_attn/compressor/position_bias/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed2fddffbc469448d05c70df6b1bd8755f495cab469fad3b82925155c52069d
+size 6277
diff --git a/model/model/layers/6/self_attn/kv_norm/weight/.zarray b/model/model/layers/6/self_attn/kv_norm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..82cda9219c678aba952afd8b130fd0430cff4770
--- /dev/null
+++ b/model/model/layers/6/self_attn/kv_norm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[512],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[512],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/kv_norm/weight/0 b/model/model/layers/6/self_attn/kv_norm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..2c5cf98ee1ce12ba9a2706b71c4cd36546a6ce23
Binary files /dev/null and b/model/model/layers/6/self_attn/kv_norm/weight/0 differ
diff --git a/model/model/layers/6/self_attn/kv_proj/weight/.zarray b/model/model/layers/6/self_attn/kv_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..97de2cf75d9d382e1ef0aeef4ebe9382efa7c3e1
--- /dev/null
+++ b/model/model/layers/6/self_attn/kv_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,512],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,512],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/kv_proj/weight/0.0 b/model/model/layers/6/self_attn/kv_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..86395268d0a3d16314b0b08359f74c33ff51f75b
--- /dev/null
+++ b/model/model/layers/6/self_attn/kv_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cdda6530e6372d510d21143c3292cc109d070c80b858b3aaf19d21a7067a8b1
+size 2297174
diff --git a/model/model/layers/6/self_attn/o_a_proj/weight/.zarray b/model/model/layers/6/self_attn/o_a_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..603a710dd96375059c786c30aa576425b92db7a6
--- /dev/null
+++ b/model/model/layers/6/self_attn/o_a_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,8192],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,8192],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/o_a_proj/weight/0.0 b/model/model/layers/6/self_attn/o_a_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..39e7c8b2c4e77d1921c30340544036a5a5ed962f
--- /dev/null
+++ b/model/model/layers/6/self_attn/o_a_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc5202b9f33896b44c88c69e806b2ee5389d87a8dd2a728a874fb1c44e7869d
+size 36824588
diff --git a/model/model/layers/6/self_attn/o_b_proj/weight/.zarray b/model/model/layers/6/self_attn/o_b_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..fa70329a51a3c0b3fb8e73baa7f5d82b411d2173
--- /dev/null
+++ b/model/model/layers/6/self_attn/o_b_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[8192,4096],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[8192,4096],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/o_b_proj/weight/0.0 b/model/model/layers/6/self_attn/o_b_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..637b713dca2d3d1ac3fa47f930f18e551d543074
--- /dev/null
+++ b/model/model/layers/6/self_attn/o_b_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da41b595fda4de5d9a9e3b9586254eed1e58b819ea0fce5b6fc35cb14aefcf54
+size 36705228
diff --git a/model/model/layers/6/self_attn/q_a_norm/weight/.zarray b/model/model/layers/6/self_attn/q_a_norm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..0566d8c6344b2bc36bc482d2262468c7a92e0582
--- /dev/null
+++ b/model/model/layers/6/self_attn/q_a_norm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[1024],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[1024],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/q_a_norm/weight/0 b/model/model/layers/6/self_attn/q_a_norm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..c93654f77652516a116edbba3dce024a53429494
Binary files /dev/null and b/model/model/layers/6/self_attn/q_a_norm/weight/0 differ
diff --git a/model/model/layers/6/self_attn/q_a_proj/weight/.zarray b/model/model/layers/6/self_attn/q_a_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..596ccedf2526077709306fcdd4608b3771524f10
--- /dev/null
+++ b/model/model/layers/6/self_attn/q_a_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096,1024],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096,1024],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/q_a_proj/weight/0.0 b/model/model/layers/6/self_attn/q_a_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..e8584657ed7fdf7760baea7d7dfdf753cce26039
--- /dev/null
+++ b/model/model/layers/6/self_attn/q_a_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29876adf4535a77beb48c1f481cbcb038960a27bfeb4ebe6e643ef96fd719740
+size 4595641
diff --git a/model/model/layers/6/self_attn/q_b_proj/weight/.zarray b/model/model/layers/6/self_attn/q_b_proj/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..49d11e9a9da84f4753e7fd69167bd1605f559940
--- /dev/null
+++ b/model/model/layers/6/self_attn/q_b_proj/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[1024,32768],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[1024,32768],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/q_b_proj/weight/0.0 b/model/model/layers/6/self_attn/q_b_proj/weight/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..3279afc7c3e372701dd12306828e548a2eb82c64
--- /dev/null
+++ b/model/model/layers/6/self_attn/q_b_proj/weight/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa261cf86d71d4f69a09db67bd58a6683291cb0c65edb3eba88ce3d0a1b6d17b
+size 36885455
diff --git a/model/model/layers/6/self_attn/sinks/.zarray b/model/model/layers/6/self_attn/sinks/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..75fb6ffa552a3ab35caf4dd9123b81fce2aa7eaa
--- /dev/null
+++ b/model/model/layers/6/self_attn/sinks/.zarray
@@ -0,0 +1 @@
+{"chunks":[64],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[64],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/6/self_attn/sinks/0 b/model/model/layers/6/self_attn/sinks/0
new file mode 100644
index 0000000000000000000000000000000000000000..e14797b0b20b046b723bd7d3efef1c50aa8e3412
Binary files /dev/null and b/model/model/layers/6/self_attn/sinks/0 differ
diff --git a/model/model/layers/7/attn_hc/base/.zarray b/model/model/layers/7/attn_hc/base/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..4d11f87d89a75c3774087040f52385e2d6bda040
--- /dev/null
+++ b/model/model/layers/7/attn_hc/base/.zarray
@@ -0,0 +1 @@
+{"chunks":[24],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[24],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/attn_hc/base/0 b/model/model/layers/7/attn_hc/base/0
new file mode 100644
index 0000000000000000000000000000000000000000..dd4e370eabb4bbaa31a56e4d9d93affb67a575bf
Binary files /dev/null and b/model/model/layers/7/attn_hc/base/0 differ
diff --git a/model/model/layers/7/attn_hc/fn/.zarray b/model/model/layers/7/attn_hc/fn/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..5bd8199afd0db0080db1b2df3c5bd70588f8382c
--- /dev/null
+++ b/model/model/layers/7/attn_hc/fn/.zarray
@@ -0,0 +1 @@
+{"chunks":[24,16384],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[24,16384],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/attn_hc/fn/0.0 b/model/model/layers/7/attn_hc/fn/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..836c4f4bfdeb85e3f112c521d9a7a3a8d14f5439
--- /dev/null
+++ b/model/model/layers/7/attn_hc/fn/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f56fe612f2ae2288b19210cb59e04456afed2ff19eddc673bf123ef7bbdd926b
+size 628802
diff --git a/model/model/layers/7/attn_hc/scale/.zarray b/model/model/layers/7/attn_hc/scale/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..0f852799568f02151d44abcc5d47c061ee02cc1c
--- /dev/null
+++ b/model/model/layers/7/attn_hc/scale/.zarray
@@ -0,0 +1 @@
+{"chunks":[3],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[3],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/attn_hc/scale/0 b/model/model/layers/7/attn_hc/scale/0
new file mode 100644
index 0000000000000000000000000000000000000000..57783cde5c623e1f3ae9740552b71ddd26753426
Binary files /dev/null and b/model/model/layers/7/attn_hc/scale/0 differ
diff --git a/model/model/layers/7/ffn_hc/base/.zarray b/model/model/layers/7/ffn_hc/base/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..4d11f87d89a75c3774087040f52385e2d6bda040
--- /dev/null
+++ b/model/model/layers/7/ffn_hc/base/.zarray
@@ -0,0 +1 @@
+{"chunks":[24],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[24],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/ffn_hc/base/0 b/model/model/layers/7/ffn_hc/base/0
new file mode 100644
index 0000000000000000000000000000000000000000..39997937f627b0fb48a4458456f4bf4b0760d6ec
Binary files /dev/null and b/model/model/layers/7/ffn_hc/base/0 differ
diff --git a/model/model/layers/7/ffn_hc/fn/.zarray b/model/model/layers/7/ffn_hc/fn/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..5bd8199afd0db0080db1b2df3c5bd70588f8382c
--- /dev/null
+++ b/model/model/layers/7/ffn_hc/fn/.zarray
@@ -0,0 +1 @@
+{"chunks":[24,16384],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[24,16384],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/ffn_hc/fn/0.0 b/model/model/layers/7/ffn_hc/fn/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..355cc7d6bc1cf3cb8dc9bf1a1c30c622242d6c10
--- /dev/null
+++ b/model/model/layers/7/ffn_hc/fn/0.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:199a37d628e0c014477052eb7677d0b650ce135d9e38254e508a584b9e4ed6cd
+size 629893
diff --git a/model/model/layers/7/ffn_hc/scale/.zarray b/model/model/layers/7/ffn_hc/scale/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..0f852799568f02151d44abcc5d47c061ee02cc1c
--- /dev/null
+++ b/model/model/layers/7/ffn_hc/scale/.zarray
@@ -0,0 +1 @@
+{"chunks":[3],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[3],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/ffn_hc/scale/0 b/model/model/layers/7/ffn_hc/scale/0
new file mode 100644
index 0000000000000000000000000000000000000000..1b8217372989ea1ac900a26f9e24075ca74dac93
Binary files /dev/null and b/model/model/layers/7/ffn_hc/scale/0 differ
diff --git a/model/model/layers/7/input_layernorm/weight/.zarray b/model/model/layers/7/input_layernorm/weight/.zarray
new file mode 100644
index 0000000000000000000000000000000000000000..6cb36eccfd612cd6d39e72807aaa8527aa28075d
--- /dev/null
+++ b/model/model/layers/7/input_layernorm/weight/.zarray
@@ -0,0 +1 @@
+{"chunks":[4096],"compressor":{"id":"zstd","level":1},"dimension_separator":".","dtype":"bfloat16","fill_value":null,"filters":null,"order":"C","shape":[4096],"zarr_format":2}
\ No newline at end of file
diff --git a/model/model/layers/7/input_layernorm/weight/0 b/model/model/layers/7/input_layernorm/weight/0
new file mode 100644
index 0000000000000000000000000000000000000000..1d002c6f951f7cafee1ba89c6c3ce059c35ba2b8
Binary files /dev/null and b/model/model/layers/7/input_layernorm/weight/0 differ