Add WebGPU pack for Maple 361db5da

.gitattributes

@@ -33,3 +33,30 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+embeddings.mwg filter=lfs diff=lfs merge=lfs -text
+flash-head.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-00.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-01.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-02.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-03.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-04.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-05.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-06.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-07.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-08.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-09.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-10.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-11.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-12.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-13.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-14.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-15.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-16.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-17.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-18.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-19.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-20.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-21.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-22.mwg filter=lfs diff=lfs merge=lfs -text
+layers/layer-23.mwg filter=lfs diff=lfs merge=lfs -text
+lm-head.mwg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,29 @@
+---
+license: mit
+library_name: webgpu
+base_model: deepgrove/maple-preview
+tags:
+  - webgpu
+  - browser
+  - text-generation
+  - quantized
+---
+
+# Maple-Preview WebGPU Pack
+
+Lossless browser-streaming repack of
+[`deepgrove/maple-preview-2bit-mlx`](https://huggingface.co/deepgrove/maple-preview-2bit-mlx)
+for [`ProCreations/maple-webgpu`](https://huggingface.co/spaces/ProCreations/maple-webgpu).
+
+No model values are changed and no additional quantization is applied. The
+official 2-bit/4-bit MLX tensors are regrouped into one aligned binary file per
+transformer layer, reducing browser loading from hundreds of range requests to
+a small set of sequential CDN downloads.
+
+- Source revision: `361db5da5e74ff6fcdd852d478e1f266ce11013a`
+- Packed tensor bytes: `5.31 GB`
+- Container format: `maple-webgpu-pack-v1`
+- Original model and weights: MIT, © DeepGrove AI
+
+This is a runtime artifact, not a separate model release. See `manifest.json`
+for exact byte offsets, shapes, dtypes, and source tensor names.

config.json

@@ -0,0 +1,120 @@
+{
+  "architectures": [
+    "MapleForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 151645,
+  "first_k_dense_replace": 0,
+  "flash_head": {
+    "bits": 4,
+    "cluster_size": 32,
+    "force_tokens": [
+      151645,
+      151668,
+      151643
+    ],
+    "group_size": 64,
+    "head_bits": 4,
+    "head_group_size": 64,
+    "n_clusters": 4748,
+    "n_probes": 512,
+    "scaled_centroids": true
+  },
+  "hc": false,
+  "head_dim": 128,
+  "headwise_gate_swa": false,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 128000,
+  "max_window_layers": 24,
+  "model_file": "maple.py",
+  "model_type": "maple",
+  "moe_intermediate_size": 512,
+  "moe_router_enable_expert_bias": false,
+  "moe_shared_expert_intermediate_size": 512,
+  "mtp_loss_scaling_factor": 0,
+  "nope_on_global_attention": false,
+  "norm_topk_prob": true,
+  "num_attention_heads": 16,
+  "num_experts": 256,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 4,
+  "num_nextn_predict_layers": 0,
+  "num_shared_experts": 0,
+  "output_dropout": 0.0,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "partial_rotary_factor": 0.5,
+  "preaffine": false,
+  "quantization": {
+    "bits": 2,
+    "group_size": 128,
+    "lm_head": {
+      "bits": 4,
+      "group_size": 64
+    },
+    "mode": "affine",
+    "model.word_embeddings": {
+      "bits": 4,
+      "group_size": 64
+    }
+  },
+  "quantization_config": {
+    "bits": 2,
+    "group_size": 128,
+    "lm_head": {
+      "bits": 4,
+      "group_size": 64
+    },
+    "mode": "affine",
+    "model.word_embeddings": {
+      "bits": 4,
+      "group_size": 64
+    }
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "router_dtype": "fp32",
+  "sliding_window": 512,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "use_bias": false,
+  "use_cache": true,
+  "use_qk_norm": true,
+  "use_qkv_bias": false,
+  "use_rmsnorm": true,
+  "vocab_size": 151936
+}

embeddings.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aa71252d85643970971d08aec5401ad1380b9330726e1536203b1487aae43e4
+size 175030272

final-norm.mwg

@@ -0,0 +1 @@
+�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?{?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?v?�?�?�?�?�?�?�?�?t?�?�?�?�?�?�?�?D?�?�?�?w?�?�?�?�?�?�?�?�?�?�?�?�?�?�?`?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?;��?h?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?~?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?z?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�??�?�?�?�?�?�?�?f?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?M?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?x?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?z?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?`?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?]?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?~?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?r?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?a?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?t?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?\?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�??�?�?�?�?[?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�???�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?r?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?z?�?�?�?�?�?�?�?�?�?{?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?S?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?~?�?�?�?�?�?�?�?�?�?�?e?�?�?�?�?�?�?�?i?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?^?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?E?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?_?�?�?�?@L?�?�?C?�?�?�?�?�?�?�?�?�?�?�?�?f?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?8?�?�?�?�?�?�?�?�?�?�?A?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?g?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?n?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�???�?�?�?�?�?{?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�;�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�??�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?y?�?�?�?�?�?�?�?�?�?�?�?�?Z?�?�?�?�?�?�?�?�?�?�?�?�?�?�?[?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?l?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?y?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?]?�?�?�?�?�?�?�?�?�?�?�?�?�?|?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?x?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?x?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?q?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?{?�?�?T?�?�?�?L?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?y?�?�?�?�?`?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?l?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?}?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?n?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�??�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?X?�?�?>?�?�?�?�?�?�?F?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?~?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?]?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�??�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?a?�?�?�?�?�?�?�?�?�?�?\?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?a?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?�?

flash-head.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aaaf06eeef6e0ba5b75dc08e1ef333fc8d0984962d602fccb04ad796c40214b
+size 6087168

layers/layer-00.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1053db29e0808e5591fc4a474ea7b28a564ef15e50362ffd199d21f349898b26
+size 206588416

layers/layer-01.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bedfb318ad72cfee27bb80dce3626f2f14c6fcf846193b6b5bcee8141e2a9b56
+size 206588416

layers/layer-02.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905a24d044534ab7dd4b80160f750b1be435d9219247931e068076ccc32caf39
+size 206588416

layers/layer-03.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f664e54c29ce6030ab6f53b7ee089c06b4efddd852569832ad07afbdc63fe03f
+size 206588416

layers/layer-04.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8341b8e9c76c94e483ca324cb27e01fff2f8081996f3fcb981f973a264db7d9
+size 206588416

layers/layer-05.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff09d57ef313906abb432268a4f7ba5a68fcbb7b125969dafc9493ba20bf51ef
+size 206588416

layers/layer-06.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a98732c2b2f89cdb0e16db558ccaf3f00293338a446fbdf9dacbab55dafa0ed7
+size 206588416

layers/layer-07.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d2260ec99b7ded37f76010d089d1a64099b1b07e78f643bc10cac1b91d0807a
+size 206588416

layers/layer-08.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1842131738e2f51644c59b55cad9b065d1313c40efea195888c48755718208c2
+size 206588416

layers/layer-09.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df604a15e9d863b17647b4e9f16996fbb1d947947c5377327d193b8b48dd190
+size 206588416

layers/layer-10.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6893d64244fb2c949f274ed76fa90e388ba6027a262c4c18d8e2abf49fd849ab
+size 206588416

layers/layer-11.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05c6047ac144dd36a46be8aaba2526de817895ce9881178c1fcf0b9edc6e603b
+size 206588416

layers/layer-12.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721f36873bc9b863a67305cea3854e03bd5c2cb398ac87cfeb6a95aefa472dd2
+size 206588416

layers/layer-13.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d544ab7652df34f7857ec32294a4b4db9add7f5d0ab032d55480a7486bef2e9
+size 206588416

layers/layer-14.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73560b606ecbcad3c47cb22eea6a8f6cf8f778d17b2a6391dff41e0ed61ecd6
+size 206588416

layers/layer-15.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a5a38e65250df412c7b18adfea9009b5bd66a60bf5ce460cd0cc52fe8b4f029
+size 206588416

layers/layer-16.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03720e30d8f23502dbaa027bd948b171ff0c2013555f133260ce85a171ccd194
+size 206588416

layers/layer-17.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905b927d311bd71a8889e95e4c6325368507fbc679f451043650f55e34bfae20
+size 206588416

layers/layer-18.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e81d077bdb804b4f06cfc101787dd76505680a8d5d18b15d4230c2658179fb34
+size 206588416

layers/layer-19.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a578082041cbb2e3794ef89cc4e17bcfda941c6626194eeacb872e35f2a37452
+size 206588416

layers/layer-20.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93198fe8e1bf67adabc1a121be27a6daf32c6187b74fadf4e351172aa86c3dd3
+size 206588416

layers/layer-21.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c793752185414a683f8e8a2d82899fa3c7eab07dca6d12a1984bed381b2a88e7
+size 206588416

layers/layer-22.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dea25dc7f666b0ec2e46d588db09552f672ff53dc97ca0d7c9a7193246658a1
+size 206588416

layers/layer-23.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5028a1b4296bbc72cb8a41a3917c6595de97067a4340dd1f107c3ae6d4cddd53
+size 206588416

lm-head.mwg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a621ca715d5fec85b2d8c7707c2f757e5fd425ebe68ddd3ae3cbcf2c1506fea
+size 175030272

manifest.json

@@ -0,0 +1 @@
+{"format":"maple-webgpu-pack-v1","source":"deepgrove/maple-preview-2bit-mlx","source_revision":"361db5da5e74ff6fcdd852d478e1f266ce11013a","alignment":256,"files":{"embeddings.mwg":{"size":175030272,"tensors":["model.word_embeddings.biases","model.word_embeddings.scales","model.word_embeddings.weight"]},"final-norm.mwg":{"size":4096,"tensors":["model.norm.weight"]},"flash-head.mwg":{"size":6087168,"tensors":["lm_head_flash.centroids.biases","lm_head_flash.centroids.scales","lm_head_flash.centroids.weight","lm_head_flash.cluster_scale","lm_head_flash.token_map"]},"layers/layer-00.mwg":{"size":206588416,"tensors":["model.layers.0.input_layernorm.weight","model.layers.0.mlp.gate.weight","model.layers.0.mlp.switch_mlp.down_proj.row_alpha","model.layers.0.mlp.switch_mlp.down_proj.weight","model.layers.0.mlp.switch_mlp.gate_proj.row_alpha","model.layers.0.mlp.switch_mlp.gate_proj.weight","model.layers.0.mlp.switch_mlp.up_proj.row_alpha","model.layers.0.mlp.switch_mlp.up_proj.weight","model.layers.0.post_attention_layernorm.weight","model.layers.0.self_attn.k_norm.weight","model.layers.0.self_attn.k_proj.row_alpha","model.layers.0.self_attn.k_proj.weight","model.layers.0.self_attn.o_proj.row_alpha","model.layers.0.self_attn.o_proj.weight","model.layers.0.self_attn.q_norm.weight","model.layers.0.self_attn.q_proj.row_alpha","model.layers.0.self_attn.q_proj.weight","model.layers.0.self_attn.v_proj.row_alpha","model.layers.0.self_attn.v_proj.weight"]},"layers/layer-01.mwg":{"size":206588416,"tensors":["model.layers.1.input_layernorm.weight","model.layers.1.mlp.gate.weight","model.layers.1.mlp.switch_mlp.down_proj.row_alpha","model.layers.1.mlp.switch_mlp.down_proj.weight","model.layers.1.mlp.switch_mlp.gate_proj.row_alpha","model.layers.1.mlp.switch_mlp.gate_proj.weight","model.layers.1.mlp.switch_mlp.up_proj.row_alpha","model.layers.1.mlp.switch_mlp.up_proj.weight","model.layers.1.post_attention_layernorm.weight","model.layers.1.self_attn.k_norm.weight","model.layers.1.self_attn.k_proj.row_alpha","model.layers.1.self_attn.k_proj.weight","model.layers.1.self_attn.o_proj.row_alpha","model.layers.1.self_attn.o_proj.weight","model.layers.1.self_attn.q_norm.weight","model.layers.1.self_attn.q_proj.row_alpha","model.layers.1.self_attn.q_proj.weight","model.layers.1.self_attn.v_proj.row_alpha","model.layers.1.self_attn.v_proj.weight"]},"layers/layer-02.mwg":{"size":206588416,"tensors":["model.layers.2.input_layernorm.weight","model.layers.2.mlp.gate.weight","model.layers.2.mlp.switch_mlp.down_proj.row_alpha","model.layers.2.mlp.switch_mlp.down_proj.weight","model.layers.2.mlp.switch_mlp.gate_proj.row_alpha","model.layers.2.mlp.switch_mlp.gate_proj.weight","model.layers.2.mlp.switch_mlp.up_proj.row_alpha","model.layers.2.mlp.switch_mlp.up_proj.weight","model.layers.2.post_attention_layernorm.weight","model.layers.2.self_attn.k_norm.weight","model.layers.2.self_attn.k_proj.row_alpha","model.layers.2.self_attn.k_proj.weight","model.layers.2.self_attn.o_proj.row_alpha","model.layers.2.self_attn.o_proj.weight","model.layers.2.self_attn.q_norm.weight","model.layers.2.self_attn.q_proj.row_alpha","model.layers.2.self_attn.q_proj.weight","model.layers.2.self_attn.v_proj.row_alpha","model.layers.2.self_attn.v_proj.weight"]},"layers/layer-03.mwg":{"size":206588416,"tensors":["model.layers.3.input_layernorm.weight","model.layers.3.mlp.gate.weight","model.layers.3.mlp.switch_mlp.down_proj.row_alpha","model.layers.3.mlp.switch_mlp.down_proj.weight","model.layers.3.mlp.switch_mlp.gate_proj.row_alpha","model.layers.3.mlp.switch_mlp.gate_proj.weight","model.layers.3.mlp.switch_mlp.up_proj.row_alpha","model.layers.3.mlp.switch_mlp.up_proj.weight","model.layers.3.post_attention_layernorm.weight","model.layers.3.self_attn.k_norm.weight","model.layers.3.self_attn.k_proj.row_alpha","model.layers.3.self_attn.k_proj.weight","model.layers.3.self_attn.o_proj.row_alpha","model.layers.3.self_attn.o_proj.weight","model.layers.3.self_attn.q_norm.weight","model.layers.3.self_attn.q_proj.row_alpha","model.layers.3.self_attn.q_proj.weight","model.layers.3.self_attn.v_proj.row_alpha","model.layers.3.self_attn.v_proj.weight"]},"layers/layer-04.mwg":{"size":206588416,"tensors":["model.layers.4.input_layernorm.weight","model.layers.4.mlp.gate.weight","model.layers.4.mlp.switch_mlp.down_proj.row_alpha","model.layers.4.mlp.switch_mlp.down_proj.weight","model.layers.4.mlp.switch_mlp.gate_proj.row_alpha","model.layers.4.mlp.switch_mlp.gate_proj.weight","model.layers.4.mlp.switch_mlp.up_proj.row_alpha","model.layers.4.mlp.switch_mlp.up_proj.weight","model.layers.4.post_attention_layernorm.weight","model.layers.4.self_attn.k_norm.weight","model.layers.4.self_attn.k_proj.row_alpha","model.layers.4.self_attn.k_proj.weight","model.layers.4.self_attn.o_proj.row_alpha","model.layers.4.self_attn.o_proj.weight","model.layers.4.self_attn.q_norm.weight","model.layers.4.self_attn.q_proj.row_alpha","model.layers.4.self_attn.q_proj.weight","model.layers.4.self_attn.v_proj.row_alpha","model.layers.4.self_attn.v_proj.weight"]},"layers/layer-05.mwg":{"size":206588416,"tensors":["model.layers.5.input_layernorm.weight","model.layers.5.mlp.gate.weight","model.layers.5.mlp.switch_mlp.down_proj.row_alpha","model.layers.5.mlp.switch_mlp.down_proj.weight","model.layers.5.mlp.switch_mlp.gate_proj.row_alpha","model.layers.5.mlp.switch_mlp.gate_proj.weight","model.layers.5.mlp.switch_mlp.up_proj.row_alpha","model.layers.5.mlp.switch_mlp.up_proj.weight","model.layers.5.post_attention_layernorm.weight","model.layers.5.self_attn.k_norm.weight","model.layers.5.self_attn.k_proj.row_alpha","model.layers.5.self_attn.k_proj.weight","model.layers.5.self_attn.o_proj.row_alpha","model.layers.5.self_attn.o_proj.weight","model.layers.5.self_attn.q_norm.weight","model.layers.5.self_attn.q_proj.row_alpha","model.layers.5.self_attn.q_proj.weight","model.layers.5.self_attn.v_proj.row_alpha","model.layers.5.self_attn.v_proj.weight"]},"layers/layer-06.mwg":{"size":206588416,"tensors":["model.layers.6.input_layernorm.weight","model.layers.6.mlp.gate.weight","model.layers.6.mlp.switch_mlp.down_proj.row_alpha","model.layers.6.mlp.switch_mlp.down_proj.weight","model.layers.6.mlp.switch_mlp.gate_proj.row_alpha","model.layers.6.mlp.switch_mlp.gate_proj.weight","model.layers.6.mlp.switch_mlp.up_proj.row_alpha","model.layers.6.mlp.switch_mlp.up_proj.weight","model.layers.6.post_attention_layernorm.weight","model.layers.6.self_attn.k_norm.weight","model.layers.6.self_attn.k_proj.row_alpha","model.layers.6.self_attn.k_proj.weight","model.layers.6.self_attn.o_proj.row_alpha","model.layers.6.self_attn.o_proj.weight","model.layers.6.self_attn.q_norm.weight","model.layers.6.self_attn.q_proj.row_alpha","model.layers.6.self_attn.q_proj.weight","model.layers.6.self_attn.v_proj.row_alpha","model.layers.6.self_attn.v_proj.weight"]},"layers/layer-07.mwg":{"size":206588416,"tensors":["model.layers.7.input_layernorm.weight","model.layers.7.mlp.gate.weight","model.layers.7.mlp.switch_mlp.down_proj.row_alpha","model.layers.7.mlp.switch_mlp.down_proj.weight","model.layers.7.mlp.switch_mlp.gate_proj.row_alpha","model.layers.7.mlp.switch_mlp.gate_proj.weight","model.layers.7.mlp.switch_mlp.up_proj.row_alpha","model.layers.7.mlp.switch_mlp.up_proj.weight","model.layers.7.post_attention_layernorm.weight","model.layers.7.self_attn.k_norm.weight","model.layers.7.self_attn.k_proj.row_alpha","model.layers.7.self_attn.k_proj.weight","model.layers.7.self_attn.o_proj.row_alpha","model.layers.7.self_attn.o_proj.weight","model.layers.7.self_attn.q_norm.weight","model.layers.7.self_attn.q_proj.row_alpha","model.layers.7.self_attn.q_proj.weight","model.layers.7.self_attn.v_proj.row_alpha","model.layers.7.self_attn.v_proj.weight"]},"layers/layer-08.mwg":{"size":206588416,"tensors":["model.layers.8.input_layernorm.weight","model.layers.8.mlp.gate.weight","model.layers.8.mlp.switch_mlp.down_proj.row_alpha","model.layers.8.mlp.switch_mlp.down_proj.weight","model.layers.8.mlp.switch_mlp.gate_proj.row_alpha","model.layers.8.mlp.switch_mlp.gate_proj.weight","model.layers.8.mlp.switch_mlp.up_proj.row_alpha","model.layers.8.mlp.switch_mlp.up_proj.weight","model.layers.8.post_attention_layernorm.weight","model.layers.8.self_attn.k_norm.weight","model.layers.8.self_attn.k_proj.row_alpha","model.layers.8.self_attn.k_proj.weight","model.layers.8.self_attn.o_proj.row_alpha","model.layers.8.self_attn.o_proj.weight","model.layers.8.self_attn.q_norm.weight","model.layers.8.self_attn.q_proj.row_alpha","model.layers.8.self_attn.q_proj.weight","model.layers.8.self_attn.v_proj.row_alpha","model.layers.8.self_attn.v_proj.weight"]},"layers/layer-09.mwg":{"size":206588416,"tensors":["model.layers.9.input_layernorm.weight","model.layers.9.mlp.gate.weight","model.layers.9.mlp.switch_mlp.down_proj.row_alpha","model.layers.9.mlp.switch_mlp.down_proj.weight","model.layers.9.mlp.switch_mlp.gate_proj.row_alpha","model.layers.9.mlp.switch_mlp.gate_proj.weight","model.layers.9.mlp.switch_mlp.up_proj.row_alpha","model.layers.9.mlp.switch_mlp.up_proj.weight","model.layers.9.post_attention_layernorm.weight","model.layers.9.self_attn.k_norm.weight","model.layers.9.self_attn.k_proj.row_alpha","model.layers.9.self_attn.k_proj.weight","model.layers.9.self_attn.o_proj.row_alpha","model.layers.9.self_attn.o_proj.weight","model.layers.9.self_attn.q_norm.weight","model.layers.9.self_attn.q_proj.row_alpha","model.layers.9.self_attn.q_proj.weight","model.layers.9.self_attn.v_proj.row_alpha","model.layers.9.self_attn.v_proj.weight"]},"layers/layer-10.mwg":{"size":206588416,"tensors":["model.layers.10.input_layernorm.weight","model.layers.10.mlp.gate.weight","model.layers.10.mlp.switch_mlp.down_proj.row_alpha","model.layers.10.mlp.switch_mlp.down_proj.weight","model.layers.10.mlp.switch_mlp.gate_proj.row_alpha","model.layers.10.mlp.switch_mlp.gate_proj.weight","model.layers.10.mlp.switch_mlp.up_proj.row_alpha","model.layers.10.mlp.switch_mlp.up_proj.weight","model.layers.10.post_attention_layernorm.weight","model.layers.10.self_attn.k_norm.weight","model.layers.10.self_attn.k_proj.row_alpha","model.layers.10.self_attn.k_proj.weight","model.layers.10.self_attn.o_proj.row_alpha","model.layers.10.self_attn.o_proj.weight","model.layers.10.self_attn.q_norm.weight","model.layers.10.self_attn.q_proj.row_alpha","model.layers.10.self_attn.q_proj.weight","model.layers.10.self_attn.v_proj.row_alpha","model.layers.10.self_attn.v_proj.weight"]},"layers/layer-11.mwg":{"size":206588416,"tensors":["model.layers.11.input_layernorm.weight","model.layers.11.mlp.gate.weight","model.layers.11.mlp.switch_mlp.down_proj.row_alpha","model.layers.11.mlp.switch_mlp.down_proj.weight","model.layers.11.mlp.switch_mlp.gate_proj.row_alpha","model.layers.11.mlp.switch_mlp.gate_proj.weight","model.layers.11.mlp.switch_mlp.up_proj.row_alpha","model.layers.11.mlp.switch_mlp.up_proj.weight","model.layers.11.post_attention_layernorm.weight","model.layers.11.self_attn.k_norm.weight","model.layers.11.self_attn.k_proj.row_alpha","model.layers.11.self_attn.k_proj.weight","model.layers.11.self_attn.o_proj.row_alpha","model.layers.11.self_attn.o_proj.weight","model.layers.11.self_attn.q_norm.weight","model.layers.11.self_attn.q_proj.row_alpha","model.layers.11.self_attn.q_proj.weight","model.layers.11.self_attn.v_proj.row_alpha","model.layers.11.self_attn.v_proj.weight"]},"layers/layer-12.mwg":{"size":206588416,"tensors":["model.layers.12.input_layernorm.weight","model.layers.12.mlp.gate.weight","model.layers.12.mlp.switch_mlp.down_proj.row_alpha","model.layers.12.mlp.switch_mlp.down_proj.weight","model.layers.12.mlp.switch_mlp.gate_proj.row_alpha","model.layers.12.mlp.switch_mlp.gate_proj.weight","model.layers.12.mlp.switch_mlp.up_proj.row_alpha","model.layers.12.mlp.switch_mlp.up_proj.weight","model.layers.12.post_attention_layernorm.weight","model.layers.12.self_attn.k_norm.weight","model.layers.12.self_attn.k_proj.row_alpha","model.layers.12.self_attn.k_proj.weight","model.layers.12.self_attn.o_proj.row_alpha","model.layers.12.self_attn.o_proj.weight","model.layers.12.self_attn.q_norm.weight","model.layers.12.self_attn.q_proj.row_alpha","model.layers.12.self_attn.q_proj.weight","model.layers.12.self_attn.v_proj.row_alpha","model.layers.12.self_attn.v_proj.weight"]},"layers/layer-13.mwg":{"size":206588416,"tensors":["model.layers.13.input_layernorm.weight","model.layers.13.mlp.gate.weight","model.layers.13.mlp.switch_mlp.down_proj.row_alpha","model.layers.13.mlp.switch_mlp.down_proj.weight","model.layers.13.mlp.switch_mlp.gate_proj.row_alpha","model.layers.13.mlp.switch_mlp.gate_proj.weight","model.layers.13.mlp.switch_mlp.up_proj.row_alpha","model.layers.13.mlp.switch_mlp.up_proj.weight","model.layers.13.post_attention_layernorm.weight","model.layers.13.self_attn.k_norm.weight","model.layers.13.self_attn.k_proj.row_alpha","model.layers.13.self_attn.k_proj.weight","model.layers.13.self_attn.o_proj.row_alpha","model.layers.13.self_attn.o_proj.weight","model.layers.13.self_attn.q_norm.weight","model.layers.13.self_attn.q_proj.row_alpha","model.layers.13.self_attn.q_proj.weight","model.layers.13.self_attn.v_proj.row_alpha","model.layers.13.self_attn.v_proj.weight"]},"layers/layer-14.mwg":{"size":206588416,"tensors":["model.layers.14.input_layernorm.weight","model.layers.14.mlp.gate.weight","model.layers.14.mlp.switch_mlp.down_proj.row_alpha","model.layers.14.mlp.switch_mlp.down_proj.weight","model.layers.14.mlp.switch_mlp.gate_proj.row_alpha","model.layers.14.mlp.switch_mlp.gate_proj.weight","model.layers.14.mlp.switch_mlp.up_proj.row_alpha","model.layers.14.mlp.switch_mlp.up_proj.weight","model.layers.14.post_attention_layernorm.weight","model.layers.14.self_attn.k_norm.weight","model.layers.14.self_attn.k_proj.row_alpha","model.layers.14.self_attn.k_proj.weight","model.layers.14.self_attn.o_proj.row_alpha","model.layers.14.self_attn.o_proj.weight","model.layers.14.self_attn.q_norm.weight","model.layers.14.self_attn.q_proj.row_alpha","model.layers.14.self_attn.q_proj.weight","model.layers.14.self_attn.v_proj.row_alpha","model.layers.14.self_attn.v_proj.weight"]},"layers/layer-15.mwg":{"size":206588416,"tensors":["model.layers.15.input_layernorm.weight","model.layers.15.mlp.gate.weight","model.layers.15.mlp.switch_mlp.down_proj.row_alpha","model.layers.15.mlp.switch_mlp.down_proj.weight","model.layers.15.mlp.switch_mlp.gate_proj.row_alpha","model.layers.15.mlp.switch_mlp.gate_proj.weight","model.layers.15.mlp.switch_mlp.up_proj.row_alpha","model.layers.15.mlp.switch_mlp.up_proj.weight","model.layers.15.post_attention_layernorm.weight","model.layers.15.self_attn.k_norm.weight","model.layers.15.self_attn.k_proj.row_alpha","model.layers.15.self_attn.k_proj.weight","model.layers.15.self_attn.o_proj.row_alpha","model.layers.15.self_attn.o_proj.weight","model.layers.15.self_attn.q_norm.weight","model.layers.15.self_attn.q_proj.row_alpha","model.layers.15.self_attn.q_proj.weight","model.layers.15.self_attn.v_proj.row_alpha","model.layers.15.self_attn.v_proj.weight"]},"layers/layer-16.mwg":{"size":206588416,"tensors":["model.layers.16.input_layernorm.weight","model.layers.16.mlp.gate.weight","model.layers.16.mlp.switch_mlp.down_proj.row_alpha","model.layers.16.mlp.switch_mlp.down_proj.weight","model.layers.16.mlp.switch_mlp.gate_proj.row_alpha","model.layers.16.mlp.switch_mlp.gate_proj.weight","model.layers.16.mlp.switch_mlp.up_proj.row_alpha","model.layers.16.mlp.switch_mlp.up_proj.weight","model.layers.16.post_attention_layernorm.weight","model.layers.16.self_attn.k_norm.weight","model.layers.16.self_attn.k_proj.row_alpha","model.layers.16.self_attn.k_proj.weight","model.layers.16.self_attn.o_proj.row_alpha","model.layers.16.self_attn.o_proj.weight","model.layers.16.self_attn.q_norm.weight","model.layers.16.self_attn.q_proj.row_alpha","model.layers.16.self_attn.q_proj.weight","model.layers.16.self_attn.v_proj.row_alpha","model.layers.16.self_attn.v_proj.weight"]},"layers/layer-17.mwg":{"size":206588416,"tensors":["model.layers.17.input_layernorm.weight","model.layers.17.mlp.gate.weight","model.layers.17.mlp.switch_mlp.down_proj.row_alpha","model.layers.17.mlp.switch_mlp.down_proj.weight","model.layers.17.mlp.switch_mlp.gate_proj.row_alpha","model.layers.17.mlp.switch_mlp.gate_proj.weight","model.layers.17.mlp.switch_mlp.up_proj.row_alpha","model.layers.17.mlp.switch_mlp.up_proj.weight","model.layers.17.post_attention_layernorm.weight","model.layers.17.self_attn.k_norm.weight","model.layers.17.self_attn.k_proj.row_alpha","model.layers.17.self_attn.k_proj.weight","model.layers.17.self_attn.o_proj.row_alpha","model.layers.17.self_attn.o_proj.weight","model.layers.17.self_attn.q_norm.weight","model.layers.17.self_attn.q_proj.row_alpha","model.layers.17.self_attn.q_proj.weight","model.layers.17.self_attn.v_proj.row_alpha","model.layers.17.self_attn.v_proj.weight"]},"layers/layer-18.mwg":{"size":206588416,"tensors":["model.layers.18.input_layernorm.weight","model.layers.18.mlp.gate.weight","model.layers.18.mlp.switch_mlp.down_proj.row_alpha","model.layers.18.mlp.switch_mlp.down_proj.weight","model.layers.18.mlp.switch_mlp.gate_proj.row_alpha","model.layers.18.mlp.switch_mlp.gate_proj.weight","model.layers.18.mlp.switch_mlp.up_proj.row_alpha","model.layers.18.mlp.switch_mlp.up_proj.weight","model.layers.18.post_attention_layernorm.weight","model.layers.18.self_attn.k_norm.weight","model.layers.18.self_attn.k_proj.row_alpha","model.layers.18.self_attn.k_proj.weight","model.layers.18.self_attn.o_proj.row_alpha","model.layers.18.self_attn.o_proj.weight","model.layers.18.self_attn.q_norm.weight","model.layers.18.self_attn.q_proj.row_alpha","model.layers.18.self_attn.q_proj.weight","model.layers.18.self_attn.v_proj.row_alpha","model.layers.18.self_attn.v_proj.weight"]},"layers/layer-19.mwg":{"size":206588416,"tensors":["model.layers.19.input_layernorm.weight","model.layers.19.mlp.gate.weight","model.layers.19.mlp.switch_mlp.down_proj.row_alpha","model.layers.19.mlp.switch_mlp.down_proj.weight","model.layers.19.mlp.switch_mlp.gate_proj.row_alpha","model.layers.19.mlp.switch_mlp.gate_proj.weight","model.layers.19.mlp.switch_mlp.up_proj.row_alpha","model.layers.19.mlp.switch_mlp.up_proj.weight","model.layers.19.post_attention_layernorm.weight","model.layers.19.self_attn.k_norm.weight","model.layers.19.self_attn.k_proj.row_alpha","model.layers.19.self_attn.k_proj.weight","model.layers.19.self_attn.o_proj.row_alpha","model.layers.19.self_attn.o_proj.weight","model.layers.19.self_attn.q_norm.weight","model.layers.19.self_attn.q_proj.row_alpha","model.layers.19.self_attn.q_proj.weight","model.layers.19.self_attn.v_proj.row_alpha","model.layers.19.self_attn.v_proj.weight"]},"layers/layer-20.mwg":{"size":206588416,"tensors":["model.layers.20.input_layernorm.weight","model.layers.20.mlp.gate.weight","model.layers.20.mlp.switch_mlp.down_proj.row_alpha","model.layers.20.mlp.switch_mlp.down_proj.weight","model.layers.20.mlp.switch_mlp.gate_proj.row_alpha","model.layers.20.mlp.switch_mlp.gate_proj.weight","model.layers.20.mlp.switch_mlp.up_proj.row_alpha","model.layers.20.mlp.switch_mlp.up_proj.weight","model.layers.20.post_attention_layernorm.weight","model.layers.20.self_attn.k_norm.weight","model.layers.20.self_attn.k_proj.row_alpha","model.layers.20.self_attn.k_proj.weight","model.layers.20.self_attn.o_proj.row_alpha","model.layers.20.self_attn.o_proj.weight","model.layers.20.self_attn.q_norm.weight","model.layers.20.self_attn.q_proj.row_alpha","model.layers.20.self_attn.q_proj.weight","model.layers.20.self_attn.v_proj.row_alpha","model.layers.20.self_attn.v_proj.weight"]},"layers/layer-21.mwg":{"size":206588416,"tensors":["model.layers.21.input_layernorm.weight","model.layers.21.mlp.gate.weight","model.layers.21.mlp.switch_mlp.down_proj.row_alpha","model.layers.21.mlp.switch_mlp.down_proj.weight","model.layers.21.mlp.switch_mlp.gate_proj.row_alpha","model.layers.21.mlp.switch_mlp.gate_proj.weight","model.layers.21.mlp.switch_mlp.up_proj.row_alpha","model.layers.21.mlp.switch_mlp.up_proj.weight","model.layers.21.post_attention_layernorm.weight","model.layers.21.self_attn.k_norm.weight","model.layers.21.self_attn.k_proj.row_alpha","model.layers.21.self_attn.k_proj.weight","model.layers.21.self_attn.o_proj.row_alpha","model.layers.21.self_attn.o_proj.weight","model.layers.21.self_attn.q_norm.weight","model.layers.21.self_attn.q_proj.row_alpha","model.layers.21.self_attn.q_proj.weight","model.layers.21.self_attn.v_proj.row_alpha","model.layers.21.self_attn.v_proj.weight"]},"layers/layer-22.mwg":{"size":206588416,"tensors":["model.layers.22.input_layernorm.weight","model.layers.22.mlp.gate.weight","model.layers.22.mlp.switch_mlp.down_proj.row_alpha","model.layers.22.mlp.switch_mlp.down_proj.weight","model.layers.22.mlp.switch_mlp.gate_proj.row_alpha","model.layers.22.mlp.switch_mlp.gate_proj.weight","model.layers.22.mlp.switch_mlp.up_proj.row_alpha","model.layers.22.mlp.switch_mlp.up_proj.weight","model.layers.22.post_attention_layernorm.weight","model.layers.22.self_attn.k_norm.weight","model.layers.22.self_attn.k_proj.row_alpha","model.layers.22.self_attn.k_proj.weight","model.layers.22.self_attn.o_proj.row_alpha","model.layers.22.self_attn.o_proj.weight","model.layers.22.self_attn.q_norm.weight","model.layers.22.self_attn.q_proj.row_alpha","model.layers.22.self_attn.q_proj.weight","model.layers.22.self_attn.v_proj.row_alpha","model.layers.22.self_attn.v_proj.weight"]},"layers/layer-23.mwg":{"size":206588416,"tensors":["model.layers.23.input_layernorm.weight","model.layers.23.mlp.gate.weight","model.layers.23.mlp.switch_mlp.down_proj.row_alpha","model.layers.23.mlp.switch_mlp.down_proj.weight","model.layers.23.mlp.switch_mlp.gate_proj.row_alpha","model.layers.23.mlp.switch_mlp.gate_proj.weight","model.layers.23.mlp.switch_mlp.up_proj.row_alpha","model.layers.23.mlp.switch_mlp.up_proj.weight","model.layers.23.post_attention_layernorm.weight","model.layers.23.self_attn.k_norm.weight","model.layers.23.self_attn.k_proj.row_alpha","model.layers.23.self_attn.k_proj.weight","model.layers.23.self_attn.o_proj.row_alpha","model.layers.23.self_attn.o_proj.weight","model.layers.23.self_attn.q_norm.weight","model.layers.23.self_attn.q_proj.row_alpha","model.layers.23.self_attn.q_proj.weight","model.layers.23.self_attn.v_proj.row_alpha","model.layers.23.self_attn.v_proj.weight"]},"lm-head.mwg":{"size":175030272,"tensors":["lm_head.biases","lm_head.scales","lm_head.weight"]}},"tensors":{"model.word_embeddings.biases":{"file":"embeddings.mwg","offset":0,"nbytes":9723904,"dtype":"BF16","shape":[151936,32]},"model.word_embeddings.scales":{"file":"embeddings.mwg","offset":9723904,"nbytes":9723904,"dtype":"BF16","shape":[151936,32]},"model.word_embeddings.weight":{"file":"embeddings.mwg","offset":19447808,"nbytes":155582464,"dtype":"U32","shape":[151936,256]},"model.norm.weight":{"file":"final-norm.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"lm_head_flash.centroids.biases":{"file":"flash-head.mwg","offset":0,"nbytes":303872,"dtype":"BF16","shape":[4748,32]},"lm_head_flash.centroids.scales":{"file":"flash-head.mwg","offset":303872,"nbytes":303872,"dtype":"BF16","shape":[4748,32]},"lm_head_flash.centroids.weight":{"file":"flash-head.mwg","offset":607744,"nbytes":4861952,"dtype":"U32","shape":[4748,256]},"lm_head_flash.cluster_scale":{"file":"flash-head.mwg","offset":5469696,"nbytes":9496,"dtype":"BF16","shape":[4748]},"lm_head_flash.token_map":{"file":"flash-head.mwg","offset":5479424,"nbytes":607744,"dtype":"I32","shape":[4748,32]},"model.layers.0.input_layernorm.weight":{"file":"layers/layer-00.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.0.mlp.gate.weight":{"file":"layers/layer-00.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.0.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.0.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-00.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.0.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.0.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-00.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.0.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.0.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-00.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.0.post_attention_layernorm.weight":{"file":"layers/layer-00.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.0.self_attn.k_norm.weight":{"file":"layers/layer-00.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.0.self_attn.k_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.0.self_attn.k_proj.weight":{"file":"layers/layer-00.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.0.self_attn.o_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.0.self_attn.o_proj.weight":{"file":"layers/layer-00.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.0.self_attn.q_norm.weight":{"file":"layers/layer-00.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.0.self_attn.q_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.0.self_attn.q_proj.weight":{"file":"layers/layer-00.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.0.self_attn.v_proj.row_alpha":{"file":"layers/layer-00.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.0.self_attn.v_proj.weight":{"file":"layers/layer-00.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.1.input_layernorm.weight":{"file":"layers/layer-01.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.1.mlp.gate.weight":{"file":"layers/layer-01.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.1.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.1.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-01.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.1.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.1.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-01.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.1.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.1.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-01.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.1.post_attention_layernorm.weight":{"file":"layers/layer-01.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.1.self_attn.k_norm.weight":{"file":"layers/layer-01.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.1.self_attn.k_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.1.self_attn.k_proj.weight":{"file":"layers/layer-01.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.1.self_attn.o_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.1.self_attn.o_proj.weight":{"file":"layers/layer-01.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.1.self_attn.q_norm.weight":{"file":"layers/layer-01.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.1.self_attn.q_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.1.self_attn.q_proj.weight":{"file":"layers/layer-01.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.1.self_attn.v_proj.row_alpha":{"file":"layers/layer-01.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.1.self_attn.v_proj.weight":{"file":"layers/layer-01.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.2.input_layernorm.weight":{"file":"layers/layer-02.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.2.mlp.gate.weight":{"file":"layers/layer-02.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.2.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.2.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-02.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.2.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.2.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-02.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.2.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.2.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-02.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.2.post_attention_layernorm.weight":{"file":"layers/layer-02.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.2.self_attn.k_norm.weight":{"file":"layers/layer-02.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.2.self_attn.k_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.2.self_attn.k_proj.weight":{"file":"layers/layer-02.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.2.self_attn.o_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.2.self_attn.o_proj.weight":{"file":"layers/layer-02.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.2.self_attn.q_norm.weight":{"file":"layers/layer-02.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.2.self_attn.q_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.2.self_attn.q_proj.weight":{"file":"layers/layer-02.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.2.self_attn.v_proj.row_alpha":{"file":"layers/layer-02.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.2.self_attn.v_proj.weight":{"file":"layers/layer-02.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.3.input_layernorm.weight":{"file":"layers/layer-03.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.3.mlp.gate.weight":{"file":"layers/layer-03.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.3.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.3.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-03.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.3.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.3.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-03.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.3.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.3.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-03.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.3.post_attention_layernorm.weight":{"file":"layers/layer-03.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.3.self_attn.k_norm.weight":{"file":"layers/layer-03.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.3.self_attn.k_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.3.self_attn.k_proj.weight":{"file":"layers/layer-03.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.3.self_attn.o_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.3.self_attn.o_proj.weight":{"file":"layers/layer-03.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.3.self_attn.q_norm.weight":{"file":"layers/layer-03.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.3.self_attn.q_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.3.self_attn.q_proj.weight":{"file":"layers/layer-03.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.3.self_attn.v_proj.row_alpha":{"file":"layers/layer-03.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.3.self_attn.v_proj.weight":{"file":"layers/layer-03.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.4.input_layernorm.weight":{"file":"layers/layer-04.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.4.mlp.gate.weight":{"file":"layers/layer-04.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.4.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.4.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-04.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.4.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.4.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-04.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.4.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.4.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-04.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.4.post_attention_layernorm.weight":{"file":"layers/layer-04.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.4.self_attn.k_norm.weight":{"file":"layers/layer-04.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.4.self_attn.k_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.4.self_attn.k_proj.weight":{"file":"layers/layer-04.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.4.self_attn.o_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.4.self_attn.o_proj.weight":{"file":"layers/layer-04.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.4.self_attn.q_norm.weight":{"file":"layers/layer-04.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.4.self_attn.q_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.4.self_attn.q_proj.weight":{"file":"layers/layer-04.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.4.self_attn.v_proj.row_alpha":{"file":"layers/layer-04.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.4.self_attn.v_proj.weight":{"file":"layers/layer-04.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.5.input_layernorm.weight":{"file":"layers/layer-05.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.5.mlp.gate.weight":{"file":"layers/layer-05.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.5.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.5.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-05.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.5.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.5.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-05.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.5.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.5.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-05.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.5.post_attention_layernorm.weight":{"file":"layers/layer-05.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.5.self_attn.k_norm.weight":{"file":"layers/layer-05.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.5.self_attn.k_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.5.self_attn.k_proj.weight":{"file":"layers/layer-05.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.5.self_attn.o_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.5.self_attn.o_proj.weight":{"file":"layers/layer-05.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.5.self_attn.q_norm.weight":{"file":"layers/layer-05.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.5.self_attn.q_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.5.self_attn.q_proj.weight":{"file":"layers/layer-05.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.5.self_attn.v_proj.row_alpha":{"file":"layers/layer-05.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.5.self_attn.v_proj.weight":{"file":"layers/layer-05.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.6.input_layernorm.weight":{"file":"layers/layer-06.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.6.mlp.gate.weight":{"file":"layers/layer-06.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.6.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.6.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-06.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.6.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.6.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-06.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.6.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.6.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-06.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.6.post_attention_layernorm.weight":{"file":"layers/layer-06.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.6.self_attn.k_norm.weight":{"file":"layers/layer-06.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.6.self_attn.k_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.6.self_attn.k_proj.weight":{"file":"layers/layer-06.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.6.self_attn.o_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.6.self_attn.o_proj.weight":{"file":"layers/layer-06.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.6.self_attn.q_norm.weight":{"file":"layers/layer-06.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.6.self_attn.q_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.6.self_attn.q_proj.weight":{"file":"layers/layer-06.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.6.self_attn.v_proj.row_alpha":{"file":"layers/layer-06.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.6.self_attn.v_proj.weight":{"file":"layers/layer-06.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.7.input_layernorm.weight":{"file":"layers/layer-07.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.7.mlp.gate.weight":{"file":"layers/layer-07.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.7.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.7.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-07.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.7.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.7.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-07.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.7.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.7.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-07.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.7.post_attention_layernorm.weight":{"file":"layers/layer-07.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.7.self_attn.k_norm.weight":{"file":"layers/layer-07.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.7.self_attn.k_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.7.self_attn.k_proj.weight":{"file":"layers/layer-07.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.7.self_attn.o_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.7.self_attn.o_proj.weight":{"file":"layers/layer-07.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.7.self_attn.q_norm.weight":{"file":"layers/layer-07.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.7.self_attn.q_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.7.self_attn.q_proj.weight":{"file":"layers/layer-07.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.7.self_attn.v_proj.row_alpha":{"file":"layers/layer-07.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.7.self_attn.v_proj.weight":{"file":"layers/layer-07.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.8.input_layernorm.weight":{"file":"layers/layer-08.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.8.mlp.gate.weight":{"file":"layers/layer-08.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.8.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.8.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-08.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.8.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.8.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-08.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.8.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.8.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-08.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.8.post_attention_layernorm.weight":{"file":"layers/layer-08.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.8.self_attn.k_norm.weight":{"file":"layers/layer-08.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.8.self_attn.k_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.8.self_attn.k_proj.weight":{"file":"layers/layer-08.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.8.self_attn.o_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.8.self_attn.o_proj.weight":{"file":"layers/layer-08.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.8.self_attn.q_norm.weight":{"file":"layers/layer-08.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.8.self_attn.q_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.8.self_attn.q_proj.weight":{"file":"layers/layer-08.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.8.self_attn.v_proj.row_alpha":{"file":"layers/layer-08.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.8.self_attn.v_proj.weight":{"file":"layers/layer-08.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.9.input_layernorm.weight":{"file":"layers/layer-09.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.9.mlp.gate.weight":{"file":"layers/layer-09.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.9.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.9.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-09.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.9.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.9.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-09.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.9.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.9.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-09.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.9.post_attention_layernorm.weight":{"file":"layers/layer-09.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.9.self_attn.k_norm.weight":{"file":"layers/layer-09.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.9.self_attn.k_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.9.self_attn.k_proj.weight":{"file":"layers/layer-09.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.9.self_attn.o_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.9.self_attn.o_proj.weight":{"file":"layers/layer-09.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.9.self_attn.q_norm.weight":{"file":"layers/layer-09.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.9.self_attn.q_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.9.self_attn.q_proj.weight":{"file":"layers/layer-09.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.9.self_attn.v_proj.row_alpha":{"file":"layers/layer-09.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.9.self_attn.v_proj.weight":{"file":"layers/layer-09.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.10.input_layernorm.weight":{"file":"layers/layer-10.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.10.mlp.gate.weight":{"file":"layers/layer-10.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.10.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.10.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-10.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.10.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.10.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-10.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.10.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.10.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-10.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.10.post_attention_layernorm.weight":{"file":"layers/layer-10.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.10.self_attn.k_norm.weight":{"file":"layers/layer-10.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.10.self_attn.k_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.10.self_attn.k_proj.weight":{"file":"layers/layer-10.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.10.self_attn.o_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.10.self_attn.o_proj.weight":{"file":"layers/layer-10.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.10.self_attn.q_norm.weight":{"file":"layers/layer-10.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.10.self_attn.q_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.10.self_attn.q_proj.weight":{"file":"layers/layer-10.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.10.self_attn.v_proj.row_alpha":{"file":"layers/layer-10.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.10.self_attn.v_proj.weight":{"file":"layers/layer-10.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.11.input_layernorm.weight":{"file":"layers/layer-11.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.11.mlp.gate.weight":{"file":"layers/layer-11.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.11.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.11.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-11.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.11.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.11.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-11.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.11.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.11.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-11.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.11.post_attention_layernorm.weight":{"file":"layers/layer-11.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.11.self_attn.k_norm.weight":{"file":"layers/layer-11.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.11.self_attn.k_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.11.self_attn.k_proj.weight":{"file":"layers/layer-11.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.11.self_attn.o_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.11.self_attn.o_proj.weight":{"file":"layers/layer-11.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.11.self_attn.q_norm.weight":{"file":"layers/layer-11.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.11.self_attn.q_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.11.self_attn.q_proj.weight":{"file":"layers/layer-11.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.11.self_attn.v_proj.row_alpha":{"file":"layers/layer-11.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.11.self_attn.v_proj.weight":{"file":"layers/layer-11.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.12.input_layernorm.weight":{"file":"layers/layer-12.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.12.mlp.gate.weight":{"file":"layers/layer-12.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.12.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.12.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-12.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.12.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.12.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-12.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.12.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.12.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-12.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.12.post_attention_layernorm.weight":{"file":"layers/layer-12.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.12.self_attn.k_norm.weight":{"file":"layers/layer-12.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.12.self_attn.k_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.12.self_attn.k_proj.weight":{"file":"layers/layer-12.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.12.self_attn.o_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.12.self_attn.o_proj.weight":{"file":"layers/layer-12.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.12.self_attn.q_norm.weight":{"file":"layers/layer-12.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.12.self_attn.q_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.12.self_attn.q_proj.weight":{"file":"layers/layer-12.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.12.self_attn.v_proj.row_alpha":{"file":"layers/layer-12.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.12.self_attn.v_proj.weight":{"file":"layers/layer-12.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.13.input_layernorm.weight":{"file":"layers/layer-13.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.13.mlp.gate.weight":{"file":"layers/layer-13.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.13.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.13.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-13.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.13.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.13.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-13.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.13.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.13.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-13.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.13.post_attention_layernorm.weight":{"file":"layers/layer-13.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.13.self_attn.k_norm.weight":{"file":"layers/layer-13.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.13.self_attn.k_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.13.self_attn.k_proj.weight":{"file":"layers/layer-13.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.13.self_attn.o_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.13.self_attn.o_proj.weight":{"file":"layers/layer-13.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.13.self_attn.q_norm.weight":{"file":"layers/layer-13.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.13.self_attn.q_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.13.self_attn.q_proj.weight":{"file":"layers/layer-13.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.13.self_attn.v_proj.row_alpha":{"file":"layers/layer-13.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.13.self_attn.v_proj.weight":{"file":"layers/layer-13.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.14.input_layernorm.weight":{"file":"layers/layer-14.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.14.mlp.gate.weight":{"file":"layers/layer-14.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.14.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.14.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-14.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.14.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.14.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-14.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.14.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.14.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-14.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.14.post_attention_layernorm.weight":{"file":"layers/layer-14.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.14.self_attn.k_norm.weight":{"file":"layers/layer-14.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.14.self_attn.k_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.14.self_attn.k_proj.weight":{"file":"layers/layer-14.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.14.self_attn.o_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.14.self_attn.o_proj.weight":{"file":"layers/layer-14.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.14.self_attn.q_norm.weight":{"file":"layers/layer-14.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.14.self_attn.q_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.14.self_attn.q_proj.weight":{"file":"layers/layer-14.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.14.self_attn.v_proj.row_alpha":{"file":"layers/layer-14.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.14.self_attn.v_proj.weight":{"file":"layers/layer-14.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.15.input_layernorm.weight":{"file":"layers/layer-15.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.15.mlp.gate.weight":{"file":"layers/layer-15.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.15.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.15.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-15.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.15.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.15.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-15.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.15.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.15.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-15.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.15.post_attention_layernorm.weight":{"file":"layers/layer-15.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.15.self_attn.k_norm.weight":{"file":"layers/layer-15.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.15.self_attn.k_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.15.self_attn.k_proj.weight":{"file":"layers/layer-15.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.15.self_attn.o_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.15.self_attn.o_proj.weight":{"file":"layers/layer-15.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.15.self_attn.q_norm.weight":{"file":"layers/layer-15.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.15.self_attn.q_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.15.self_attn.q_proj.weight":{"file":"layers/layer-15.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.15.self_attn.v_proj.row_alpha":{"file":"layers/layer-15.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.15.self_attn.v_proj.weight":{"file":"layers/layer-15.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.16.input_layernorm.weight":{"file":"layers/layer-16.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.16.mlp.gate.weight":{"file":"layers/layer-16.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.16.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.16.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-16.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.16.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.16.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-16.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.16.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.16.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-16.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.16.post_attention_layernorm.weight":{"file":"layers/layer-16.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.16.self_attn.k_norm.weight":{"file":"layers/layer-16.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.16.self_attn.k_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.16.self_attn.k_proj.weight":{"file":"layers/layer-16.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.16.self_attn.o_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.16.self_attn.o_proj.weight":{"file":"layers/layer-16.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.16.self_attn.q_norm.weight":{"file":"layers/layer-16.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.16.self_attn.q_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.16.self_attn.q_proj.weight":{"file":"layers/layer-16.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.16.self_attn.v_proj.row_alpha":{"file":"layers/layer-16.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.16.self_attn.v_proj.weight":{"file":"layers/layer-16.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.17.input_layernorm.weight":{"file":"layers/layer-17.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.17.mlp.gate.weight":{"file":"layers/layer-17.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.17.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.17.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-17.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.17.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.17.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-17.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.17.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.17.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-17.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.17.post_attention_layernorm.weight":{"file":"layers/layer-17.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.17.self_attn.k_norm.weight":{"file":"layers/layer-17.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.17.self_attn.k_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.17.self_attn.k_proj.weight":{"file":"layers/layer-17.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.17.self_attn.o_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.17.self_attn.o_proj.weight":{"file":"layers/layer-17.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.17.self_attn.q_norm.weight":{"file":"layers/layer-17.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.17.self_attn.q_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.17.self_attn.q_proj.weight":{"file":"layers/layer-17.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.17.self_attn.v_proj.row_alpha":{"file":"layers/layer-17.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.17.self_attn.v_proj.weight":{"file":"layers/layer-17.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.18.input_layernorm.weight":{"file":"layers/layer-18.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.18.mlp.gate.weight":{"file":"layers/layer-18.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.18.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.18.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-18.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.18.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.18.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-18.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.18.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.18.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-18.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.18.post_attention_layernorm.weight":{"file":"layers/layer-18.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.18.self_attn.k_norm.weight":{"file":"layers/layer-18.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.18.self_attn.k_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.18.self_attn.k_proj.weight":{"file":"layers/layer-18.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.18.self_attn.o_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.18.self_attn.o_proj.weight":{"file":"layers/layer-18.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.18.self_attn.q_norm.weight":{"file":"layers/layer-18.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.18.self_attn.q_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.18.self_attn.q_proj.weight":{"file":"layers/layer-18.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.18.self_attn.v_proj.row_alpha":{"file":"layers/layer-18.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.18.self_attn.v_proj.weight":{"file":"layers/layer-18.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.19.input_layernorm.weight":{"file":"layers/layer-19.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.19.mlp.gate.weight":{"file":"layers/layer-19.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.19.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.19.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-19.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.19.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.19.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-19.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.19.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.19.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-19.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.19.post_attention_layernorm.weight":{"file":"layers/layer-19.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.19.self_attn.k_norm.weight":{"file":"layers/layer-19.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.19.self_attn.k_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.19.self_attn.k_proj.weight":{"file":"layers/layer-19.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.19.self_attn.o_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.19.self_attn.o_proj.weight":{"file":"layers/layer-19.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.19.self_attn.q_norm.weight":{"file":"layers/layer-19.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.19.self_attn.q_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.19.self_attn.q_proj.weight":{"file":"layers/layer-19.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.19.self_attn.v_proj.row_alpha":{"file":"layers/layer-19.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.19.self_attn.v_proj.weight":{"file":"layers/layer-19.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.20.input_layernorm.weight":{"file":"layers/layer-20.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.20.mlp.gate.weight":{"file":"layers/layer-20.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.20.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.20.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-20.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.20.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.20.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-20.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.20.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.20.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-20.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.20.post_attention_layernorm.weight":{"file":"layers/layer-20.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.20.self_attn.k_norm.weight":{"file":"layers/layer-20.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.20.self_attn.k_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.20.self_attn.k_proj.weight":{"file":"layers/layer-20.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.20.self_attn.o_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.20.self_attn.o_proj.weight":{"file":"layers/layer-20.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.20.self_attn.q_norm.weight":{"file":"layers/layer-20.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.20.self_attn.q_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.20.self_attn.q_proj.weight":{"file":"layers/layer-20.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.20.self_attn.v_proj.row_alpha":{"file":"layers/layer-20.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.20.self_attn.v_proj.weight":{"file":"layers/layer-20.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.21.input_layernorm.weight":{"file":"layers/layer-21.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.21.mlp.gate.weight":{"file":"layers/layer-21.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.21.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.21.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-21.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.21.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.21.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-21.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.21.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.21.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-21.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.21.post_attention_layernorm.weight":{"file":"layers/layer-21.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.21.self_attn.k_norm.weight":{"file":"layers/layer-21.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.21.self_attn.k_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.21.self_attn.k_proj.weight":{"file":"layers/layer-21.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.21.self_attn.o_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.21.self_attn.o_proj.weight":{"file":"layers/layer-21.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.21.self_attn.q_norm.weight":{"file":"layers/layer-21.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.21.self_attn.q_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.21.self_attn.q_proj.weight":{"file":"layers/layer-21.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.21.self_attn.v_proj.row_alpha":{"file":"layers/layer-21.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.21.self_attn.v_proj.weight":{"file":"layers/layer-21.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.22.input_layernorm.weight":{"file":"layers/layer-22.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.22.mlp.gate.weight":{"file":"layers/layer-22.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.22.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.22.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-22.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.22.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.22.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-22.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.22.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.22.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-22.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.22.post_attention_layernorm.weight":{"file":"layers/layer-22.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.22.self_attn.k_norm.weight":{"file":"layers/layer-22.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.22.self_attn.k_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.22.self_attn.k_proj.weight":{"file":"layers/layer-22.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.22.self_attn.o_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.22.self_attn.o_proj.weight":{"file":"layers/layer-22.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.22.self_attn.q_norm.weight":{"file":"layers/layer-22.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.22.self_attn.q_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.22.self_attn.q_proj.weight":{"file":"layers/layer-22.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.22.self_attn.v_proj.row_alpha":{"file":"layers/layer-22.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.22.self_attn.v_proj.weight":{"file":"layers/layer-22.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.23.input_layernorm.weight":{"file":"layers/layer-23.mwg","offset":0,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.23.mlp.gate.weight":{"file":"layers/layer-23.mwg","offset":4096,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.23.mlp.switch_mlp.down_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":1052672,"nbytes":1048576,"dtype":"BF16","shape":[256,2048]},"model.layers.23.mlp.switch_mlp.down_proj.weight":{"file":"layers/layer-23.mwg","offset":2101248,"nbytes":67108864,"dtype":"U32","shape":[256,2048,32]},"model.layers.23.mlp.switch_mlp.gate_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":69210112,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.23.mlp.switch_mlp.gate_proj.weight":{"file":"layers/layer-23.mwg","offset":69472256,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.23.mlp.switch_mlp.up_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":136581120,"nbytes":262144,"dtype":"BF16","shape":[256,512]},"model.layers.23.mlp.switch_mlp.up_proj.weight":{"file":"layers/layer-23.mwg","offset":136843264,"nbytes":67108864,"dtype":"U32","shape":[256,512,128]},"model.layers.23.post_attention_layernorm.weight":{"file":"layers/layer-23.mwg","offset":203952128,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.23.self_attn.k_norm.weight":{"file":"layers/layer-23.mwg","offset":203956224,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.23.self_attn.k_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":203956480,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.23.self_attn.k_proj.weight":{"file":"layers/layer-23.mwg","offset":203957504,"nbytes":262144,"dtype":"U32","shape":[512,128]},"model.layers.23.self_attn.o_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":204219648,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.23.self_attn.o_proj.weight":{"file":"layers/layer-23.mwg","offset":204223744,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.23.self_attn.q_norm.weight":{"file":"layers/layer-23.mwg","offset":205272320,"nbytes":256,"dtype":"BF16","shape":[128]},"model.layers.23.self_attn.q_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":205272576,"nbytes":4096,"dtype":"BF16","shape":[2048]},"model.layers.23.self_attn.q_proj.weight":{"file":"layers/layer-23.mwg","offset":205276672,"nbytes":1048576,"dtype":"U32","shape":[2048,128]},"model.layers.23.self_attn.v_proj.row_alpha":{"file":"layers/layer-23.mwg","offset":206325248,"nbytes":1024,"dtype":"BF16","shape":[512]},"model.layers.23.self_attn.v_proj.weight":{"file":"layers/layer-23.mwg","offset":206326272,"nbytes":262144,"dtype":"U32","shape":[512,128]},"lm_head.biases":{"file":"lm-head.mwg","offset":0,"nbytes":9723904,"dtype":"BF16","shape":[151936,32]},"lm_head.scales":{"file":"lm-head.mwg","offset":9723904,"nbytes":9723904,"dtype":"BF16","shape":[151936,32]},"lm_head.weight":{"file":"lm-head.mwg","offset":19447808,"nbytes":155582464,"dtype":"U32","shape":[151936,256]}},"skipped":[],"total_tensor_bytes":5314273560}