Upload vision tower bundle

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Xiaomi Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE-CODE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,7 @@
+Weights: XiaomiMiMo/MiMo-V2.6-Pro-RL at b61f21852680ee02a29e946e18cf8697f4844061.
+The pinned model card declares MIT; the upstream repository supplies no separate LICENSE file.
+LICENSE reproduces the standard MIT terms, with the Xiaomi copyright attribution.
+The extracted vision implementation retains its upstream Apache-2.0 header.
+See LICENSE-CODE for the Apache-2.0 terms.
+Only visual tensors are included. No language/audio weights.
+Standalone modification: omitted the three checkpoint-absent merger biases. All other vision arithmetic is preserved.

README.md

@@ -0,0 +1,58 @@
+---
+license: mit
+pipeline_tag: image-feature-extraction
+library_name: transformers
+tags:
+- vision
+- feature-extraction
+- mimo
+base_model:
+- XiaomiMiMo/MiMo-V2.6-Pro-RL
+---
+
+# MiMoViT-2.6-Pro
+
+This repository packages the vision Tower and merger from [MiMo V2.6 Pro](https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL).
+
+## Contents
+
+| File | Tensors | What it holds |
+|---|---|---|
+| `model.safetensors` | 364 | Tower blocks, patch embedding and merger, from the parent's `visual.*` |
+| `modeling_mimo_vision.py` | | Vision classes extracted from the parent, loaded with `trust_remote_code` |
+| `config.json`, `preprocessor_config.json` | | Vision configuration and the parent's `Qwen2VLImageProcessor` settings |
+| `export_manifest.json` | | Source pin and the SHA256 of every tensor |
+| `LICENSE`, `LICENSE-CODE`, `NOTICE` | | Weight and code licenses |
+
+## Architecture
+
+| Component | Details |
+|---|---|
+| Tower | 28 blocks, 1280 hidden, SwiGLU 4608, 32 query heads, 8 KV heads, head dimension 64, RMSNorm, 2D RoPE |
+| Patches | 2 frames x 16 x 16 pixels; a still image is repeated as two frames |
+| Attention | Full attention at blocks 0, 9, 18 and 27; the other blocks use a ±64-token window and a per-head sink bias on key column 0 |
+| Merger | `LayerNorm(1280)`, 2x2 concatenation to 5120, `Linear(5120, 5120)`, GELU, `Linear(5120, 6144)`, no biases |
+
+At 448 the bench reports 784 Tower tokens of width 1280 (restored to raster
+order), 196 `merged` tokens of width 5120, and 196 `projected` tokens of width 6144.
+
+## Usage
+
+See [`examples/inference.py`](examples/inference.py) for image feature extraction.
+`AutoModel.from_pretrained(..., trust_remote_code=True)` loads `modeling_mimo_vision.py`.
+
+## Validation
+
+The [release parity test](https://github.com/immanuel-peter/vision-tower-bench/blob/main/tests/test_release_parity.py) compares every exported tensor with the pinned parent using `torch.equal`, then checks all stages against the parent's own `MiMoVisionTransformer`.
+
+## Reproduction
+
+The [export script](https://github.com/immanuel-peter/vision-tower-bench/blob/main/scripts/export_mimo26_vision.py) reads the `visual.*` tensors from the pinned parent and writes the original BF16 tensors.
+
+## Credits
+
+Xiaomi released the MiMo V2.6 Pro weights and the vision implementation.
+
+## License
+
+Weights are MIT, as declared on the source model card. `modeling_mimo_vision.py` keeps its Apache-2.0 header; see [`LICENSE-CODE`](LICENSE-CODE) and [`NOTICE`](NOTICE).

config.json

@@ -0,0 +1,67 @@
+{
+  "architectures": [
+    "MiMoVisionModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_mimo_vision.MiMoVisionConfig",
+    "AutoModel": "modeling_mimo_vision.MiMoVisionModel"
+  },
+  "depth": 28,
+  "dtype": "bfloat16",
+  "fullatt_block_indexes": [
+    0,
+    9,
+    18,
+    27
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 1280,
+  "in_chans": 3,
+  "intermediate_size": 4608,
+  "model_type": "mimo26_vision",
+  "num_heads": 32,
+  "num_key_value_heads": 8,
+  "num_query_groups": 4,
+  "out_hidden_size": 6144,
+  "patch_size": 16,
+  "qk_channels": 64,
+  "rms_norm_eps": 1e-06,
+  "spatial_merge_size": 2,
+  "spatial_patch_size": 16,
+  "temporal_patch_size": 2,
+  "tokens_per_second": 2,
+  "transformers_version": "5.17.0",
+  "use_sink": true,
+  "visual_token_window_size": 64,
+  "vit_window_attn_types": [
+    -1,
+    0,
+    0,
+    0,
+    0,
+    1,
+    1,
+    1,
+    1,
+    -1,
+    0,
+    0,
+    0,
+    0,
+    1,
+    1,
+    1,
+    1,
+    -1,
+    0,
+    0,
+    0,
+    0,
+    1,
+    1,
+    1,
+    1,
+    -1
+  ],
+  "window_size": 128
+}

examples/inference.py

@@ -0,0 +1,32 @@
+import argparse
+from pathlib import Path
+
+from PIL import Image
+import torch
+from transformers import AutoImageProcessor, AutoModel
+
+
+MODEL_ID = "immanuelpeter/MiMoViT-2.6-Pro"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Extract MiMo 2.6 Pro visual features.")
+    parser.add_argument("image", type=Path)
+    parser.add_argument("--model", default=MODEL_ID)
+    args = parser.parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
+    processor = AutoImageProcessor.from_pretrained(args.model)
+    model = AutoModel.from_pretrained(args.model, trust_remote_code=True, dtype=dtype).to(device).eval()
+    inputs = processor(images=Image.open(args.image).convert("RGB"), return_tensors="pt")
+
+    with torch.inference_mode():
+        projected = model(inputs["pixel_values"].to(device, dtype), inputs["image_grid_thw"])
+
+    print("grid_thw", inputs["image_grid_thw"].tolist())
+    print("projected", tuple(projected.shape))
+
+
+if __name__ == "__main__":
+    main()

export_manifest.json

@@ -0,0 +1,372 @@
+{
+  "source_repo": "XiaomiMiMo/MiMo-V2.6-Pro-RL",
+  "source_revision": "b61f21852680ee02a29e946e18cf8697f4844061",
+  "tensors": 364,
+  "bytes": 1478160384,
+  "tensor_sha256": {
+    "blocks.0.attn.proj.bias": "7b2480ae31c491bc2879116b8b038643dfa6f8904250078f7a5e6b16f6df1e02",
+    "blocks.0.attn.proj.weight": "6c1002201b46c8039ad69ee9df992dccf44264959b1f3f114c5cf26a975e2265",
+    "blocks.0.attn.qkv.bias": "f0f47b8a9d8e3bf6266b787695a6890475ad9ff29f5ed90d8ab06165c2660f38",
+    "blocks.0.attn.qkv.weight": "0c8ab40506332d0e11dbbbac9a704f79f138e6bd858c2968a93739f8ecf4f54a",
+    "blocks.0.mlp.down_proj.bias": "44886f29d4c8a430e71067e2bea930de266459fd18c369af7fa2d4fe67c6c57c",
+    "blocks.0.mlp.down_proj.weight": "aa7817e0627149e616ab84e8eb16a03676193d46bf023644d122ddd6f79249d1",
+    "blocks.0.mlp.gate_proj.bias": "9813af711c803a7ec8dcc43909ca4a185ffafc87d1adf76d5752a8195c7bc717",
+    "blocks.0.mlp.gate_proj.weight": "4dff217d7aae9b1cf84e0aca0764c3b50cd8a45457083727e1457412c7c6e8a7",
+    "blocks.0.mlp.up_proj.bias": "fdcc7a015845f416d2f2a7b9e2706a6c0147b2f61f92108eb98b4c897176955c",
+    "blocks.0.mlp.up_proj.weight": "09b94535e41fb79727c84b46290118fdd1fa449fe387a1abf2d1d882497e802f",
+    "blocks.0.norm1.weight": "98f618750d5b6e963ecbe0856e1060a5cd25a20a923041261763800ce9691e2c",
+    "blocks.0.norm2.weight": "4539f91f9263f0efd97708b7e7b389aa375346524edafa829d39cbed43e5e040",
+    "blocks.1.attn.proj.bias": "7668871008c656851fd7a5736e542f84c57b3a5d4f2804de304b7a9f1e712d86",
+    "blocks.1.attn.proj.weight": "cba664e66bde9c6118be615f72b7c1277ec1f843a429b253a8d49705905d36ca",
+    "blocks.1.attn.qkv.bias": "f8f655a5876e8090584e36fbf7da0544f195df143b66fa5478ee5549495c2e19",
+    "blocks.1.attn.qkv.weight": "8c02e8dac3ee0f723fc55caa163a23e4e50e4dd79bbf32b7e4416ffcb5dff6d7",
+    "blocks.1.attn.sinks": "d9e7ce692a94dc9be0ae1e51deb1d913586285115a4545ddc74b412056ec06ff",
+    "blocks.1.mlp.down_proj.bias": "a3248be58f62f0791a1d6a16a8abb73693a038edfc1c8e372995ce69c9d301bc",
+    "blocks.1.mlp.down_proj.weight": "8d71976c1d466c02b94fd7c2d03f0beb589261076c7720a7a7b0edf22a35da78",
+    "blocks.1.mlp.gate_proj.bias": "626274632ffecd43f77feae1a6ea206d8e7d8fa39d0c8bbeb230887a89888ad0",
+    "blocks.1.mlp.gate_proj.weight": "ba91b2569ed7df0cc0438fd789b967c9cd940fd8a5ff81375ec582b1e19bd8e1",
+    "blocks.1.mlp.up_proj.bias": "fb21e96386d8612fd8991c10ea7caa960a9156240c7d3a5b13d8e4c06a099d0d",
+    "blocks.1.mlp.up_proj.weight": "d9303feddbed729bb53b37191642fc40f69c405bcde8991e9ed9da91022ac180",
+    "blocks.1.norm1.weight": "70296b2714a36edb7e8974520b196dbe574eb3c1d5295b5edb52c187ce2b8824",
+    "blocks.1.norm2.weight": "293ee17e4e9dbf733660518a8dc3147e6554fe41dc0def64e1740cb42f012b1f",
+    "blocks.10.attn.proj.bias": "baeb4db9779dce0e0456f2a4786b05972e1d8ce6476e2d1b410e761adf12352d",
+    "blocks.10.attn.proj.weight": "d56565a3fb6538d341f28de1f7c321c1e59d75388386696643de2312b28aaa2c",
+    "blocks.10.attn.qkv.bias": "c2ee4cf386b52268b451ea62609423b3af6ae4f2d83b7624377dc29f51511d0a",
+    "blocks.10.attn.qkv.weight": "6807e3b124a54bf1ec9036a9a6c5eb6065c8b654997e637b68f9845b758d79fa",
+    "blocks.10.attn.sinks": "3226e66c9cf208239eba5b8a8b3202f5ff7b52badefd891141b58688b3360097",
+    "blocks.10.mlp.down_proj.bias": "62ce0b4be0aaa093418697c5a6ecf97a62ebfc1297ab7ef156236d66502c02e6",
+    "blocks.10.mlp.down_proj.weight": "4d73367d9e0d8eda53ae36184084809edd9a7e0b7188ecf42de3970e058b20f3",
+    "blocks.10.mlp.gate_proj.bias": "b45ad12e198559ed513935d5a4a373995a6a912ef551fd463c0d1a8954d82191",
+    "blocks.10.mlp.gate_proj.weight": "ab16d5f29afc58091223412988fd1f8d9c9ffbd30c4a35ce53ba2046f5b44af7",
+    "blocks.10.mlp.up_proj.bias": "bc4414cdf8cd7cf80426010d66e67247577baf4feff3c349122d8f2e1fe2190c",
+    "blocks.10.mlp.up_proj.weight": "76e3d2b6b81ba0a83e94801bf3f7668074edfa73bee948c86f253e755d315560",
+    "blocks.10.norm1.weight": "fbe63bbb8cf075dfc2491c685b1efc6137dbddcddeec3b746c47ef9ad0bcde90",
+    "blocks.10.norm2.weight": "9e1efd65403c22798e1912e5ab8545ab84c43f1e7e854b9d4233d425dc89b74b",
+    "blocks.11.attn.proj.bias": "9423dbddca63f855f271461f02567973fed998f05ce63671d1fb6d088d8a7734",
+    "blocks.11.attn.proj.weight": "add72696982a8232781d8fe791595f3dec8ca171c14371f57e30d619a5d90f88",
+    "blocks.11.attn.qkv.bias": "90edaca044ea3acc9dfe61edc205abbe4b7fa52e80774326952b9872e940ffd9",
+    "blocks.11.attn.qkv.weight": "ff0858dbc2f9f04300170949cf530474210d9b6b1c203830d6daa3cd4a6b4bd0",
+    "blocks.11.attn.sinks": "15dc67c1ebc68bd6304bbe3e611ff1e865c82204872827fc264364ee0f24706c",
+    "blocks.11.mlp.down_proj.bias": "b736864e430a5d2db1b14f6d646b1b24f07ab10b286ac6bfb5c6f0d2525e73d4",
+    "blocks.11.mlp.down_proj.weight": "885e80f861cd6cb60b193447d54ff1767d99b968546996511bf297e078b04112",
+    "blocks.11.mlp.gate_proj.bias": "81522491edc4eb6a6d4c293dba064b4280a410e3a93e2c0ca9d6e59a64819848",
+    "blocks.11.mlp.gate_proj.weight": "aef2076a21671a492f8bdc388b46ec2433e5dc5c53dde3827f4ca1db44902adc",
+    "blocks.11.mlp.up_proj.bias": "d4765d80020188bdad51defccc052873699339ea73608a1086a082dd2f6c545f",
+    "blocks.11.mlp.up_proj.weight": "f74ff8f620b8733e2a9042d4b29b596d6258c8aa41325063e2463e3c459e9474",
+    "blocks.11.norm1.weight": "134a940eb0e208b5d2ac7be9c5106d45cecf0338717eea377b79d6ce2244934b",
+    "blocks.11.norm2.weight": "558cad958c58945a05aa84cfe62a6627c84a4657bfc0d341626d1f7e08429147",
+    "blocks.12.attn.proj.bias": "ff24d5a6e16289cc25cc20b23794ba3b077cde3a9595e13452a5c6f99b304f1c",
+    "blocks.12.attn.proj.weight": "8d5e907e9ee3871a15421ad3b75e523af4281a5f3ee7e76e4536092aa670d287",
+    "blocks.12.attn.qkv.bias": "c3197962d9e46f482b04c11f06d3c8c476c5b32b6fecf0dd09ddadae559e366c",
+    "blocks.12.attn.qkv.weight": "50eb15e818cda5f49eacfa043a3059f6f53d50bd15fcdd80e288852b0cf99890",
+    "blocks.12.attn.sinks": "df98820557845146b7db331a91213f178d0458c756d0d24212e7e3ec909f0a7f",
+    "blocks.12.mlp.down_proj.bias": "73592e7ecbdd61af180dfe0fea4bf89e3d0e1264fb1446e6f9434ed8a433065b",
+    "blocks.12.mlp.down_proj.weight": "9fcfbf1998fe15713284ff6fd0c8dab47fbe7bad8bce2866b10ae1c3ce39cc28",
+    "blocks.12.mlp.gate_proj.bias": "6d8380fa4035992aa307d1f533096475fb36fe924e6b7863ff6b08bf529e64a6",
+    "blocks.12.mlp.gate_proj.weight": "fe966723e06cea6648b84aa4ccda7aa29ec039fd457977e24458f15bc468c6a1",
+    "blocks.12.mlp.up_proj.bias": "205f32a1226e5dac5174ca79083ecb87956243a48565ec0d0b19d6304004c8d3",
+    "blocks.12.mlp.up_proj.weight": "79fc74849e573285b121d8646828dc263889d7982198c905bd784fad93432b07",
+    "blocks.12.norm1.weight": "ce9899a2f34b64e9c61b3630084b0f92d69e9cf40b3db7602b5d54e515e6f6da",
+    "blocks.12.norm2.weight": "eefee2cfd4a1dad4fe9957952e09ad29e7d6cf302129c38c16fca069e816448b",
+    "blocks.13.attn.proj.bias": "c9188404a9d6b1b79ec457464b8865b8776572bb7ba23f8dc41fb2f8a8382c47",
+    "blocks.13.attn.proj.weight": "9ace9ffaa0acc231dedab23729b4b23a9fa0eb18e7dbd0aba484ef0611577b6a",
+    "blocks.13.attn.qkv.bias": "a52e1b485b8785be6ad5f0767193e7de0a14ad9f0bbc1022a590ee5837f25334",
+    "blocks.13.attn.qkv.weight": "db02e70c7fc25d4159179bf956f3566f5070e1317d69e897eebeca91bc355306",
+    "blocks.13.attn.sinks": "aeadad988a14d6a38c65e471a5843d63a9b36f07895675aa44114192fa6ca6ad",
+    "blocks.13.mlp.down_proj.bias": "93612536ece4816ad8e91b5c8307e1427f215ca79abdefdde24e7be2aff64ad7",
+    "blocks.13.mlp.down_proj.weight": "8668b963c6c3eabee50e2a42d28e6b8e760d0b48fc8786df9345898295842461",
+    "blocks.13.mlp.gate_proj.bias": "e5de614244900b36e3b1bb2c525a137cb5ffb1bfcc844972709981057bafc0d1",
+    "blocks.13.mlp.gate_proj.weight": "e8ecbcd0939a97e6c467a68c3cad9f9cbacbdf4270203b034816d6de1e37e130",
+    "blocks.13.mlp.up_proj.bias": "be8efccd8d2dd8695abd7ee09aa710bfd59013950617f55b0e83a350112ec2b7",
+    "blocks.13.mlp.up_proj.weight": "f46ccf4914b3f8c47897781ed4962caca5131cceab45c6d05b5a6b4413cfcc11",
+    "blocks.13.norm1.weight": "24ea951d16ee359c409152301a0f1d02fc2ccab589da4d9b5339bc630b74382f",
+    "blocks.13.norm2.weight": "51eaa47b70902091964b257e52ff3edf89531c23425ea720ec42b0a191797e74",
+    "blocks.14.attn.proj.bias": "5b5e53f0c1374caf41b58ebaf4039f849c9fe04a45f8799e4729ca303a13c420",
+    "blocks.14.attn.proj.weight": "35b6104b90fcac75da23ce1edb56ba9e447500c7fcd916722bc1382e5d5043e5",
+    "blocks.14.attn.qkv.bias": "9921c6699f90faf6c000cca2248fe34d93293a666a8610a1a5df369ab8ab735a",
+    "blocks.14.attn.qkv.weight": "73ad4875dc06017f5d0a66bfdfd038ee06ec13bb881bf94c3e637ba07b877884",
+    "blocks.14.attn.sinks": "23aa9893def16c4695a03397ae3ab9f3be4356f71c24a1939ce5520e70618169",
+    "blocks.14.mlp.down_proj.bias": "4ed7b22848c99c32af2677c0824dd6604cc211c41045c6840ba700183134be71",
+    "blocks.14.mlp.down_proj.weight": "8b96c08be3800d4cce43afdadfbdc641543cf75b19eb1fe170938e4391d949c1",
+    "blocks.14.mlp.gate_proj.bias": "809d3a2ea1463e7939ce8405f7fbd97e953eda22fc55f52c17e178727f724fa8",
+    "blocks.14.mlp.gate_proj.weight": "2f3873f8727e8b7b2818ef82040a4fdb214350eb0a046435c8b6f19c113ddebf",
+    "blocks.14.mlp.up_proj.bias": "e1c658690ddcfc3ddee9d456b412a0126a0161b660aabd62260557cf0441ed3a",
+    "blocks.14.mlp.up_proj.weight": "483ebd343e718f2821620c6064c115534b924696dc79c248af30abfb13dcea2a",
+    "blocks.14.norm1.weight": "b58674bafd4044d7120e77063fff4b91670de5842b4e814825266f52b45b57d7",
+    "blocks.14.norm2.weight": "00acc864c482d9774d84ad7ef2d2b9c7052e0924c0862e86c6ee9fa9ada3c328",
+    "blocks.15.attn.proj.bias": "3ae017853cb8acc1caddede70e599d28874ea1633ced11d6ba1900ab37dcc20d",
+    "blocks.15.attn.proj.weight": "2380b3ca401274e9146375f36c03219ba64aa0624463239f1185464d26487e87",
+    "blocks.15.attn.qkv.bias": "fa2a9e8f5da383805bc1ec5412679a838b4b018c4a15152c0f88c4600ab704e2",
+    "blocks.15.attn.qkv.weight": "3219aa4170a2af10019ada841563cabc637a74257a166c58d925179c3addac74",
+    "blocks.15.attn.sinks": "5ee220386279a106e0168532ce388eb502eb0a3d48750064ba7c9f3936c4d5b0",
+    "blocks.15.mlp.down_proj.bias": "9fac85bdf72c4a0b5f06755bf6305d4002e594e6a347849564885cdcb8c3a761",
+    "blocks.15.mlp.down_proj.weight": "95f4dbff7d00fc52469f4e9c9fdcfd26e5d7498be023a171d52591cdb4db9913",
+    "blocks.15.mlp.gate_proj.bias": "ca0b172c8853f5014eaf73ed0373c559a7db902293d9559c93b2d926a1b906d2",
+    "blocks.15.mlp.gate_proj.weight": "13303067e8b97a5772af64e2677be554c01fa86eaf13fb17c4c7ad05e5b100a1",
+    "blocks.15.mlp.up_proj.bias": "1d00ac6b3409f48b339a13d2d082419a356d5f14492d8831eb04f59aba39e599",
+    "blocks.15.mlp.up_proj.weight": "12e852ee88cbe69c267dfeed94e3885cc22869574b3e445cbc4a85bd554c5a0a",
+    "blocks.15.norm1.weight": "72c4e7c6ecd2d80c1f5134ffcbe738e2907b5705b0518592bbae5a0bbc84bad4",
+    "blocks.15.norm2.weight": "248c2c2cf326021ea4d03e0e1d8bb8577c5902b92ce30448d4710f6b17cf8942",
+    "blocks.16.attn.proj.bias": "6672e11bcde20f808b016e3de947a04f1b8a1582880a3976d38323a7810ae85a",
+    "blocks.16.attn.proj.weight": "95b30581f0c7996ab384b62aa1a5dca213aad292f14c8df55e71f94b7f7ccbfa",
+    "blocks.16.attn.qkv.bias": "09aa10ba9ed00af93ade65dbef91ff07ad329c0eaeaec04eb21697f8101d71cd",
+    "blocks.16.attn.qkv.weight": "dc4e051941fd69c6bed503323af408b8ddd8067bb260b9d54f4d19a4506af36e",
+    "blocks.16.attn.sinks": "98e189a6fc1e7d2e9ad777f44dda4638f46525182368b1e5ceb909da26b96016",
+    "blocks.16.mlp.down_proj.bias": "517185e00ee30c19eb4a019b0d32652693c39755de84f4c787774e09f5960323",
+    "blocks.16.mlp.down_proj.weight": "20dbe9b824ecd2842d61bcdbfed71d91b0e6a534c5741a7a6700961765100369",
+    "blocks.16.mlp.gate_proj.bias": "540f1427f1e73911814c18f182f3306b9ccbddd386dcde513a0b862bb379b4fb",
+    "blocks.16.mlp.gate_proj.weight": "7023222b4862669a10deaa88898258c59f3952a04d3af0dbdc07314599473718",
+    "blocks.16.mlp.up_proj.bias": "ea8b373a13622334249ec6865ba5abbce0f52fc9123073acc99fdb6a65b6999a",
+    "blocks.16.mlp.up_proj.weight": "2a2190a3d2c78e7b19c92f25c58a24c599304e91d43989bb4e669b19f16b0547",
+    "blocks.16.norm1.weight": "14204a4245f1773416158b30b5f30687bd80d69fd82d59d75926536f7d561a1f",
+    "blocks.16.norm2.weight": "aa7a3c280d1f9a50f21177d4802b04baf177e16e1e99615ad263ec1e1b2add00",
+    "blocks.17.attn.proj.bias": "e8201cfc7fb6aed84909f0a034193e76def80797288de9dcf3ce943f0ca3c5ee",
+    "blocks.17.attn.proj.weight": "4090de5f82ea13d2a641a9bf3b8d633a3680d1bd19dab4493fba703e431138e5",
+    "blocks.17.attn.qkv.bias": "2beed333d202ad889c796f14cd3db0ceeb0f85955505f9189ea5c9e05044a30c",
+    "blocks.17.attn.qkv.weight": "9eb46fe4161580c0e21963d4e9a4b1710838d16bf8850b2dc95b78b64f8ff9fa",
+    "blocks.17.attn.sinks": "cc2795b0293e201634958442d48a3ae4c48a3f6c61c3dad973d4e6c23e3f54c3",
+    "blocks.17.mlp.down_proj.bias": "83947676e654a795c2a398ad2bf38e8ce910a3b91e6693e18c4e008fa71a2a44",
+    "blocks.17.mlp.down_proj.weight": "203f33c1f730e4a1cad7264ec44b332a286545fbd66121281e3ffdf3a13fce0f",
+    "blocks.17.mlp.gate_proj.bias": "4793105dc83a603c0854bb587b04c6656689076593e7fa14c8e23f55d1d6f967",
+    "blocks.17.mlp.gate_proj.weight": "9a6cf5adf28d54790069fa715a5a6d5de16a4b3ad124543667e8b61a9e0039d8",
+    "blocks.17.mlp.up_proj.bias": "25d68b9d2fd213122cbdf7351d6066a5e1ebaa8d496b1130ea1ebcd7e50757d2",
+    "blocks.17.mlp.up_proj.weight": "00920d0818b52109baf1a627be11311cb8e91e8286af0d4190c48cf0effb05aa",
+    "blocks.17.norm1.weight": "948918d73e1885358aef9905c18aeebb2adb90d3055bc2e196a462031093c087",
+    "blocks.17.norm2.weight": "bb55714d187ddbc17cb97ec3474a66ba536826073df98bbfee0e092228de43fc",
+    "blocks.18.attn.proj.bias": "f9aa7eaa9d3fca6eae000fb6a02a6e4c1c063dd02a0079745a5f19c703f48006",
+    "blocks.18.attn.proj.weight": "4cb54689107dfa290f21eaa56265fa79422dac5eb83bb60239ff82c3c538ce06",
+    "blocks.18.attn.qkv.bias": "8641813e7ef9e3f4903e1912ee115b9ce533d289df8c6317b7e6783e10e152e8",
+    "blocks.18.attn.qkv.weight": "fff85d12f9a28f299d91825fb827672ca9289652a0fdcabba0925cc23d65a1c5",
+    "blocks.18.mlp.down_proj.bias": "bc178581ece1957719c01749044b31874397d6ba04acd9adb5d084eb80c062f0",
+    "blocks.18.mlp.down_proj.weight": "5a082a0d2256321caa1977c6580a73cca9f12a99ab04dc08e1049579e42efedd",
+    "blocks.18.mlp.gate_proj.bias": "5eabd9483265960fa66757346c2e141488eef8b3793abda0f10194253f99647a",
+    "blocks.18.mlp.gate_proj.weight": "f76e9d1ebd0c64aa5d2bdb36ae9789f5e5fcb8a3bdb681f74659ee83c7e93c9b",
+    "blocks.18.mlp.up_proj.bias": "7ee4067121aba3d8a8d54f5ee8f6fc896f97b87a6e98a4d56a35b333e63dc233",
+    "blocks.18.mlp.up_proj.weight": "828b8b8a89909c38f349d90cadeba3cf013567eb52adf788d9775ab73930c3dd",
+    "blocks.18.norm1.weight": "3defb305780262c5527aac3a60562372a6b422c88372a36ec8e7fc1364016bd2",
+    "blocks.18.norm2.weight": "4f67cb627bb5dfdfb96e5cf3f2ac05702e2a90ad6a8894c702b572288083b3ff",
+    "blocks.19.attn.proj.bias": "0bdd2795ecbefb4ed3db23c8a62bd7a569ac0ea84f7c1e71c832b4c9ec944402",
+    "blocks.19.attn.proj.weight": "fbe4c46fc20ac088c795bc34facbe387d0a2486011ee5cc450bf5a65a97c61de",
+    "blocks.19.attn.qkv.bias": "1c6284b809c4b1fd4bccf738e92ae0168f4cc06a2c7628813fa9fda08c4d6234",
+    "blocks.19.attn.qkv.weight": "85abd2a05b80d51fdc478ad5562ce1688a33ee5ac9a8753fe0a52f9a401b1522",
+    "blocks.19.attn.sinks": "94ffc41efcc8482b89a31d148a12e1acfcbe85a2105cc13a3ff57a635da51b3f",
+    "blocks.19.mlp.down_proj.bias": "57875ce4af8ad4480132602ffd039f4873178a99b369cea93dfb03a212852acf",
+    "blocks.19.mlp.down_proj.weight": "ee1c84ef45829d02a16aa1c72fdd67a8af17317810e901bbe15ccc824460377b",
+    "blocks.19.mlp.gate_proj.bias": "54d458d0ce2a841eae431f988b335021ad8c1d8202e11a0b838fb562bf04fbe9",
+    "blocks.19.mlp.gate_proj.weight": "f848c096e3463891632db0489e9606b6683c6402f6f3d36fe4b80284bb8e1497",
+    "blocks.19.mlp.up_proj.bias": "371d47adc9431fa4d58619273f64656e3102d3a81c46f95f20ea1cefdaeef651",
+    "blocks.19.mlp.up_proj.weight": "dda4018f4505fba3108ac0e3170f870a61a3745c6fc34adb2b44f7a38c2b6edc",
+    "blocks.19.norm1.weight": "c5d6f91953fe8b655045d6e283a46faa2537df1382ffe6d85d533d7217a8ac0a",
+    "blocks.19.norm2.weight": "29dd8f576442dd6e8b484bf8f1e17e89fe2f3d873baee7e90c1ec7d367bf5c3c",
+    "blocks.2.attn.proj.bias": "0cf496a43ab7979ac8977876ab80a0fd74173496ba103701b363360f37a2cd65",
+    "blocks.2.attn.proj.weight": "75aeb0b0736fdc9d25444d3b7b9ad9c3aaee192a4ab1e708fb59768fd33849a2",
+    "blocks.2.attn.qkv.bias": "a25c555a179c003e4894ce91fe193bbfcc06b0978a68d59edc00d9341162438a",
+    "blocks.2.attn.qkv.weight": "6277ed70af4f1d28f22505d5edc76c8437b598ed96cc360659c0071bdae8e22a",
+    "blocks.2.attn.sinks": "132349d8695233c20c4d0b56afe8b8683db7374ad3c21925c26ef7ad2c9a2203",
+    "blocks.2.mlp.down_proj.bias": "633a4c7881a62542b345ad3ef09fdda4baa9c91cb2772dcdfdda8a366f4ce923",
+    "blocks.2.mlp.down_proj.weight": "735c0fd91afe24f49497cffe21c96ca3d970299b2ffd70ecabcd3e4cf93342ed",
+    "blocks.2.mlp.gate_proj.bias": "5e4fb6fbbb280834122da7e03b3ce9a068c59dec12fd9b9d0560ffe23a0035df",
+    "blocks.2.mlp.gate_proj.weight": "32064a63716bd454f692e19d9f7fc101447d10eaed1688a494cacc96e3e561b5",
+    "blocks.2.mlp.up_proj.bias": "c962412ab3250869df0d30f4f7a24d80c2f955fed24629653d9a7ababd64b9d1",
+    "blocks.2.mlp.up_proj.weight": "5775b7c3c9b5fbff5b95407db948826905efd0dd96e45626e607d2ca974d593e",
+    "blocks.2.norm1.weight": "908bd3e82d2dd063e2e0fc20743e37566d666cfb8531ae4a37142af06554619b",
+    "blocks.2.norm2.weight": "24ddee4740312c9e495527fc1282ad2b25d36aaaf7681628c08f9495b30403e1",
+    "blocks.20.attn.proj.bias": "75879010b3e62bc60a49764a307243003d0616fdf1eb27697e04b8a9a0469ba2",
+    "blocks.20.attn.proj.weight": "e2334428579ace3a589e64b5aca3ec1aa31b633aa9890b70f0f0500ccc4984e2",
+    "blocks.20.attn.qkv.bias": "16a82121435c7d6af47feddbfac00c051fc9b7269e801b6de0d5c4930c411a4a",
+    "blocks.20.attn.qkv.weight": "740b1669892bf0e89a323398147c25599e98b16f0ce65116d2916d4a5387baef",
+    "blocks.20.attn.sinks": "c3477fbe0d63ff7c11c7bc9b8b524e7394be1c7be8564712bab782dd4e4b65e0",
+    "blocks.20.mlp.down_proj.bias": "12c361b6a2261401b7c0829257ca34e77f7f3d843c6b12a1794d611ddcc6501b",
+    "blocks.20.mlp.down_proj.weight": "3341fb173b940a6fe67705a1dabe98f30aafcd00a75c51ff6ead824d0ab183f4",
+    "blocks.20.mlp.gate_proj.bias": "92fd3287527d913c2d082dfd8cd70d96e274a926b95088c59205cdef6a24d9db",
+    "blocks.20.mlp.gate_proj.weight": "a2ade14e1a07b30bb9f0990141ab638fbbdd586ecf02ad6dbd844d0a86f78944",
+    "blocks.20.mlp.up_proj.bias": "d02419a2ad13e6ab7fa817f23cfa34f278b679ec9b2ab1de854b353f6c5581ea",
+    "blocks.20.mlp.up_proj.weight": "c83b1e707bb8fbd27184f440109df101898b7fb1295ad4c18e5d4d239b7b21a6",
+    "blocks.20.norm1.weight": "8ce96193af63cb3cd9ac96965091ac3ba93c79ee31fc83c2461515011eb8b820",
+    "blocks.20.norm2.weight": "5c9918204bc734d8f568ac5542bb0b2c8adaae7aa6da66c59c7996f323ca34ca",
+    "blocks.21.attn.proj.bias": "9b10f8ca7cde97f58ab5834eea898ed5118902479d40df930351d2f878fe1612",
+    "blocks.21.attn.proj.weight": "e8168d35b61b1aaf4f54a18840b6705c57bdeb5bc8a9d8ecd1674c5967a0b228",
+    "blocks.21.attn.qkv.bias": "9450a58337c44143a45762dbd9c04e4c7c97365244518c45d56cdf0b17bdc63a",
+    "blocks.21.attn.qkv.weight": "a77cf6b1299417ecd4d8f702c5533a587ad84f2a6549655fb7cc59871afbcd21",
+    "blocks.21.attn.sinks": "39abf73b1d89a694838e9e6e3fdbaa311ad8f959e51ff7479d7ff03beb5d5907",
+    "blocks.21.mlp.down_proj.bias": "1210199fdcd556c9c4c9ae0882dce009a078c162596c7f0f975a2068929e7ea7",
+    "blocks.21.mlp.down_proj.weight": "06067229251dedd95485185ebffd03809cb99fc716cd1c9190f684bd393b1381",
+    "blocks.21.mlp.gate_proj.bias": "18169e0ef515e1e681779bc1df13ad9d1bc52992a9905da4a83154a1c66d9a34",
+    "blocks.21.mlp.gate_proj.weight": "329288608f13d40c64f0ad18b8a036226864f97e70a05b7fe31e2610d732ee2c",
+    "blocks.21.mlp.up_proj.bias": "8c0b3a1582cf009a36707715ead7f81c9d61ce77e4442b857ed7d4752e7a604f",
+    "blocks.21.mlp.up_proj.weight": "7f0feb6fea960992a30597dbbca14bc8a480c62a827af78826f08cb989976de5",
+    "blocks.21.norm1.weight": "ee0cc9f5f2408f2098c50a83d96c53c72137d596b14a7567223dfdb23a0ab0cc",
+    "blocks.21.norm2.weight": "be0a7229e50d47f95a3d2425c7e6bf4502675a851a59b169e3ac48b5a6b07db6",
+    "blocks.22.attn.proj.bias": "ce3a83e6733e7e462ef555715f19832e5601f8afe8bedd4042d217d9f56113ee",
+    "blocks.22.attn.proj.weight": "ef41f88e6a57756c1bb9598c4babdc67038a67b2b8aec5af2d185113f453d285",
+    "blocks.22.attn.qkv.bias": "005ae3afc3a31200f2a245c2236fe0847f7a6fbd87242377883a896bd7f9a066",
+    "blocks.22.attn.qkv.weight": "6c7003657fac39dfe71981429da18a6e9a04161a893284cd2a1e2998fd8330bb",
+    "blocks.22.attn.sinks": "7f29b71ef4924180ae4b959a77214a0e5de0232102cec9f92e83ce4c33e13612",
+    "blocks.22.mlp.down_proj.bias": "4fe411259cb35fe51864c13b1c543ae832b62646c62544396d7469bb171a53e5",
+    "blocks.22.mlp.down_proj.weight": "8fc8981063f86eba9ddb770c2795e289b34815f8dd1785cc85c83e8a4fd490e5",
+    "blocks.22.mlp.gate_proj.bias": "78e6e86e4bc73c32243a32b39b1546c779333b74047cacdf532be846ed73a3fa",
+    "blocks.22.mlp.gate_proj.weight": "19c2efe9c9ccf0d2687bf098cc98c96e82c1bc674374c14921ff3fa035be88b6",
+    "blocks.22.mlp.up_proj.bias": "465c75909683a89c6dd550a2af4e06d27b943dc49e80ab96ad97a7db2a6277fd",
+    "blocks.22.mlp.up_proj.weight": "0d4b30157147ae87c3ba5d3464b45fe85595a253d02561451c4f8a821128033b",
+    "blocks.22.norm1.weight": "5e9f50f769435c89d328ddbb892eac79415eea7155cd6a3dfffee197e4f69b67",
+    "blocks.22.norm2.weight": "99bee73b41e2d7959686317848a30a97561aa014f4193122dd4e78ef3180f189",
+    "blocks.23.attn.proj.bias": "72a4d7663d203cfd14e95e95131516efc186f48275b4489361fe18c57ffe367a",
+    "blocks.23.attn.proj.weight": "d6c440a945a1500a8ffeeb91fe365bbc4a14b9b15841d607702077dfc38e057c",
+    "blocks.23.attn.qkv.bias": "8cf63f53a2198c962d3f8539a8815b00b2916c2da66fc5d151e1c69b6629eb6a",
+    "blocks.23.attn.qkv.weight": "be92c2dc4e0874f35648804af622d1613718099a9ff006deb7f1d9e773c267b1",
+    "blocks.23.attn.sinks": "704f9fa565cce9db810a92623a1d98bc5724ba53848892a869f47a8da2d46570",
+    "blocks.23.mlp.down_proj.bias": "0df84ef077dd10f465637781c5e9cd4ae85dabaeb79288956f70b801ae98f699",
+    "blocks.23.mlp.down_proj.weight": "25edd300db59457905a84cc921e92ca3183340aa074212a1b88642b9723ead72",
+    "blocks.23.mlp.gate_proj.bias": "fc34a373c6cc86089c23891d2cb4db49fa56af875068b6493d7f705a63884ce5",
+    "blocks.23.mlp.gate_proj.weight": "43bfd4573f3c088627704f5e73e532b8926513d79d9ef253f45708792515e5a7",
+    "blocks.23.mlp.up_proj.bias": "3528879306f306bf8662e233c4cb117e3455eeef67180ed4c8fe2bbe699fd4f8",
+    "blocks.23.mlp.up_proj.weight": "bf84ea79318e9884f1a1253e61d0d12fffa447e2fe9010f1e6aabe6786fa717d",
+    "blocks.23.norm1.weight": "4bcec3d0d14598fedfa3234a5188dab30d9b083adf71ac95a808a9b6b93dd54e",
+    "blocks.23.norm2.weight": "2329bd32824ffd24de873e059f6a57764462d28d555905db4a5f513fbcdbe111",
+    "blocks.24.attn.proj.bias": "880384fac241bc94ef6f5b06752d273ba0ddeedeeb0007dda8d15857ad037580",
+    "blocks.24.attn.proj.weight": "72b760600bd00ae19753d3c2e72c92c1afeac6403c30919cb6e46ea2a0bf8e78",
+    "blocks.24.attn.qkv.bias": "eedbe55d329e35e36a17c464520f9d5ca73dd957e24199f0356c72488e5b5437",
+    "blocks.24.attn.qkv.weight": "4eea1c7f71f46a16bb21fd15c69d52754e17a042be03324ea11d709a85573ac8",
+    "blocks.24.attn.sinks": "d637639a453860923ba4ac86323faa0649d7d10d8d6548d2f21975ccf95b9271",
+    "blocks.24.mlp.down_proj.bias": "ea3fd94f790ef890e0e697782d6b337c5015ac5b0662f03b8c2be3551fc0d201",
+    "blocks.24.mlp.down_proj.weight": "07ad47fdf8fd477437c13c70c1ea5093725f406d11a9cdcaa876b8f8efbb576c",
+    "blocks.24.mlp.gate_proj.bias": "bba9187c592fde2688817e54b37b23c54b7283984a48ad4603317ffdcbcc3b9b",
+    "blocks.24.mlp.gate_proj.weight": "819dfa3df1e815dada1981bb02cd8439881df0c8779490810a23d4f6f0088c37",
+    "blocks.24.mlp.up_proj.bias": "19d1632fdc0b96ab6014b06c15d9e263dd30fbd8096569e4747bc566e839b086",
+    "blocks.24.mlp.up_proj.weight": "4b80044b4daaf2fa5cb98f4f78ba5aa89f9e7d8d587f1cbd07d87108b6529123",
+    "blocks.24.norm1.weight": "ec494e7aa665357759ba90bb162e577bfc96daf96fe3a9ebd04d7bbb79120ff6",
+    "blocks.24.norm2.weight": "62b933908279401aacaec568a68543a3cd7106a6dd006008147b079b62ff9e45",
+    "blocks.25.attn.proj.bias": "edbc120231006bb9d8002b734a5e6a769f9aec259aa4c456bf6f3931f383b760",
+    "blocks.25.attn.proj.weight": "951a23eac686b5722edecc3af6303b7b9ede0efa1cc8b945c8292ae889d205be",
+    "blocks.25.attn.qkv.bias": "b0355a10e8f8156b9f296e80186fea8dc9c35c93865138e7e5a534dd4ef6ae6f",
+    "blocks.25.attn.qkv.weight": "5107ae86fee64653d6f9ef993789694c3ad74d7d974db6e730967b089c03c85b",
+    "blocks.25.attn.sinks": "11a88ff10c5d1d7ad1c15dd2af0aa67a08919dc74f2ce52d9ac8ec7d6f28d54b",
+    "blocks.25.mlp.down_proj.bias": "63ff1493976d5937ca2e85253b022f7b0a6e5ea5344518438dfae78793a446dc",
+    "blocks.25.mlp.down_proj.weight": "468b45a3e662f507661a0c39715cccd75a0217d790014542602d999962884115",
+    "blocks.25.mlp.gate_proj.bias": "c9db4c87c83d36323fc5b1ecc2b65679ff84c3c7dbc7b9889af135065286a4d8",
+    "blocks.25.mlp.gate_proj.weight": "d472f5d7c0b54e336c4232ed1c7b2eb3edead9d3a11af1dbdaa8195a11b76eb5",
+    "blocks.25.mlp.up_proj.bias": "7d9d9601f49855e52d66638d481fc3c6bf7aebfdcfbc1e15cd26194ab98f3b11",
+    "blocks.25.mlp.up_proj.weight": "9ce0afae8e64279979f18141bb3c6d0322e7a368cc06522ce6bc4aed2227f417",
+    "blocks.25.norm1.weight": "a71ab1da1311930caa4360160b7f110cf7e2c362835db02b5e9b35a54affc5d4",
+    "blocks.25.norm2.weight": "9a25a05f9dfa1b6447f77bed1029f102625f2edacc50d9628e75261d68a19f51",
+    "blocks.26.attn.proj.bias": "2da4fee62542d73d7c7049da19c7cbd5607287e6c76337f684bf5d3d62a8fa65",
+    "blocks.26.attn.proj.weight": "b49d646999d31464eb066f8582609c3b3703dcd4922c2f47098a628a217a82f7",
+    "blocks.26.attn.qkv.bias": "c8a4b10e351c2b4b631eff1d21d8c5e5125070030577ba6635da99c45a0d9c49",
+    "blocks.26.attn.qkv.weight": "148257549f46a31b51a455967865ead082fd30c59d777f4118cc1be842950e3e",
+    "blocks.26.attn.sinks": "73ff5325f72013f8e41817d6209efb17cd65fe0c1748e596fc6efe14cf7242dd",
+    "blocks.26.mlp.down_proj.bias": "7ecc18cfefcb70d735b8bcf469e6b2d372ded5d75c7621aa2ff768065576c0e8",
+    "blocks.26.mlp.down_proj.weight": "3f8e08c7931f17d553f291e1434cf4bc12d53a9540a58853a58bed494a45ef9c",
+    "blocks.26.mlp.gate_proj.bias": "0fe0e795831c2ab34a73234e3601250ad4e0737db2b30188edb691b6f4100b79",
+    "blocks.26.mlp.gate_proj.weight": "2095178c915aa176990f804129e41dfd6337d5d234d79ecd517dd5e7013f9663",
+    "blocks.26.mlp.up_proj.bias": "adf7dfde6dac4bc062e761d96dd5fc2181b3478738734a81f1acf291094a0754",
+    "blocks.26.mlp.up_proj.weight": "d0e1cf1b9776e489c213860bb27920336ce666eedbdc6b316e37bf59bd3b0557",
+    "blocks.26.norm1.weight": "8c6b25d8d64a81023400a40a8876579876cb8fadbf2dccb0b95071c8d0088572",
+    "blocks.26.norm2.weight": "5dd0541158c4702f123169122412a1d9c8555601e2ae7afdfe81887973c50045",
+    "blocks.27.attn.proj.bias": "5706ef6d52184b6d5ea93135d647778c3cf75cb86866463d88135ca2a6b17df8",
+    "blocks.27.attn.proj.weight": "99d9b4640b5fe56e3513f3ec39309880a2b0e6243f696b0278bcfe68fcdb630b",
+    "blocks.27.attn.qkv.bias": "cf07f5c31f055b6aacae81d1f09b50519e415b1a4fd689cc83d32d8d974e23ed",
+    "blocks.27.attn.qkv.weight": "00268ae0c2bf892706b541dd5f0707f9932b1b5d53b755b00c0c959cb2be7b05",
+    "blocks.27.mlp.down_proj.bias": "ea2cf1c72d7ddd7c1ade4356e71479c2aebc00ef96b4f5bc14654a564b145093",
+    "blocks.27.mlp.down_proj.weight": "58067514f71b0fa0ee8798442822d6f5a249a61df270c0f285b500c1ae78afde",
+    "blocks.27.mlp.gate_proj.bias": "530e9ce0e89a43a08411c8b5f6ebb2200ddea9377f73b5c670bde3301cfb3683",
+    "blocks.27.mlp.gate_proj.weight": "84209394bc4166c5aa510cbcd61d5ac1d6a6c14fc31ce2712d29f661c7e1b1d2",
+    "blocks.27.mlp.up_proj.bias": "97ed5385931d47b19d90f99e4a246ab4289d910cc094100600e52daac2cae9c9",
+    "blocks.27.mlp.up_proj.weight": "f4b08eb20cd60853a4af9cbcf19f7caeb3772858e73069391bae5b7a884fa70c",
+    "blocks.27.norm1.weight": "b83195388f51b4e1d60b1179362d4121b2fb482a7703fc3bebeed5af6eb8fe0d",
+    "blocks.27.norm2.weight": "2afae7f8b410025a3c1f630b36f1019c0d168315b0d0445ec16caf733588d570",
+    "blocks.3.attn.proj.bias": "549deea1130442dea3f30100eea5dd339aacc391dd517998af52edb628f6808f",
+    "blocks.3.attn.proj.weight": "d800b94db1218179b3c0784561648c7568059322b5b3c7220fd3d54ad675d828",
+    "blocks.3.attn.qkv.bias": "13d5e9d971333b016d3aa27aa08d71b601e1699aba2c21640ae64e9123446dee",
+    "blocks.3.attn.qkv.weight": "aa4c4b1c9fd2e21f5899086466f130baa4ef7b51aa72f6c184f586e1a7c6d221",
+    "blocks.3.attn.sinks": "1e35a435b9bf5db0013308f5ee656ef772114c44318108ae095b6a0d89f7939a",
+    "blocks.3.mlp.down_proj.bias": "4dee502979769f537989d7e70565e8ab26557b9c2d026fdcbf7d10bfd4e81ba8",
+    "blocks.3.mlp.down_proj.weight": "8e6b4e5daf345c039134901ce636c3a9868c93d6e5e001559ed6899d9068c1a6",
+    "blocks.3.mlp.gate_proj.bias": "b36d9a6bc0eb3b4d36b8b9d35628e6a593c1e46c4824580124cccf09ebf8e272",
+    "blocks.3.mlp.gate_proj.weight": "df41c57b0d37ada0e3007bca68f8dc0f2a6842c7c42688ad88a5210736db832e",
+    "blocks.3.mlp.up_proj.bias": "c34a155153550bfebdaed6a45c39bc5367da8acbed582a6b323e42bf82ea53c0",
+    "blocks.3.mlp.up_proj.weight": "d444545f534c9b3f95a80d2b646fa60de81761834fdc4159320d38c0b728f442",
+    "blocks.3.norm1.weight": "9333e3f13b0925c3afc7c495d182806b34089ca62588b3957c2d8febeb7df0e1",
+    "blocks.3.norm2.weight": "571dfcfa73f705150aed0e9b847347838c0ddd7dabac6fe1301e64fced5b8f02",
+    "blocks.4.attn.proj.bias": "a4d3c57fb72a2114167cf0bcc78246905686ad17ace636a16aa1488dea032da0",
+    "blocks.4.attn.proj.weight": "6b6e5539d993092f3d668e3a805883de53abb0db07e67b8c643402d469ddef38",
+    "blocks.4.attn.qkv.bias": "ea1653650190f6d51df963809a8c21594e97f13199a23ff45e29099bdefa6902",
+    "blocks.4.attn.qkv.weight": "ede66db6dc98fd25f9fe4b7d164ce321e2842ea8e3b3768f03eaa2d3a9f76f57",
+    "blocks.4.attn.sinks": "dd1cbf287d710820c96fdb284c127883201aacaf35e4ca02ccfe399bfe2629e9",
+    "blocks.4.mlp.down_proj.bias": "5a9183130c3bb93e75aa2cff7c95325d15245997ad93c06e64327d10c013f097",
+    "blocks.4.mlp.down_proj.weight": "765ff65ad9783c5182677f733fd0721ec8a268b2f678a27dabb4d3d6298da5ee",
+    "blocks.4.mlp.gate_proj.bias": "9dd5d9c960a7657fe65104f76ab58b893c9ca7bcb99994bcc6fa4d2e814060e0",
+    "blocks.4.mlp.gate_proj.weight": "41c4056d9755685fd7291fba63bd442b2ade5f56f69a84a5f676f793a2cadd44",
+    "blocks.4.mlp.up_proj.bias": "ca7b122b6e30a3c401025a30458d882d92288954bc2536f81ef556f93eff03c7",
+    "blocks.4.mlp.up_proj.weight": "ac00e3cd64ca6252a4ca8d6ef4dddb58368b72f309dfbe1f6642e9176d993a96",
+    "blocks.4.norm1.weight": "d26f9dd03c22151f3c6a42633c2413a3c958969a3485724fa52955dfc4e763b8",
+    "blocks.4.norm2.weight": "36d95bb1218599dc8a592bc36aad1173c924a8a2d0e865f0412442dbb7aa1c2d",
+    "blocks.5.attn.proj.bias": "bfee01ba14312af8acb504406f0612d0a3c4b8e7c11ad408b7ec32198595a478",
+    "blocks.5.attn.proj.weight": "b5f81eeecb7dc303fd7aa3181b31f314085374ab9f1f53072946deb9ec6be6f3",
+    "blocks.5.attn.qkv.bias": "149676331ff2ce7736084325289429c9cbc4bbab8e854be71c6122cac9966aff",
+    "blocks.5.attn.qkv.weight": "c14971c073f8c8a83ca3b099e9d786a028a86f6233bc6834a631a9c9eb8bcec6",
+    "blocks.5.attn.sinks": "95138000ee6636efe4ec26a6b6fae9f84b0edadbe870c708a3e084600a0405c9",
+    "blocks.5.mlp.down_proj.bias": "6726cc00f237c5d7c410edbf975fa194d8d98cea5eecf8dbd819da43e8c2a5cc",
+    "blocks.5.mlp.down_proj.weight": "c35b491b620081d5bab58fae46d1ece28ed0e12f8bcdd047fb17984ebfb2c6b4",
+    "blocks.5.mlp.gate_proj.bias": "230fb7434ad05a6b65be7bfb11e401d1c95c6264e4aec1b4928c2211b91e0e93",
+    "blocks.5.mlp.gate_proj.weight": "b508ba976de020d94aa6a2f5d28783a0e0008a3ebb0983953cfb2e659e4526a7",
+    "blocks.5.mlp.up_proj.bias": "573d3cf4a707699b70ec5da4c3d5811717538bfd92bcfc0caaad3266c9bb2896",
+    "blocks.5.mlp.up_proj.weight": "fe43f319084268936f3d888e28bb152ba554a1b33331d0cdaf9c9707a7629fda",
+    "blocks.5.norm1.weight": "126f2ae17451f9d5d32a9b81c4630266933f75fafc33fba21a29b104725094fa",
+    "blocks.5.norm2.weight": "dd08790e2157af068acf4d30f269eca25773a5fd17a2cba75a2a8a9964b39845",
+    "blocks.6.attn.proj.bias": "ca8a8773c23426fa50447f58a5b1079480466e039ed3e26ae4e3e43ad276663e",
+    "blocks.6.attn.proj.weight": "957e4ae654b966abb041a85f2d1e91a69f284fc8257ea5250678dfa626d1ea59",
+    "blocks.6.attn.qkv.bias": "339505c814d13cebd246095aaf46ff5ec017cdfae7f32c5353e817a80cdf7377",
+    "blocks.6.attn.qkv.weight": "e5344aadb6472872c1c4c0d5f14feacfe377c32e3440e2e942fd5ace0d723b93",
+    "blocks.6.attn.sinks": "4ec1ce43b67e546a1a08a327e33d2e73f4db818b00177eefc68aa219ebbfdfbf",
+    "blocks.6.mlp.down_proj.bias": "90d2cfd870e9c74f5f0ff7cbce3b84e43e0636025d3c8ddcf78ea2aeaf8de54b",
+    "blocks.6.mlp.down_proj.weight": "191e4252688ad5e14750b792261aecd33546121770a5700ae18d8b963092d006",
+    "blocks.6.mlp.gate_proj.bias": "0054f6bcb6317e37c0bc7650cfb951070ec9af787a6b6be47e2ebcdd3a7204ef",
+    "blocks.6.mlp.gate_proj.weight": "d801a2054b78b196ccc3cfe59ef58dce30b9e636c8731b2706adb819ac14e106",
+    "blocks.6.mlp.up_proj.bias": "25c6e889eea6a21fbee83b51e8e3a0c79f0f7562a0790d816b77619a93469955",
+    "blocks.6.mlp.up_proj.weight": "f3da9d971570b04b91b535a7e08c5d87783b0d59d7403eb350c7ba972ff34ca6",
+    "blocks.6.norm1.weight": "744523028f02f45d9c8025a8d010e4f714ed0dfc6fc6c12349b32c6384f104ee",
+    "blocks.6.norm2.weight": "e40658b34db39650a119e4870cbfa9cf336635bd25ea5d8af86ffce749707c8d",
+    "blocks.7.attn.proj.bias": "cd734c6ad995a6512312344114aebff06f98eb400a772b1b6d3299e9e70262c9",
+    "blocks.7.attn.proj.weight": "0a4369aea59a3d6df6f4200522ee5f1ed4daefcd0be5f133933f3236208cfdae",
+    "blocks.7.attn.qkv.bias": "8b81d62d383578627dcb4b4362f0fe79f9a3ade2c7c669762eb6c2b8c42c8251",
+    "blocks.7.attn.qkv.weight": "0f45a206ad85dacfdc3681525393704e7ba8586bfcf1ae80c1816473a2679cf4",
+    "blocks.7.attn.sinks": "f33dfa44019863e1fe69724171d5ab318a1b77e3057494197fc51ab5cbf7288c",
+    "blocks.7.mlp.down_proj.bias": "28d050bd180bca8ef01a363d3e2d0627c0f40cd3d9dfd5b9ed2bd3cb4b154662",
+    "blocks.7.mlp.down_proj.weight": "7b4d26ede8fa3d6a81c59a742e74947dcb2060f312daebf186cdf320bf2816f2",
+    "blocks.7.mlp.gate_proj.bias": "7a76ce9f5d75af3f90a01f52c536dabbfc948eee6f154ecf3e1290c685e19ae7",
+    "blocks.7.mlp.gate_proj.weight": "6bff7a1c1560632ed8345c438adf21a1bd12f17a61febd9b25d032873de50ccc",
+    "blocks.7.mlp.up_proj.bias": "78eda8efaedc4cbadccec681b06d70ed6b924629205cba9ef016fd976798de1c",
+    "blocks.7.mlp.up_proj.weight": "ca52e3af533deb56d0641b0a0780aee4f9c7730fa40102ff3b349a2a2bcdc88d",
+    "blocks.7.norm1.weight": "32dc2e4102f76ddfd6f12c5dbf15b0f586e9c4b6227b86dbf7b6c8a9eadd988c",
+    "blocks.7.norm2.weight": "5da2e43714d414d5445e7b3efd918bc682d4ff2422887c7f91a8ea384f9bdba7",
+    "blocks.8.attn.proj.bias": "c763a651a9a1530152aff5227702ab3ece7a540c28fd237c7ffa8ce2b1e2f9cc",
+    "blocks.8.attn.proj.weight": "15fb5a0afcc3f25ae061f35fa13d879663738294ca135785ed7c93c589affb55",
+    "blocks.8.attn.qkv.bias": "e8dfcf1b9c7c95e72d2897d18576d61782d19a00705082e2772c2cf6b75ec036",
+    "blocks.8.attn.qkv.weight": "e96da8c934de7e9b5ea2f34e192bf397d20e59280baf0c5138bbaff823d4cdf6",
+    "blocks.8.attn.sinks": "46e969d2834aaa9d37de85091606a0c234661d7cbfa571fea1f3195eeac56da8",
+    "blocks.8.mlp.down_proj.bias": "c37553345b310f558ded83386417ac6cd595ac437aecea67414808f8c05d012f",
+    "blocks.8.mlp.down_proj.weight": "b829d03e7b12769dd98469de53719a666afdb22293c8cbeb137b1b243c9ea0b4",
+    "blocks.8.mlp.gate_proj.bias": "b4d89154dd1c37b53899f77c48849a6eec21f73eb29cc812e7a4c1a06430a556",
+    "blocks.8.mlp.gate_proj.weight": "cb0f516371364d5dcc2ae6e6d7dcbfec461a7b848c2036c914a92d5156aa15b8",
+    "blocks.8.mlp.up_proj.bias": "8dab2c28bce34acbb5091983f6699f42fc23f82d0dee7787e178725e7d470aba",
+    "blocks.8.mlp.up_proj.weight": "b3ee78fc549e98bb90649e8bb3f2405281e43780d4258787a1a52ef2cb474f2d",
+    "blocks.8.norm1.weight": "6ac055dd4d5b93e7c500a71991c7137459a88419df1dad2d238ba17ac50c48a5",
+    "blocks.8.norm2.weight": "dd1fc7d04d8c43f3322a3ae5c2177eb481e138e3134239cb543dc78bebd32181",
+    "blocks.9.attn.proj.bias": "6e630b6cc075dc5fcf0466f9e2f4e57d99a09a93b4fe1f84018182e6fc7a4f5a",
+    "blocks.9.attn.proj.weight": "420258cf68900e28af7d760b08c05229cb9617ce7888642ab2aad757e77a0cb0",
+    "blocks.9.attn.qkv.bias": "8415406ecab8f311e3bc9ff07eb4364d1aed6a04aeb832886e376c7261ecb3cb",
+    "blocks.9.attn.qkv.weight": "86629946a1080d802e0159b0018b890afef4304a9075d06e56d25fe31778e0d8",
+    "blocks.9.mlp.down_proj.bias": "50a3ba680d9f50d65376c250d87a816a59249d72cf9ee88b4c9253582170d211",
+    "blocks.9.mlp.down_proj.weight": "73efed8f6ddee4e651a74330248f83bd40d06daced7d657b52482039e1723072",
+    "blocks.9.mlp.gate_proj.bias": "246968d0135d78f8337f1f6b9c31ad6fc9112536c624671056965c6715b213a5",
+    "blocks.9.mlp.gate_proj.weight": "1f1bd2d4ce4eb6bbd7d634510742aeeba789d34c844d60f97e5889ef2f6afe4d",
+    "blocks.9.mlp.up_proj.bias": "0aa31809a670f638050b2615cc0042654b51e4bc667a1a9100c9b774258f223c",
+    "blocks.9.mlp.up_proj.weight": "788d6bf48eeee5c81a601a99baa94844a67f4962bee236ae8ea6160a1f4973a9",
+    "blocks.9.norm1.weight": "5b8e68729c1ed6b7580ddd50d3820043214de5fff65743510666ab0e6703226c",
+    "blocks.9.norm2.weight": "136c0c50dc59914a14fa042010de55d6c0224ea3436c504dbbd225dba3e5df13",
+    "merger.ln_q.weight": "1086dd19074bc0572b11c820ab12eee9e17ec66fc7d84b14a422ab72a34351d4",
+    "merger.mlp.0.weight": "19fac6d5a8b7c259a2fb911e8aee2060ce17a8bedbee8bbe0e03d0746fe59e9f",
+    "merger.mlp.2.weight": "b97e643633d95df1ed5c46adb9f4fa5d1ecdc7ad35b0f8de370af8b44b2050e9",
+    "patch_embed.proj.weight": "d8d44e93ef6ff896ba8fab0d485d092e67f49c8815388fc307e29e9b578f81b1"
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:140430ae858b2ae46c062bd141ba1054d10cb1e9a2e38eb138daa0a1269b4fd0
+size 1478199144

modeling_mimo_vision.py

@@ -0,0 +1,399 @@
+# coding=utf-8
+#
+# Copyright 2026 Xiaomi Corporation.
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Vision classes extracted from XiaomiMiMo/MiMo-V2.6-Pro-RL
+# b61f21852680ee02a29e946e18cf8697f4844061. Flash uses identical vision source.
+# No text/audio model is instantiated or imported by this standalone module.
+# The three absent merger biases are omitted, matching zero-bias HF completion
+# and SGLang's LayerNorm merger. Attention/block arithmetic is unchanged.
+import torch
+from torch import nn
+from torch.nn import functional as F
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.activations import ACT2FN
+
+
+class MiMoVisionConfig(PretrainedConfig):
+    model_type = "mimo26_vision"
+
+    def __init__(self, **kwargs):
+        defaults = dict(depth=28, hidden_size=1280, intermediate_size=4608,
+                        num_heads=32, num_key_value_heads=8, qk_channels=64,
+                        patch_size=16, temporal_patch_size=2, in_chans=3,
+                        spatial_merge_size=2, out_hidden_size=6144,
+                        hidden_act="silu", rms_norm_eps=1e-6,
+                        fullatt_block_indexes=[0, 9, 18, 27], use_sink=True,
+                        visual_token_window_size=64,
+                        vit_window_attn_types=[-1,0,0,0,0,1,1,1,1,-1,0,0,0,0,1,1,1,1,-1,0,0,0,0,1,1,1,1,-1])
+        defaults.update(kwargs)
+        super().__init__(**defaults)
+
+
+def _rotate_half_vision(x: torch.Tensor) -> torch.Tensor:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def _apply_rotary_pos_emb_vision(
+    q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
+) -> tuple[torch.Tensor, torch.Tensor]:
+    orig_q_dtype, orig_k_dtype = q.dtype, k.dtype
+    q, k = q.float(), k.float()
+    cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
+    q_embed = (q * cos) + (_rotate_half_vision(q) * sin)
+    k_embed = (k * cos) + (_rotate_half_vision(k) * sin)
+    return q_embed.to(orig_q_dtype), k_embed.to(orig_k_dtype)
+
+
+class MiMoVisionRotaryEmbedding(nn.Module):
+    def __init__(self, dim: int, theta: float = 10000.0) -> None:
+        super().__init__()
+        inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, seqlen: int) -> torch.Tensor:
+        seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
+        return torch.outer(seq, self.inv_freq)
+
+
+class MiMoVisionPatchEmbed(nn.Module):
+    def __init__(
+        self, patch_size: int = 16, temporal_patch_size: int = 2, in_channels: int = 3, embed_dim: int = 1280
+    ):
+        super().__init__()
+        self.patch_size = patch_size
+        self.temporal_patch_size = temporal_patch_size
+        self.in_channels = in_channels
+        self.embed_dim = embed_dim
+        kernel_size = [temporal_patch_size, patch_size, patch_size]
+        self.proj = nn.Conv3d(in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        target_dtype = self.proj.weight.dtype
+        hidden_states = hidden_states.view(
+            -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size
+        )
+        return self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim)
+
+
+class MiMoVisionSwiGLUMLP(nn.Module):
+    def __init__(self, dim: int, intermediate_dim: int, hidden_act: str = "silu"):
+        super().__init__()
+        self.gate_proj = nn.Linear(dim, intermediate_dim, bias=True)
+        self.up_proj = nn.Linear(dim, intermediate_dim, bias=True)
+        self.down_proj = nn.Linear(intermediate_dim, dim, bias=True)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class MiMoVisionAttention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        num_kv_heads: int | None = None,
+        head_dim: int | None = None,
+        use_sinks: bool = False,
+        window_size: int = -1,
+    ):
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
+        self.head_dim = head_dim if head_dim is not None else dim // num_heads
+        self.num_kv_groups = self.num_heads // self.num_kv_heads
+        self.scaling = self.head_dim**-0.5
+        self.window_size = window_size
+
+        qkv_dim = (self.num_heads + 2 * self.num_kv_heads) * self.head_dim
+        self.qkv = nn.Linear(dim, qkv_dim, bias=True)
+        self.proj = nn.Linear(self.num_heads * self.head_dim, dim, bias=True)
+        self.sinks = nn.Parameter(torch.zeros(self.num_heads)) if use_sinks else None
+
+    def _build_window_mask(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor | None:
+        if self.window_size <= 0:
+            return None
+        row_idx = torch.arange(seq_len, device=device).unsqueeze(1)
+        col_idx = torch.arange(seq_len, device=device).unsqueeze(0)
+        mask = torch.zeros(seq_len, seq_len, device=device, dtype=dtype)
+        mask = mask.masked_fill((row_idx - col_idx).abs() > self.window_size, float("-inf"))
+        return mask
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        full_attn: bool = False,
+    ) -> torch.Tensor:
+        seq_len = hidden_states.shape[0]
+        qkv = self.qkv(hidden_states)
+
+        q_dim = self.num_heads * self.head_dim
+        kv_dim = self.num_kv_heads * self.head_dim
+        q = qkv[:, :q_dim].view(seq_len, self.num_heads, self.head_dim)
+        k = qkv[:, q_dim : q_dim + kv_dim].view(seq_len, self.num_kv_heads, self.head_dim)
+        v = qkv[:, q_dim + kv_dim :].view(seq_len, self.num_kv_heads, self.head_dim)
+
+        cos, sin = position_embeddings
+        q, k = _apply_rotary_pos_emb_vision(q, k, cos, sin)
+
+        lengths = cu_seqlens[1:] - cu_seqlens[:-1]
+        q_chunks = torch.split(q, lengths.tolist(), dim=0)
+        k_chunks = torch.split(k, lengths.tolist(), dim=0)
+        v_chunks = torch.split(v, lengths.tolist(), dim=0)
+
+        outputs = []
+        for q_c, k_c, v_c in zip(q_chunks, k_chunks, v_chunks):
+            q_c = q_c.unsqueeze(0).transpose(1, 2)
+            k_c = k_c.unsqueeze(0).transpose(1, 2)
+            v_c = v_c.unsqueeze(0).transpose(1, 2)
+
+            if self.num_kv_groups > 1:
+                k_c = k_c.repeat_interleave(self.num_kv_groups, dim=1)
+                v_c = v_c.repeat_interleave(self.num_kv_groups, dim=1)
+
+            attn_mask = None
+            if not full_attn:
+                attn_mask = self._build_window_mask(q_c.shape[2], q_c.device, q_c.dtype)
+
+            if self.sinks is not None:
+                sink_bias = torch.zeros(
+                    1, self.num_heads, q_c.shape[2], k_c.shape[2], device=q_c.device, dtype=q_c.dtype
+                )
+                sink_bias[..., 0] = self.sinks.view(1, self.num_heads, 1)
+                attn_mask = sink_bias if attn_mask is None else attn_mask + sink_bias
+
+            attn_out = F.scaled_dot_product_attention(q_c, k_c, v_c, attn_mask=attn_mask, scale=self.scaling)
+            outputs.append(attn_out.squeeze(0).transpose(0, 1))
+
+        attn_output = torch.cat(outputs, dim=0)
+        attn_output = attn_output.reshape(seq_len, -1)
+        return self.proj(attn_output)
+
+
+class MiMoVisionBlock(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        intermediate_dim: int,
+        num_heads: int,
+        num_kv_heads: int | None = None,
+        head_dim: int | None = None,
+        hidden_act: str = "silu",
+        rms_norm_eps: float = 1e-6,
+        use_sinks: bool = False,
+        window_size: int = -1,
+    ):
+        super().__init__()
+        self.norm1 = nn.RMSNorm(dim, eps=rms_norm_eps)
+        self.norm2 = nn.RMSNorm(dim, eps=rms_norm_eps)
+        self.attn = MiMoVisionAttention(
+            dim=dim, num_heads=num_heads, num_kv_heads=num_kv_heads, head_dim=head_dim,
+            use_sinks=use_sinks, window_size=window_size,
+        )
+        self.mlp = MiMoVisionSwiGLUMLP(dim=dim, intermediate_dim=intermediate_dim, hidden_act=hidden_act)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        full_attn: bool = False,
+    ) -> torch.Tensor:
+        hidden_states = hidden_states + self.attn(
+            self.norm1(hidden_states), cu_seqlens=cu_seqlens,
+            position_embeddings=position_embeddings, full_attn=full_attn,
+        )
+        hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
+        return hidden_states
+
+
+class MiMoVisionPatchMerger(nn.Module):
+    def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2):
+        super().__init__()
+        self.hidden_size = context_dim * (spatial_merge_size**2)
+        self.ln_q = nn.LayerNorm(context_dim, eps=1e-6, bias=False)
+        self.mlp = nn.Sequential(
+            nn.Linear(self.hidden_size, self.hidden_size, bias=False),
+            nn.GELU(),
+            nn.Linear(self.hidden_size, dim, bias=False),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.mlp(self.ln_q(x).view(-1, self.hidden_size))
+
+
+class MiMoVisionTransformer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        hidden_size = config.hidden_size
+        depth = config.depth
+        num_heads = config.num_heads
+        num_kv_heads = getattr(config, "num_key_value_heads", num_heads)
+        head_dim = getattr(config, "qk_channels", 64)
+        spatial_merge_size = getattr(config, "spatial_merge_size", 2)
+        rms_norm_eps = getattr(config, "rms_norm_eps", 1e-6)
+        self.fullatt_block_indexes = getattr(config, "fullatt_block_indexes", [])
+        use_sink = getattr(config, "use_sink", False)
+        visual_token_window_size = getattr(config, "visual_token_window_size", -1)
+        self.vit_window_attn_types = getattr(config, "vit_window_attn_types", None) or [-1] * depth
+
+        self.spatial_merge_size = spatial_merge_size
+        self.spatial_merge_unit = spatial_merge_size * spatial_merge_size
+
+        self.patch_embed = MiMoVisionPatchEmbed(
+            patch_size=config.patch_size,
+            temporal_patch_size=config.temporal_patch_size,
+            in_channels=getattr(config, "in_channels", None) or getattr(config, "in_chans", 3),
+            embed_dim=hidden_size,
+        )
+
+        self.rotary_pos_emb = MiMoVisionRotaryEmbedding(head_dim // 2)
+
+        self.blocks = nn.ModuleList(
+            [
+                MiMoVisionBlock(
+                    dim=hidden_size,
+                    intermediate_dim=config.intermediate_size,
+                    num_heads=num_heads,
+                    num_kv_heads=num_kv_heads,
+                    head_dim=head_dim,
+                    hidden_act=config.hidden_act,
+                    rms_norm_eps=rms_norm_eps,
+                    use_sinks=use_sink and (i not in self.fullatt_block_indexes),
+                    window_size=visual_token_window_size,
+                )
+                for i in range(depth)
+            ]
+        )
+
+        self.merger = MiMoVisionPatchMerger(
+            dim=config.out_hidden_size,
+            context_dim=hidden_size,
+            spatial_merge_size=spatial_merge_size,
+        )
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return self.patch_embed.proj.weight.dtype
+
+    def apply_index(self, tensor: torch.Tensor, index: torch.Tensor) -> torch.Tensor:
+        tensor = tensor.unflatten(0, (-1, self.spatial_merge_unit))
+        tensor = tensor[index]
+        return tensor.flatten(0, 1)
+
+    def get_window_index_1d(self, grid_thw: torch.Tensor, col: bool = True) -> torch.Tensor:
+        window_index = []
+        window_index_id = 0
+        for grid_t, grid_h, grid_w in grid_thw:
+            llm_grid_h = grid_h // self.spatial_merge_size
+            llm_grid_w = grid_w // self.spatial_merge_size
+            index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
+            index_new = index.transpose(1, 2).reshape(-1) if col else index.reshape(-1)
+            window_index.append(index_new + window_index_id)
+            window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
+        return torch.cat(window_index, dim=0)
+
+    def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
+        pos_ids = []
+        for t, h, w in grid_thw:
+            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
+            hpos_ids = hpos_ids.reshape(
+                h // self.spatial_merge_size, self.spatial_merge_size,
+                w // self.spatial_merge_size, self.spatial_merge_size,
+            )
+            hpos_ids = hpos_ids.permute(0, 2, 1, 3).flatten()
+
+            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
+            wpos_ids = wpos_ids.reshape(
+                h // self.spatial_merge_size, self.spatial_merge_size,
+                w // self.spatial_merge_size, self.spatial_merge_size,
+            )
+            wpos_ids = wpos_ids.permute(0, 2, 1, 3).flatten()
+
+            pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
+        pos_ids = torch.cat(pos_ids, dim=0)
+        max_grid_size = grid_thw[:, 1:].max()
+        rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
+        return rotary_pos_emb_full[pos_ids].flatten(1)
+
+    def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
+        x = pixel_values.to(device=self.patch_embed.proj.weight.device, dtype=self.dtype)
+        x = self.patch_embed(x)
+
+        rotary_emb = self.rot_pos_emb(grid_thw)
+        rotary_emb = rotary_emb.to(device=x.device)
+        emb = torch.cat((rotary_emb, rotary_emb), dim=-1)
+
+        window_index_1d_col = self.get_window_index_1d(grid_thw, col=True).to(device=x.device)
+        reverse_window_index_1d_col = torch.argsort(window_index_1d_col).to(device=x.device)
+
+        row_based_embeddings = (emb.cos(), emb.sin())
+        col_emb = self.apply_index(emb, window_index_1d_col)
+        col_based_embeddings = (col_emb.cos(), col_emb.sin())
+
+        cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(
+            dim=0, dtype=torch.int32
+        )
+        cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0).to(device=x.device)
+
+        for i, blk in enumerate(self.blocks):
+            window_attn_type = self.vit_window_attn_types[i]
+
+            if window_attn_type == 1 and (i == 0 or self.vit_window_attn_types[i - 1] != 1):
+                x = self.apply_index(x, window_index_1d_col)
+
+            if i > 0 and window_attn_type != 1 and self.vit_window_attn_types[i - 1] == 1:
+                x = self.apply_index(x, reverse_window_index_1d_col)
+
+            position_embeddings = col_based_embeddings if window_attn_type == 1 else row_based_embeddings
+            full_attn = i in self.fullatt_block_indexes
+            x = blk(x, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, full_attn=full_attn)
+
+        return self.merger(x)
+
+
+
+
+class MiMoVisionModel(PreTrainedModel):
+    config_class = MiMoVisionConfig
+    base_model_prefix = "visual"
+    main_input_name = "pixel_values"
+    _no_split_modules = ["MiMoVisionBlock"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.visual = MiMoVisionTransformer(config)
+        self.post_init()
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, MiMoVisionRotaryEmbedding):
+            # HF materializes nonpersistent meta buffers without running their
+            # constructors again. Rebuild frequencies during missing-key init.
+            dim = module.inv_freq.numel() * 2
+            frequencies = 1.0 / (10000.0 ** (torch.arange(0, dim, 2, dtype=torch.float32, device=module.inv_freq.device) / dim))
+            module.inv_freq.copy_(frequencies)
+
+    def forward(self, pixel_values, grid_thw):
+        # Parent-compatible output after LayerNorm, 2x2 merge and projector.
+        return self.visual(pixel_values, grid_thw)

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "min_pixels": 8192,
+  "max_pixels": 8388608,
+  "patch_size": 16,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "processor_class": "AutoImageProcessor"
+}