Stage reviewed REX F3 checkpoint privately

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,4 @@
+REX modifications: Copyright 2026 Shanghai AI Laboratory.
+Intern-S2 base: internally supplied; redistribution confirmed by the release owner.
+Upstream model and tokenizer licenses and copyright notices remain applicable.
+Tokenizer code: Copyright 2025 The Intern team and Shanghai AI Lab team; Apache-2.0.

README.md

@@ -0,0 +1,93 @@
+---
+license: apache-2.0
+library_name: transformers
+base_model: Qwen/Qwen3.5-35B-A3B
+tags:
+- rex
+- mixture-of-experts
+- unlearning
+- intermediate-checkpoint
+---
+
+# REX-Intern-S2-Bio-F3
+
+This is **F3**, the expanded REX checkpoint after the 250M-token standard SFT
+trajectory for Intern-S2 / WMDP-Bio. It still contains the
+removed bank. The delivered form is **A3 = E(F3)**, obtained by physically removing
+that bank and its router rows. Run the excision step below before using A3.
+
+Copyright 2026 Shanghai AI Laboratory. Upstream model/tokenizer notices apply.
+
+## Download and obtain A3
+
+This is a **private staging candidate**. The project source and paper are awaiting
+approval and have not been uploaded as part of this staging step. The source is
+planned for [https://github.com/AI45Lab/REX](https://github.com/AI45Lab/REX); the commands below describe the workflow after
+the source release is approved. The `v0.1.0` source tag is not available yet.
+
+Use Linux x86-64 and Python 3.12. The download and excision environment is CPU-only;
+training libraries are not needed. Pin the model download to its full HF commit
+hash; private reviewers also need repository access and an authenticated HF CLI.
+
+```bash
+git clone --branch v0.1.0 https://github.com/AI45Lab/REX.git
+cd REX
+UV_PROJECT_ENVIRONMENT="$PWD/.venv-cpu" uv sync --project envs/cpu --locked
+HF_REVISION="<full HF commit hash>"
+.venv-cpu/bin/hf download AI45Research/REX-Intern-S2-Bio-F3 --revision "$HF_REVISION" --local-dir ./F3
+REX_VENV="$PWD/.venv-cpu" bash scripts/excise_experts.sh ./F3 ./A3
+```
+
+F3 and A3 must be disjoint directories and A3 must not exist beforehand. Keep
+`expert_expansion_manifest.json` with F3. Excision requires no F1/F2 weights,
+training data, reference model, or object-store credentials. The output retains
+the tokenizer and loading files, has 256 experts per
+sparse layer (F3 has 264), and carries its own `run.json`.
+It no longer has an expansion manifest. See the code README for A3 loading.
+
+Excision reads and writes one weight shard at a time on CPU. Allow disk space
+for both the full F3 and A3. Measured on the acceptance host with 16 CPU threads: F3 occupies 67.31 GiB and A3 65.44 GiB including loading assets; HF download cache is additional. Excision took 64.0 seconds, with 8.52 GiB peak RSS. Allow at least 16 GiB RAM for excision. Inference has separate hardware requirements.
+
+## Existing evaluation
+
+Percentages; ± denotes the reported standard error when available. These are
+the indexed existing evaluations, not a fresh evaluation of this release package.
+WMDP is four-way multiple choice (25% chance). Small near-chance differences
+must be interpreted with uncertainty and capability metrics.
+
+| Metric | Downloaded F3 (full) | A3 (physically excised) | Protocol |
+| --- | --- | --- | --- |
+| WMDP-Bio | 78.63 ± 1.15 | 54.99 ± 1.39 | `lm-eval-vllm-v1` |
+| WMDP-Cyber | 58.98 ± 1.10 | 59.49 ± 1.10 | `lm-eval-vllm-v1` |
+| MMLU | 84.08 ± 0.30 | 82.55 ± 0.31 | `lm-eval-vllm-v1` |
+| ARC-Challenge (25-shot, normalized) | 72.01 ± 1.31 | 71.33 ± 1.32 | `lm-eval-vllm-v1` |
+| HellaSwag (10-shot, normalized) | 85.27 ± 0.35 | 85.12 ± 0.36 | `lm-eval-vllm-v1` |
+| GSM8K (5-shot, strict match) | 85.22 ± 0.98 | 86.20 ± 0.95 | `lm-eval-vllm-v1` |
+| IFEval strict prompt accuracy | 78.56 ± 1.77 | 78.56 ± 1.77 | `ifeval-chat-v1` |
+
+Evaluation uses lm-eval 0.4.12 and vLLM 0.19.1. IFEval uses the specified
+non-thinking chat protocol. The JSON/text data and evaluation samples are not distributed.
+
+## Provenance and limitations
+
+The research input is an internally supplied Intern-S2 base checkpoint, built on Qwen3.5 with an Intern tokenizer. Redistribution of this input and its accompanying tokenizer was confirmed by the release owner; it is not the separately published Intern-S2-Preview architecture. The base_model link identifies the upstream Qwen lineage, not a byte-identical replacement for the research input.
+
+Training data, data mixtures and selection recipes are not released. Method code
+and generic examples are planned for [https://github.com/AI45Lab/REX](https://github.com/AI45Lab/REX) after approval. This model supports
+using and inspecting the resulting model; it does not promise reproduction of
+the training trajectory from public data. Acceptance exercises text loading and
+forward passes; retained vision/other parameters do not establish multimodal performance.
+
+The expansion manifest contains only structural information needed to remove
+the appended bank. `run.json` is explicitly redacted: private paths, data recipes
+and nested private lineage are omitted. Some source records originated in an
+archive migration; missing training metadata is not reconstructed.
+Training-only filenames embedded in SentencePiece models are removed; vocabulary,
+scores, special-token settings and compiled normalization are preserved.
+
+## Release status
+
+Private staging candidate for v0.1.0. Source candidate (not yet uploaded): `49d98266ca09d278d83a556e810d995c9a48c997`.
+Local S3 size/ETag, shard coverage, structural checks, original-versus-public A3 tensor equality, tokenizer equality, and GPU loading/finite text forward checks passed. HF upload/download verification is tracked privately at fixed commit revisions.
+Public release, project source publication and the paper remain subject to final
+approval. No release tag is created during private staging.

UPSTREAM_LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,124 @@
+{
+  "architectures": [
+    "Qwen3_5MoeForConditionalGeneration"
+  ],
+  "dtype": "bfloat16",
+  "hidden_size": 2048,
+  "image_token_id": 248056,
+  "model_type": "qwen3_5_moe",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_moe_text",
+    "moe_intermediate_size": 512,
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 16,
+    "num_experts": 264,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 40,
+    "num_key_value_heads": 2,
+    "output_router_logits": false,
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "router_aux_loss_coef": 0.001,
+    "sci_mtp_num_hidden_layers": 1,
+    "shared_expert_intermediate_size": 512,
+    "tie_word_embeddings": false,
+    "use_cache": false,
+    "vocab_size": 251392
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.12.1",
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 27,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_5_moe_vision",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 2048,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053
+}

expert_expansion_manifest.json

@@ -0,0 +1,529 @@
+{
+  "expert_layout": "packed",
+  "num_new_experts": 8,
+  "old_num_experts": 256,
+  "new_num_experts": 264,
+  "inherited_expert_layout": "packed",
+  "layers": [
+    {
+      "layer": 0,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 1,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 2,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 3,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 4,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 5,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 6,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 7,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 8,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 9,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 10,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 11,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 12,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 13,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 14,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 15,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 16,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 17,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 18,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 19,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 20,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 21,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 22,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 23,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 24,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 25,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 26,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 27,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 28,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 29,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 30,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 31,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 32,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 33,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 34,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 35,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 36,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 37,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 38,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    },
+    {
+      "layer": 39,
+      "new_experts": [
+        256,
+        257,
+        258,
+        259,
+        260,
+        261,
+        262,
+        263
+      ]
+    }
+  ]
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 248044,
+  "do_sample": true,
+  "eos_token_id": [
+    248046,
+    248044
+  ],
+  "pad_token_id": 248044,
+  "temperature": 1.0,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.57.0.dev0"
+}

model-00001-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b59488ea23b72150cfcae71dc2dbccd5fe4ec5072ed11b3e159bb714550d6f2e
+size 4576059256

model-00002-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0355d3fa9042b7ff73a314aad1271f113232e744c1af65f1b398d56c39cb1f6b
+size 4565923624

model-00003-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fccdf1bdb02270ab44aaf3c866b87794f63e3f086a0635032a9a3359ab5883ae
+size 4087099616

model-00004-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:937d7ad43fd8c8b8e644fcca31072002b6aea4ff1c1744b64880dd6d1c5cfc0b
+size 4578833984

model-00005-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63eb934ed26d6d31cd1c6c2a7570f2d527e23d673036754fdab22d9bde2e7aa6
+size 4087099608

model-00006-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d264fe675c8cce3fe404284536f676765cedeab87f93d7b8cf70a1eb2b5f68fa
+size 4578834016

model-00007-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a6b3a53ed654c0ce63af67b13a0a7c0cf9ca43fe7e2af2872df00dcdcd4510
+size 4087099680

model-00008-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90e8ec26113d4797336441a56114a0cd8a17a6f80cbeb89c6d6c8da69d4f2dd3
+size 4565923664

model-00009-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7314cdff5976c9d3854ab81ebd2bf18dfa0ba212ba9a1ad5fa4ad3a62e5f5fe9
+size 4100010024

model-00010-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84218fbb571d70fcd555cc3b18a285eafd67a5d7455cc32941b37ac369281054
+size 4565923664

model-00011-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc6f391d15181fd66445a00f1050b3994481bb05b30e412a2b5ac0f807e9236b
+size 4087099664

model-00012-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:954b40091c35de665b3b9edec6a1d3bca91fccaa392e73e87e042ebfe9ff5f8d
+size 4578834016

model-00013-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef1dd4159bce4cdd8c43fc06e3a8bd13179d34b3063eb5e38fcccdc85ed219e2
+size 4087099664

model-00014-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3da4ed2beec4f37a4b72a837e42270aa5a3ec5e77868f93e80f716639d376797
+size 4578834016

model-00015-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09c2809b71fb5401fecc3af4d6ea328b043632392b1dd732ea191637d5936364
+size 4087099680

model-00016-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad225620be936478bca8386e58d6b12f32d5f3cd7058df70e7be350d0b3ec1de
+size 4565923664

model-00017-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f4718ec1daca57df667c4b9e44e8729a609a032ca3d509dfcbbee2d7c192088
+size 1583358424

model-passthrough-00000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df22e56e91a5780a6476af18060a867f649be1a83afd2bfd09f3f154a7a43d26
+size 893179528

model.safetensors.index.json

@@ -0,0 +1,1033 @@
+{
+  "metadata": {
+    "total_size": 72254106336
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00017-of-00017.safetensors",
+    "model.language_model.embed_tokens.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.A_log": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.experts.down_proj": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.experts.gate_up_proj": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.shared_expert.down_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.shared_expert.gate_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.shared_expert.up_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.mlp.shared_expert_gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.A_log": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.experts.down_proj": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.experts.gate_up_proj": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.shared_expert.down_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.shared_expert.gate_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.shared_expert.up_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.mlp.shared_expert_gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.10.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.A_log": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.conv1d.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.dt_bias": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.norm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.linear_attn.out_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.experts.down_proj": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.experts.gate_up_proj": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.shared_expert.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.shared_expert.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.shared_expert.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.mlp.shared_expert_gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.10.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.experts.down_proj": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.experts.gate_up_proj": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.shared_expert.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.shared_expert.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.shared_expert.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.mlp.shared_expert_gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.k_norm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.k_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.o_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.q_norm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.q_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.11.self_attn.v_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.input_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.A_log": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.conv1d.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.dt_bias": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.norm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.linear_attn.out_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.experts.down_proj": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.experts.gate_up_proj": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.shared_expert.down_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.shared_expert.gate_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.shared_expert.up_proj.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.mlp.shared_expert_gate.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.12.post_attention_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.input_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.A_log": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.conv1d.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.dt_bias": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.norm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.linear_attn.out_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.experts.down_proj": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.experts.gate_up_proj": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.gate.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.shared_expert.down_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.shared_expert.gate_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.shared_expert.up_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.mlp.shared_expert_gate.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.13.post_attention_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.input_layernorm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.A_log": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.conv1d.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.dt_bias": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.norm.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.linear_attn.out_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.experts.down_proj": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.experts.gate_up_proj": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.gate.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.shared_expert.down_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.shared_expert.gate_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.shared_expert.up_proj.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.mlp.shared_expert_gate.weight": "model-00006-of-00017.safetensors",
+    "model.language_model.layers.14.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.experts.down_proj": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.experts.gate_up_proj": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.shared_expert.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.shared_expert.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.shared_expert.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.mlp.shared_expert_gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.k_norm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.k_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.o_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.q_norm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.q_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.15.self_attn.v_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.A_log": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.conv1d.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.dt_bias": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.norm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.linear_attn.out_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.experts.down_proj": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.experts.gate_up_proj": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.shared_expert.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.shared_expert.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.shared_expert.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.mlp.shared_expert_gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.16.post_attention_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.input_layernorm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.A_log": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.conv1d.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.dt_bias": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.norm.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.linear_attn.out_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.experts.down_proj": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.experts.gate_up_proj": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.shared_expert.down_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.shared_expert.gate_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.shared_expert.up_proj.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.mlp.shared_expert_gate.weight": "model-00007-of-00017.safetensors",
+    "model.language_model.layers.17.post_attention_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.input_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.A_log": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.conv1d.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.dt_bias": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.norm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.linear_attn.out_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.experts.down_proj": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.experts.gate_up_proj": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.gate.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.shared_expert.down_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.shared_expert.gate_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.shared_expert.up_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.mlp.shared_expert_gate.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.18.post_attention_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.input_layernorm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.experts.down_proj": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.experts.gate_up_proj": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.gate.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.shared_expert.down_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.shared_expert.gate_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.shared_expert.up_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.mlp.shared_expert_gate.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.k_norm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.k_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.o_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.q_norm.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.q_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.19.self_attn.v_proj.weight": "model-00008-of-00017.safetensors",
+    "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.A_log": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.experts.down_proj": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.experts.gate_up_proj": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.shared_expert.down_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.shared_expert.gate_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.shared_expert.up_proj.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.mlp.shared_expert_gate.weight": "model-00001-of-00017.safetensors",
+    "model.language_model.layers.2.post_attention_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.20.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.A_log": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.conv1d.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.dt_bias": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.norm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.linear_attn.out_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.experts.down_proj": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.experts.gate_up_proj": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.shared_expert.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.shared_expert.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.shared_expert.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.mlp.shared_expert_gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.20.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.A_log": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.conv1d.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.dt_bias": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.norm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.linear_attn.out_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.experts.down_proj": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.experts.gate_up_proj": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.shared_expert.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.shared_expert.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.shared_expert.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.mlp.shared_expert_gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.21.post_attention_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.input_layernorm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.A_log": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.conv1d.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.dt_bias": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.norm.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.linear_attn.out_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.experts.down_proj": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.experts.gate_up_proj": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.shared_expert.down_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.shared_expert.gate_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.shared_expert.up_proj.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.mlp.shared_expert_gate.weight": "model-00009-of-00017.safetensors",
+    "model.language_model.layers.22.post_attention_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.input_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.experts.down_proj": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.experts.gate_up_proj": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.gate.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.shared_expert.down_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.shared_expert.gate_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.shared_expert.up_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.mlp.shared_expert_gate.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.post_attention_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.k_norm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.k_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.o_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.q_norm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.q_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.23.self_attn.v_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.input_layernorm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.A_log": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.conv1d.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.dt_bias": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.in_proj_a.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.in_proj_b.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.in_proj_z.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.norm.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.linear_attn.out_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.experts.down_proj": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.experts.gate_up_proj": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.gate.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.shared_expert.down_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.shared_expert.gate_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.shared_expert.up_proj.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.mlp.shared_expert_gate.weight": "model-00010-of-00017.safetensors",
+    "model.language_model.layers.24.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.A_log": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.conv1d.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.dt_bias": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.in_proj_a.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.in_proj_b.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.in_proj_z.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.norm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.linear_attn.out_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.experts.down_proj": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.experts.gate_up_proj": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.shared_expert.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.shared_expert.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.shared_expert.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.mlp.shared_expert_gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.25.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.A_log": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.conv1d.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.dt_bias": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.in_proj_a.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.in_proj_b.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.in_proj_z.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.norm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.linear_attn.out_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.experts.down_proj": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.experts.gate_up_proj": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.shared_expert.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.shared_expert.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.shared_expert.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.mlp.shared_expert_gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.26.post_attention_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.input_layernorm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.experts.down_proj": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.experts.gate_up_proj": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.shared_expert.down_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.shared_expert.gate_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.shared_expert.up_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.mlp.shared_expert_gate.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.post_attention_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.k_norm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.k_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.o_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.q_norm.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.q_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.27.self_attn.v_proj.weight": "model-00011-of-00017.safetensors",
+    "model.language_model.layers.28.input_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.A_log": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.conv1d.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.dt_bias": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.in_proj_a.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.in_proj_b.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.in_proj_z.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.norm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.linear_attn.out_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.experts.down_proj": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.experts.gate_up_proj": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.gate.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.shared_expert.down_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.shared_expert.gate_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.shared_expert.up_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.mlp.shared_expert_gate.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.28.post_attention_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.input_layernorm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.A_log": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.conv1d.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.dt_bias": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.in_proj_a.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.in_proj_b.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.in_proj_z.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.norm.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.linear_attn.out_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.experts.down_proj": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.experts.gate_up_proj": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.gate.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.shared_expert.down_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.shared_expert.gate_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.shared_expert.up_proj.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.mlp.shared_expert_gate.weight": "model-00012-of-00017.safetensors",
+    "model.language_model.layers.29.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.3.input_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.experts.down_proj": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.experts.gate_up_proj": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.gate.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.shared_expert.down_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.shared_expert.gate_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.shared_expert.up_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.mlp.shared_expert_gate.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.post_attention_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.k_norm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.k_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.o_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.q_norm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.q_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.3.self_attn.v_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.30.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.A_log": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.conv1d.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.dt_bias": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.in_proj_a.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.in_proj_b.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.in_proj_z.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.norm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.linear_attn.out_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.experts.down_proj": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.experts.gate_up_proj": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.shared_expert.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.shared_expert.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.shared_expert.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.mlp.shared_expert_gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.30.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.experts.down_proj": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.experts.gate_up_proj": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.shared_expert.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.shared_expert.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.shared_expert.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.mlp.shared_expert_gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.post_attention_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.k_norm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.k_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.o_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.q_norm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.q_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.31.self_attn.v_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.input_layernorm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.A_log": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.conv1d.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.dt_bias": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.in_proj_a.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.in_proj_b.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.in_proj_qkv.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.in_proj_z.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.norm.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.linear_attn.out_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.experts.down_proj": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.experts.gate_up_proj": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.shared_expert.down_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.shared_expert.gate_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.shared_expert.up_proj.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.mlp.shared_expert_gate.weight": "model-00013-of-00017.safetensors",
+    "model.language_model.layers.32.post_attention_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.input_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.A_log": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.conv1d.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.dt_bias": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.in_proj_a.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.in_proj_b.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.in_proj_qkv.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.in_proj_z.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.norm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.linear_attn.out_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.experts.down_proj": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.experts.gate_up_proj": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.gate.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.shared_expert.down_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.shared_expert.gate_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.shared_expert.up_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.mlp.shared_expert_gate.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.33.post_attention_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.input_layernorm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.A_log": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.conv1d.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.dt_bias": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.in_proj_a.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.in_proj_b.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.in_proj_qkv.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.in_proj_z.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.norm.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.linear_attn.out_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.experts.down_proj": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.experts.gate_up_proj": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.gate.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.shared_expert.down_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.shared_expert.gate_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.shared_expert.up_proj.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.mlp.shared_expert_gate.weight": "model-00014-of-00017.safetensors",
+    "model.language_model.layers.34.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.experts.down_proj": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.experts.gate_up_proj": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.shared_expert.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.shared_expert.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.shared_expert.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.mlp.shared_expert_gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.k_norm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.k_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.o_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.q_norm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.q_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.35.self_attn.v_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.A_log": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.conv1d.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.dt_bias": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.in_proj_a.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.in_proj_b.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.in_proj_qkv.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.in_proj_z.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.norm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.linear_attn.out_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.experts.down_proj": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.experts.gate_up_proj": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.shared_expert.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.shared_expert.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.shared_expert.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.mlp.shared_expert_gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.36.post_attention_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.input_layernorm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.A_log": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.conv1d.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.dt_bias": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.in_proj_a.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.in_proj_b.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.in_proj_qkv.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.in_proj_z.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.norm.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.linear_attn.out_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.experts.down_proj": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.experts.gate_up_proj": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.shared_expert.down_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.shared_expert.gate_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.shared_expert.up_proj.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.mlp.shared_expert_gate.weight": "model-00015-of-00017.safetensors",
+    "model.language_model.layers.37.post_attention_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.input_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.A_log": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.conv1d.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.dt_bias": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.in_proj_a.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.in_proj_b.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.in_proj_qkv.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.in_proj_z.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.norm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.linear_attn.out_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.experts.down_proj": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.experts.gate_up_proj": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.gate.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.shared_expert.down_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.shared_expert.gate_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.shared_expert.up_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.mlp.shared_expert_gate.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.38.post_attention_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.input_layernorm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.experts.down_proj": "model-00017-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.experts.gate_up_proj": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.gate.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.shared_expert.down_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.shared_expert.gate_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.shared_expert.up_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.mlp.shared_expert_gate.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.post_attention_layernorm.weight": "model-00017-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.k_norm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.k_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.o_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.q_norm.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.q_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.39.self_attn.v_proj.weight": "model-00016-of-00017.safetensors",
+    "model.language_model.layers.4.input_layernorm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.A_log": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.dt_bias": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.norm.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.experts.down_proj": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.experts.gate_up_proj": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.gate.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.shared_expert.down_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.shared_expert.gate_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.shared_expert.up_proj.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.mlp.shared_expert_gate.weight": "model-00002-of-00017.safetensors",
+    "model.language_model.layers.4.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.A_log": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.conv1d.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.dt_bias": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.norm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.linear_attn.out_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.experts.down_proj": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.experts.gate_up_proj": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.shared_expert.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.shared_expert.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.shared_expert.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.mlp.shared_expert_gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.5.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.A_log": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.conv1d.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.dt_bias": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.norm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.linear_attn.out_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.experts.down_proj": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.experts.gate_up_proj": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.shared_expert.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.shared_expert.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.shared_expert.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.mlp.shared_expert_gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.6.post_attention_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.input_layernorm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.experts.down_proj": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.experts.gate_up_proj": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.shared_expert.down_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.shared_expert.gate_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.shared_expert.up_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.mlp.shared_expert_gate.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.post_attention_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.k_norm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.q_norm.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.weight": "model-00003-of-00017.safetensors",
+    "model.language_model.layers.8.input_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.A_log": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.conv1d.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.dt_bias": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.norm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.linear_attn.out_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.experts.down_proj": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.experts.gate_up_proj": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.gate.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.shared_expert.down_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.shared_expert.gate_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.shared_expert.up_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.mlp.shared_expert_gate.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.8.post_attention_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.input_layernorm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.A_log": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.conv1d.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.dt_bias": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.norm.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.linear_attn.out_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.experts.down_proj": "model-00005-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.experts.gate_up_proj": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.gate.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.shared_expert.down_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.shared_expert.gate_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.shared_expert.up_proj.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.mlp.shared_expert_gate.weight": "model-00004-of-00017.safetensors",
+    "model.language_model.layers.9.post_attention_layernorm.weight": "model-00005-of-00017.safetensors",
+    "model.language_model.norm.weight": "model-00017-of-00017.safetensors",
+    "model.visual.blocks.0.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.0.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.1.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.10.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.11.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.12.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.13.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.14.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.15.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.16.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.17.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.18.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.19.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.2.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.20.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.21.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.22.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.23.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.24.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.25.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.26.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.3.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.4.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.5.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.6.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.7.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.8.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.attn.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.attn.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.attn.qkv.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.attn.qkv.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.mlp.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.mlp.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.mlp.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.mlp.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.norm1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.norm1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.norm2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.blocks.9.norm2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.merger.linear_fc1.bias": "model-passthrough-00000.safetensors",
+    "model.visual.merger.linear_fc1.weight": "model-passthrough-00000.safetensors",
+    "model.visual.merger.linear_fc2.bias": "model-passthrough-00000.safetensors",
+    "model.visual.merger.linear_fc2.weight": "model-passthrough-00000.safetensors",
+    "model.visual.merger.norm.bias": "model-passthrough-00000.safetensors",
+    "model.visual.merger.norm.weight": "model-passthrough-00000.safetensors",
+    "model.visual.patch_embed.proj.bias": "model-passthrough-00000.safetensors",
+    "model.visual.patch_embed.proj.weight": "model-passthrough-00000.safetensors",
+    "model.visual.pos_embed.weight": "model-passthrough-00000.safetensors"
+  }
+}

preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "patch_size": 16,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "Qwen3VLProcessor",
+  "image_processor_type": "Qwen2VLImageProcessorFast"
+}

processor_config.json

@@ -0,0 +1,60 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

run.json

@@ -0,0 +1,59 @@
+{
+  "step": "train_sft",
+  "created_at": "2026-10-11T01:01:15+00:00",
+  "git_commit": "8b5b96deec230e17c55784177f60235498b92bb2-dirty",
+  "inputs": {},
+  "input_records": {},
+  "config": {
+    "training_purpose": "benign_sft",
+    "seed": 42,
+    "max_tokens": 250000000,
+    "tokens_per_step": 500000,
+    "save_token_milestones": [
+      250000000
+    ],
+    "pack_length": {
+      "qwen3_5_moe": 31250,
+      "gemma4": 15625
+    },
+    "optimizer": {
+      "lr": 2e-06,
+      "min_lr": 1e-07,
+      "lr_decay_style": "cosine",
+      "lr_warmup_fraction": 0.1,
+      "weight_decay": 0.05,
+      "clip_grad": 1.0,
+      "muon_momentum": 0.95,
+      "muon_use_nesterov": true,
+      "muon_scale_mode": "spectral",
+      "muon_extra_scale_factor": 0.2,
+      "muon_scalar_optimizer": "adam"
+    }
+  },
+  "args": {},
+  "results": {
+    "step": 500,
+    "total_steps": 500,
+    "nominal_tokens": 250000000,
+    "consumed_tokens": 249586222,
+    "nominal_tokens_per_step": 500000
+  },
+  "public_release": {
+    "repo_id": "AI45Research/REX-Intern-S2-Bio-F3",
+    "version": "v0.1.0",
+    "node": "F3",
+    "sft_nominal_tokens": 250000000,
+    "record_is_redacted": true,
+    "omitted": [
+      "private_paths",
+      "data_recipe",
+      "nested_private_provenance",
+      "training_logs"
+    ],
+    "source_record_step": "train_sft",
+    "original_experts": 256,
+    "expanded_experts": 264,
+    "note": "Release metadata is a public projection; it is not a complete training record. The endpoint is selected by the private final index; missing source provenance is not reconstructed.",
+    "code_revision": "b0178042de27ad2c0679a483da93bad6b2dbb76b"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,45 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<|image_pad|>",
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

tokenization_interns1.py

@@ -0,0 +1,1009 @@
+# coding=utf-8
+# Copyright 2025 The Intern team and Shanghai AI Lab team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for InternS1."""
+
+import json
+import os
+import unicodedata
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+from functools import lru_cache
+
+import regex as re
+import sentencepiece as spm
+
+from transformers.tokenization_utils_base import AddedToken, TextInput
+from transformers.utils import logging
+from packaging import version
+import transformers
+if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+    from transformers.tokenization_python import PreTrainedTokenizer
+else:
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
+logger = logging.get_logger(__name__)
+
+try:
+    from rdkit import Chem, RDLogger
+
+    RDLogger.DisableLog("rdApp.error")
+    RDLogger.DisableLog("rdApp.*")
+    RDKIT_AVAILABLE = True
+except ImportError:
+    logger.warning_once(
+        "If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
+    )
+    RDKIT_AVAILABLE = False
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+    "sp_model_SMILES": "tokenizer_SMILES.model",
+    "sp_model_PROT": "tokenizer_PROT.model",
+    "sp_model_XNA": "tokenizer_XNA.model",
+}
+
+PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+
+
+class InternS1CheckModuleMixin(ABC):
+    """
+    Basic auto-detection module.
+
+    Note that short strings are ignored by this module.
+    """
+
+    def __init__(self, *, min_length: int):
+        self.min_length = min_length
+        self.REGEX = self._build_regex()
+        self.all_auto_detect_token_start = ["<SMILES_AUTO_DETECT>", "<PROT_AUTO_DETECT>", "<XNA_AUTO_DETECT>"]
+        self.all_auto_detect_token_end = ["</SMILES_AUTO_DETECT>", "</PROT_AUTO_DETECT>", "</XNA_AUTO_DETECT>"]
+        self.auto_detect_token = []
+        self.truncation = False
+
+    @abstractmethod
+    def _build_regex(self):
+        pass
+
+    @abstractmethod
+    def check_legitimacy(self, candidate: str) -> bool:
+        pass
+
+    def re_split(self, texts: Union[str, list[str]]) -> list[str]:
+        if isinstance(texts, str):
+            texts = [texts]
+
+        total_results = []
+
+        no_split_flag = 0
+
+        for text in texts:
+            if text in self.all_auto_detect_token_start:
+                total_results.append(text)
+                no_split_flag += 1
+                continue
+            elif text in self.all_auto_detect_token_end:
+                total_results.append(text)
+                no_split_flag = max(0, no_split_flag - 1)
+                continue
+
+            if no_split_flag > 0:
+                total_results.append(text)
+                continue
+
+            results = []
+            current_pos = 0
+            for match in self.REGEX.finditer(text):
+                candidate = match.group(1)
+
+                if len(candidate) >= self.min_length:
+                    match_start, match_end = match.span(1)
+
+                    if not self.check_legitimacy(candidate):
+                        continue
+
+                    if not self.truncation:
+                        if match_start > 0 and text[match_start - 1].encode("UTF-8").isalpha():
+                            continue
+                        if match_end < len(text) and text[match_end].encode("UTF-8").isalpha():
+                            continue
+
+                    if match_start > current_pos:
+                        non_candidate_part = text[current_pos:match_start]
+                        results.append(non_candidate_part)
+                else:
+                    continue
+
+                results.extend([self.auto_detect_token[0], candidate, self.auto_detect_token[1]])
+                current_pos = match_end
+
+            if current_pos < len(text):
+                remaining_part = text[current_pos:]
+                results.append(remaining_part)
+
+            total_results.extend(results)
+
+        return total_results
+
+
+class XnaCheckModule(InternS1CheckModuleMixin):
+    """
+    XNA sequence auto-detection module.
+
+    Automatically detects XNA sequence using regex patterns.
+    """
+    def __init__(self, *, min_length: int = 27):
+        super().__init__(min_length=min_length)
+        self.auto_detect_token = ["<XNA_AUTO_DETECT>", "</XNA_AUTO_DETECT>"]
+        self.truncation = True
+
+    def _build_regex(self):
+        return re.compile(r"([ATCGU]{" + str(self.min_length) + r",})")
+
+    def check_legitimacy(self, candidate: str):
+        return True
+
+
+class ProtCheckModule(InternS1CheckModuleMixin):
+    """
+    Protein sequence auto-detection module.
+
+    Automatically detects protein sequence using regex patterns.
+    """
+    def __init__(self, *, min_length: int = 27):
+        super().__init__(min_length=min_length)
+        self.auto_detect_token = ["<PROT_AUTO_DETECT>", "</PROT_AUTO_DETECT>"]
+        self.truncation = True
+        self._xna_pattern = re.compile(r"^[ATCGU]+$")
+
+    def _build_regex(self):
+        return re.compile(r"([A-Z]{" + str(self.min_length) + r",})")
+
+    def check_legitimacy(self, candidate: str):
+        if self._xna_pattern.match(candidate):
+            return False
+        return True
+
+
+# fmt: off
+bonds = ["-", "=", "#", ":", "/", "\\", ".", "$"]
+organic_symbols = ["B", "C", "N", "O", "P", "S", "F", "Cl", "Br", "I"]
+other_allows = bonds + ["[", "]", "(", ")", ";"]
+aromatic_symbols = ["b", "c", "n", "o", "s", "p"]
+elements = [
+    "H", "He", "Li", "Be", "B", "C", "N", "O", "F", "Ne",
+    "Na", "Mg", "Al", "Si", "P", "S", "Cl", "Ar", "K", "Ca",
+    "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn",
+    "Ga", "Ge", "As", "Se", "Br", "Kr", "Rb", "Sr", "Y", "Zr",
+    "Nb", "Mo", "Tc", "Ru", "Rh", "Pd", "Ag", "Cd", "In", "Sn",
+    "Sb", "Te", "I", "Xe", "Cs", "Ba", "La", "Ce", "Pr", "Nd",
+    "Pm", "Sm", "Eu", "Gd", "Tb", "Dy", "Ho", "Er", "Tm", "Yb",
+    "Lu", "Hf", "Ta", "W", "Re", "Os", "Ir", "Pt", "Au", "Hg",
+    "Tl", "Pb", "Bi", "Po", "At", "Rn", "Fr", "Ra", "Ac", "Th",
+    "Pa", "U", "Np", "Pu", "Am", "Cm", "Bk", "Cf", "Es", "Fm",
+    "Md", "No", "Lr", "Rf", "Db", "Sg", "Bh", "Hs", "Mt", "Ds",
+    "Rg", "Cn", "Nh", "Fl", "Mc", "Lv", "Ts", "Og"
+]
+# fmt: on
+
+
+class SmilesCheckModule(InternS1CheckModuleMixin):
+    """
+    SMILES molecular sequence auto-detection module.
+
+    Automatically detects and validates SMILES strings in text using regex patterns
+    or chemical syntax rules. Uses RDKit for precise validation when available,
+    otherwise falls back to rule-based validation.
+    """
+
+    def __init__(self, *, min_length: int = 10):
+        super().__init__(min_length=min_length)
+        self.auto_detect_token = ["<SMILES_AUTO_DETECT>", "</SMILES_AUTO_DETECT>"]
+        self._SQ_BRACKET_BAN_1 = re.compile(r"(?:[A-GI-Z]|[a-z]){3,}")
+        self._SQ_BRACKET_BAN_2 = re.compile(r"\d{4,}")
+
+    def _build_regex(self):
+        # fmt: off
+        _two_letter_elements = [
+            'Ac', 'Ag', 'Al', 'Am', 'Ar', 'As', 'At', 'Au', 'Ba', 'Be', 'Bh', 'Bi', 'Bk', 'Br', 'Ca', 'Cd',
+            'Ce', 'Cf', 'Cl', 'Cm', 'Cn', 'Co', 'Cr', 'Cs', 'Cu', 'Db', 'Ds', 'Dy', 'Er', 'Es', 'Eu', 'Fe',
+            'Fl', 'Fm', 'Fr', 'Ga', 'Gd', 'Ge', 'He', 'Hf', 'Hg', 'Ho', 'Hs', 'In', 'Ir', 'Kr', 'La', 'Li',
+            'Lr', 'Lu', 'Lv', 'Mc', 'Md', 'Mg', 'Mn', 'Mo', 'Mt', 'Na', 'Nb', 'Nd', 'Ne', 'Nh', 'Ni', 'No',
+            'Np', 'Og', 'Os', 'Pa', 'Pb', 'Pd', 'Pm', 'Po', 'Pr', 'Pt', 'Pu', 'Ra', 'Rb', 'Re', 'Rf', 'Rg',
+            'Rh', 'Rn', 'Ru', 'Sb', 'Sc', 'Se', 'Sg', 'Si', 'Sm', 'Sn', 'Sr', 'Ta', 'Tb', 'Tc', 'Te', 'Th',
+            'Ti', 'Tl', 'Tm', 'Ts', 'Xe', 'Yb', 'Zn', 'Zr'
+        ]
+        _single_letter_elements = [
+            "B", "C", "F", "H", "I", "K", "N", "O", "P", "S", "U", "V", "W", "Y", 'b', 'c', 'n', 'o', 'p', 's'
+        ]
+        # fmt: on
+        all_elements_sorted = sorted(_two_letter_elements + _single_letter_elements, key=lambda x: (-len(x), x))
+        elements_pattern_str = "|".join(all_elements_sorted)
+
+        bracket_atom_pattern_str = r"\[[^\]]+\]"
+        other_single_chars_pattern_str = r"[\(\)\.=\-#@\d\$\%\*:\+\-\/\\]"
+        smiles_unit_pattern = (
+            r"(?:"
+            + bracket_atom_pattern_str
+            + r"|"
+            + elements_pattern_str
+            + r"|"
+            + other_single_chars_pattern_str
+            + r")"
+        )
+        core_sequence_pattern = rf"(?>{smiles_unit_pattern}){{10,}}"
+        constrained_core_sequence_pattern = rf"(?![:.=]){core_sequence_pattern}(?<![:.=])"
+
+        final_regex_str = rf"({constrained_core_sequence_pattern})"
+
+        COMPILED_REGEX = re.compile(final_regex_str)
+        return COMPILED_REGEX
+
+    def check_legitimacy_slow(self, candidate: str) -> bool:
+        """Check legitimacy with RDKit"""
+        if sum(1 for char in candidate if char.encode("UTF-8").isalpha()) < 5:
+            return False
+
+        mol = Chem.MolFromSmiles(candidate)
+        if mol is None:
+            return False
+        else:
+            return True
+
+    def check_legitimacy_fast(self, candidate: str) -> bool:
+        """Check legitimacy with hard rules"""
+        if sum(1 for char in candidate if char.encode("UTF-8").isalpha()) < 5:
+            return False
+
+        if not self.check_rings_and_brackets(candidate):
+            return False
+        else:
+            return True
+
+    def check_legitimacy(self, candidate: str) -> bool:
+        if RDKIT_AVAILABLE:
+            return self.check_legitimacy_slow(candidate)
+        else:
+            return self.check_legitimacy_fast(candidate)
+
+    def check_brackets(self, text):
+        matches = re.findall(r"\[([^\[\]]*)\]", text)
+        for part in matches:
+            if "(" in part or ")" in part:
+                return False
+            if len(part) == 0:
+                return False
+            if part[0] in elements or part[0] in aromatic_symbols or part[:2] in elements:
+                return True
+        return True
+
+    def check_rings_and_brackets(self, text):
+        rings = {}
+        left_sq_bracket, right_sq_bracket = 0, 0
+        left_pt_bracket, right_pt_bracket = 0, 0
+        all_lower = True
+        digits_cnt = 0
+        pos = 0
+        while pos < len(text):
+            step = 0
+            c = text[pos]
+            if ord(c) >= 65 and ord(c) <= 90:
+                all_lower = False
+            if (pos == len(text) - 1 or pos == 0) and c in bonds:
+                return False
+            if pos > 0 and text[pos - 1] in bonds and text[pos] in bonds:
+                return False
+            if c == "[":
+                step = 1
+                left_sq_bracket += 1
+                if left_sq_bracket > right_sq_bracket + 1:
+                    return False
+                if pos == len(text) - 1:
+                    return False
+                if "]" not in text[pos + 1 :]:
+                    return False
+                bracket_span = text[pos + 1 : text.find("]")]
+
+                if self._SQ_BRACKET_BAN_1.search(bracket_span) or self._SQ_BRACKET_BAN_2.search(bracket_span):
+                    return False
+
+                matches = re.findall(r"\d+", bracket_span)
+                if len(matches) > 2:
+                    return False
+            if c == "]":
+                step = 1
+                right_sq_bracket += 1
+                if right_sq_bracket > left_sq_bracket:
+                    return False
+
+            if c == "(":
+                step = 1
+                left_pt_bracket += 1
+            if c == ")":
+                step = 1
+                right_pt_bracket += 1
+                if right_pt_bracket > left_pt_bracket:
+                    return False
+
+            if left_sq_bracket == right_sq_bracket:
+                if c.isdigit():
+                    digits_cnt += 1
+                    step = 1
+                    if (
+                        pos == 0
+                        or (pos == 1 and text[pos - 1] != "%")
+                        or (pos > 1 and text[pos - 1] != "%" and text[pos - 2] != "%")
+                    ):
+                        if c in rings:
+                            if rings[c] == "unclosed":
+                                rings[c] = "closed"
+                            else:
+                                rings[c] = "unclosed"
+                        else:
+                            rings[c] = "unclosed"
+                if c == "%":
+                    if pos >= len(text) - 2 or not text[pos + 1].isdigit() or not text[pos + 2].isdigit():
+                        return False
+                    step = 3
+                    digits_cnt += 1
+                    num = text[pos + 1 : pos + 3]
+                    if num in rings:
+                        if rings[num] == "unclosed":
+                            rings[num] = "closed"
+                        else:
+                            rings[num] = "unclosed"
+                    else:
+                        rings[num] = "unclosed"
+                if step == 0:
+                    if (
+                        pos < len(text) - 1
+                        and text[pos : pos + 2] in organic_symbols + aromatic_symbols + other_allows
+                    ):
+                        step = 2
+                    elif c in organic_symbols + aromatic_symbols + other_allows:
+                        step = 1
+                    else:
+                        return False
+
+            if step == 0:
+                step = 1
+            pos += step
+
+        if left_sq_bracket != right_sq_bracket or any(v == "unclosed" for v in rings.values()):
+            return False
+        if all_lower and digits_cnt < 2:
+            return False
+        return self.check_brackets(text)
+
+
+@lru_cache
+# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+# Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+# @requires(backends=("sentencepiece",))
+class InternS1Tokenizer(PreTrainedTokenizer):
+    """
+    Construct an InternS1 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import AutoTokenizer
+
+    >>> tokenizer = AutoTokenizer.from_pretrained("InternS1Tokenizer", trust_remote_code=True)
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    Include custom extension to support better domain-specific text tokenization, leveraging a separately trained tokenizer model.
+
+    ```python
+    >>> from transformers import AutoTokenizer
+
+    >>> tokenizer = AutoTokenizer.from_pretrained("InternS1Tokenizer", trust_remote_code=True)
+    >>> tokenizer.tokenize("Describe <SMILES>C1=CC=C(C=C1)C=O</SMILES> and CC1=CC=CC=C1C=O")
+    ["Describe ", "<SMILES>", "C1=CC=C(C=C1)C=O", "</SMILES>", " and ", "<SMILES_AUTO_DETECT>",
+        "CC1=CC=CC=C1C=O", "</SMILES_AUTO_DETECT>"]
+    >>> token_ids = tokenizer("Describe <SMILES>C1=CC=C(C=C1)C=O</SMILES> and CC1=CC=CC=C1C=O")["input_ids"]
+    >>> token_ids
+    [74785, 220, 151925, 151854, 151860, 151698, 151707, 151860, 151690, 151726, 151926, 323, 220, 151672, 151860, 151701, 151860, 151854, 151726]
+
+    >>> tokenizer.convert_ids_to_tokens(token_ids)
+    ['Describe', 'Ġ', '<SMILES>', 'C', '1', '=CC=C(', 'C=C', '1', ')C', '=O', '</SMILES>', 'Ġand', 'Ġ', 'CC', '1', '=CC=CC=C', '1', 'C', '=O']
+    ```
+
+    Users should refer to this superclass [`PreTrainedTokenizer`] for more information regarding those overloaded methods
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8. See
+            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+        split_special_tokens (`bool`, *optional*, defaults to `False`):
+            Whether or not the special tokens should be split during the tokenization process. The default behavior is
+            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        clean_up_tokenization_spaces=False,
+        split_special_tokens=False,
+        special_tokens_pattern="none",
+        **kwargs,
+    ):
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_merges = []
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            for i, line in enumerate(merges_handle):
+                line = line.strip()
+                if (i == 0 and line.startswith("#version:")) or not line:
+                    continue
+                bpe_merges.append(tuple(line.split()))
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        # NOTE: the cache can grow without bound and will get really large for long running processes
+        # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
+        # not a memory leak but appears as one.
+        # GPT2Tokenizer has the same problem, so let's be consistent.
+        self.cache = {}
+
+        self.pat = re.compile(PRETOKENIZE_REGEX)
+
+        if kwargs.get("add_prefix_space", False):
+            logger.warning_once(
+                f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
+            )
+
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            split_special_tokens=split_special_tokens,
+            special_tokens_pattern=special_tokens_pattern,
+            **kwargs,
+        )
+
+        self.prepare_extra_tokenizers(vocab_file)
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self.encoder)
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
+    def prepare_extra_tokenizers(self, vocab_file: str) -> None:
+        """
+        Prepare domain-specific tokenizers.
+        
+        Define variables/maps here which guide domain-specific tokenization later.
+        """
+        # Load extra tokenizers with SentencePiece model
+        dir_name = os.path.dirname(vocab_file)
+
+        self.sp_model_SMILES = spm.SentencePieceProcessor()
+        self.sp_model_SMILES.Load(os.path.join(dir_name, "tokenizer_SMILES.model"))
+        self.sp_model_SMILES.offset = self.init_kwargs["offset_SMILES"]
+
+        self.sp_model_PROT = spm.SentencePieceProcessor()
+        self.sp_model_PROT.Load(os.path.join(dir_name, "tokenizer_PROT.model"))
+        self.sp_model_PROT.offset = self.init_kwargs["offset_PROT"]
+
+        self.sp_model_XNA = spm.SentencePieceProcessor()
+        self.sp_model_XNA.Load(os.path.join(dir_name, "tokenizer_XNA.model"))
+        self.sp_model_XNA.offset = self.init_kwargs["offset_XNA"]
+
+        base_mapping = {
+            "SMILES": self.sp_model_SMILES,
+            "protein": self.sp_model_PROT,
+            "dna": self.sp_model_XNA,
+            "rna": self.sp_model_XNA,
+        }
+        auto_detect_mapping = {
+            "SMILES": self.sp_model_SMILES,
+            "PROT": self.sp_model_PROT,
+            "XNA": self.sp_model_XNA,
+        }
+        # Guiding tokens of domain-specific tokenization
+        self.ex_begin_mapping = {f"<{key}>": value for key, value in base_mapping.items()}
+        self.ex_end_mapping = {f"</{key}>": value for key, value in base_mapping.items()}
+        # Transient markers for auto-detection, these tokens will not be assigned token ids
+        self.ex_auto_begin_mapping = {f"<{key}_AUTO_DETECT>": value for key, value in auto_detect_mapping.items()}
+        self.ex_auto_end_mapping = {f"</{key}_AUTO_DETECT>": value for key, value in auto_detect_mapping.items()}
+        # Token markers to prevent unwanted auto-detection
+        self.ex_protect_begin_tokens = ["<MOLFORMULA>"]
+        self.ex_protect_end_tokens = ["</MOLFORMULA>"]
+        # For simplicity
+        self.ex_protect_tokens = self.ex_protect_begin_tokens + self.ex_protect_end_tokens
+        self.ex_all_begin_mapping = self.ex_begin_mapping | self.ex_auto_begin_mapping
+        self.ex_all_end_mapping = self.ex_end_mapping | self.ex_auto_end_mapping
+
+        # Update encoder & decoder with extra tokenizers
+        for tokenizer_name, sp_model in [
+            ("SMILES", self.sp_model_SMILES),
+            ("PROT", self.sp_model_PROT),
+            ("XNA", self.sp_model_XNA),
+        ]:
+            self.decoder.update(
+                {i + sp_model.offset: sp_model.id_to_piece(i) for i in range(sp_model.get_piece_size())}
+            )
+            # Not really used, only to fill holes in encoder, to keep methods like `add_tokens` working
+            self.encoder.update(
+                {
+                    f"<|{tokenizer_name}_{sp_model.id_to_piece(i)}|>": i + sp_model.offset
+                    for i in range(sp_model.get_piece_size())
+                }
+            )
+
+        # protect-tokens should keep complete temporarily to guide later tokenization
+        # it will be segmented later
+        for token in self.ex_protect_tokens:
+            self.tokens_trie.add(token)
+
+        self._unk_token = "<unk>"  # Fall-back
+        self.check_module_list = [SmilesCheckModule(), ProtCheckModule(), XnaCheckModule()]
+
+    def _pop_logical_sp_token(self, extra_tokenizer_stack: list, mapping_name: str) -> None:
+        """Switch tokenizer when it comes to an end sp token"""
+        extra_tokenizer = extra_tokenizer_stack.pop()
+        if extra_tokenizer != self.ex_all_end_mapping[mapping_name]:
+            logger.warning_once(
+                f"Encounter incorrect nesting of extra tokenizer: {self.ex_all_end_mapping[mapping_name]} and {extra_tokenizer}"
+            )
+            logger.warning_once("This may lead to unexpected behaviour of the tokenizer, please check your input.")
+
+    def tokenize(self, text: TextInput, **kwargs) -> list[str]:
+        """
+        Converts a string into a sequence of tokens, using the tokenizer.
+
+        It will switch to domain-specific tokenizer once encountering extra/logical sp tokens.
+
+        Args:
+            text: TextInput
+        """
+        split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)
+
+        text, kwargs = self.prepare_for_tokenization(text, **kwargs)
+
+        if hasattr(self, "do_lower_case") and self.do_lower_case:
+            # convert non-special tokens to lowercase. Might be super slow as well?
+            escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
+            escaped_special_toks += [
+                re.escape(s_tok.content)
+                for s_tok in (self._added_tokens_decoder.values())
+                if not s_tok.special and s_tok.normalized
+            ]
+            pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
+            text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)
+
+        if split_special_tokens:
+            no_split_token = []
+            tokens = [text]
+        else:
+            no_split_token = self._added_tokens_encoder.keys()  # don't split on any of the added tokens
+            # "This is something<special_token_1>  else"
+            tokens = self.tokens_trie.split(text)
+
+        # ["This is something", "<special_token_1>", "  else"]
+        for i, token in enumerate(tokens):
+            if token in no_split_token:
+                tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)
+                left = tokens[i - 1] if i > 0 else None
+                right = tokens[i + 1] if i < len(tokens) - 1 else None
+                if isinstance(tok_extended, AddedToken):
+                    if tok_extended.rstrip and right:
+                        # A bit counter-intuitive but we strip the left of the string
+                        # since tok_extended.rstrip means the special token is eating all white spaces on its right
+                        tokens[i + 1] = right.lstrip()
+                    # Strip white spaces on the left
+                    if tok_extended.lstrip and left:
+                        tokens[i - 1] = left.rstrip()  # Opposite here
+                    if tok_extended.single_word and left and left[-1] != " ":
+                        tokens[i - 1] += token
+                        tokens[i] = ""
+                    elif tok_extended.single_word and right and right[0] != " ":
+                        tokens[i + 1] = token + tokens[i + 1]
+                        tokens[i] = ""
+                else:
+                    raise ValueError(
+                        f"{tok_extended} cannot be tokenized because it was not properly added"
+                        f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"
+                    )
+
+        # ["This is something", "<special_token_1>", "else"]
+        tokenized_text = []
+
+        # Codes for automatically detecting domain-specific content
+        # All parts that have been marked by domain-specific or protection tokens will not be subject to auto detection
+        # See transformers/tests/models/intern_s1/test_tokenization_intern_s1.py::test_auto_detection() for more details
+        new_tokens = []
+        not_split_flag = 0
+        for token in tokens:
+            if not token:
+                continue
+            if token in no_split_token or token in self.ex_protect_tokens:
+                new_tokens.append(token)
+                if token in self.ex_begin_mapping or token in self.ex_protect_begin_tokens:
+                    not_split_flag += 1  # In case nested sp tokens
+                elif token in self.ex_end_mapping or token in self.ex_protect_end_tokens:
+                    not_split_flag = max(0, not_split_flag - 1)
+            else:
+                if not_split_flag:
+                    new_tokens.append(token)
+                else:
+                    for check_module in self.check_module_list:
+                        token = check_module.re_split(token)
+
+                    new_tokens.extend(token)
+        tokens = new_tokens
+
+        # Use stack to maintain which tokenizer should be used, considering the possibility of nested extra tokenizer
+        extra_tokenizer_stack = []
+        for token in tokens:
+            # Need to skip eventual empty (fully stripped) tokens
+            if not token:
+                continue
+            # protect-tokens are not assigned token ids, should be segmented here
+            if token in self.ex_protect_tokens:
+                tokenized_text.extend(self._tokenize(token))
+            # push tokenizer to stack when encountering begin token
+            elif token in self.ex_all_begin_mapping:
+                tokenized_text.append(token)
+                extra_tokenizer_stack.append(self.ex_all_begin_mapping[token])
+            # pop tokenizer from stack when encountering end token
+            elif token in self.ex_all_end_mapping:
+                tokenized_text.append(token)
+                if extra_tokenizer_stack:
+                    self._pop_logical_sp_token(extra_tokenizer_stack, token)
+            # other special tokens
+            elif token in no_split_token:
+                tokenized_text.append(token)
+            else:
+                tokenized_text.extend(self._tokenize(token, extra_tokenizer_stack=extra_tokenizer_stack))
+
+        # ["This", " is", " something", "<special_token_1>", "else"]
+        return tokenized_text
+
+    def _tokenize(self, text, **kwargs):
+        """
+        Modified from `transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize`.
+
+        This adaptation supports domain-specific tokenizers.
+        """
+        extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
+        if extra_tokenizer_stack:
+            tokenized_text = extra_tokenizer_stack[-1].encode(text, out_type=str)
+            tokenized_id = extra_tokenizer_stack[-1].encode(text, out_type=int)
+            final_tokenized_text = []
+            for text_piece, id_piece in zip(tokenized_text, tokenized_id):
+                if id_piece == 0:
+                    final_tokenized_text.extend(self._bpe_tokenize(text_piece))
+                else:
+                    final_tokenized_text.append(text_piece)
+            return final_tokenized_text
+        else:
+            return self._bpe_tokenize(text)
+
+    def _bpe_tokenize(self, text, **kwargs):
+        text = text.replace(
+            "▁", " "
+        )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(
+                self.byte_encoder[b] for b in token.encode("utf-8")
+            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
+            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+
+    def convert_tokens_to_ids(self, tokens: Union[str, list[str]]) -> Union[int, list[int]]:
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier.convert_tokens_to_ids`.
+
+        Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
+        vocabulary.
+
+        This adaptation supports domain-specific tokenizers.
+
+        Args:
+            tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).
+
+        Returns:
+            `int` or `List[int]`: The token id or list of token ids.
+        """
+        if tokens is None:
+            return None
+
+        if isinstance(tokens, str):
+            return self._convert_token_to_id_with_added_voc(tokens)
+
+        ids = []
+        extra_tokenizer_stack = []
+
+        for token in tokens:
+            if token not in self.ex_auto_begin_mapping and token not in self.ex_auto_end_mapping:
+                ids.append(
+                    self._convert_token_to_id_with_added_voc(token, extra_tokenizer_stack=extra_tokenizer_stack)
+                )
+            if token in self.ex_all_begin_mapping:
+                extra_tokenizer_stack.append(self.ex_all_begin_mapping[token])
+            elif token in self.ex_all_end_mapping:
+                if extra_tokenizer_stack:
+                    self._pop_logical_sp_token(extra_tokenizer_stack, token)
+        return ids
+
+    def _convert_token_to_id_with_added_voc(self, token, **kwargs):
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id_with_added_voc`.
+
+        This adaptation supports domain-specific tokenizers.
+        """
+        if token is None:
+            return None
+
+        if token in self._added_tokens_encoder:
+            return self._added_tokens_encoder[token]
+        return self._convert_token_to_id(token, **kwargs)
+
+    def _convert_token_to_id(self, token, **kwargs):
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id`.
+
+        Converts a token (str) in an id using the vocab.
+
+        Fall back to original tokenizer once OOV.
+        """
+        extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
+        if extra_tokenizer_stack:
+            token_id = extra_tokenizer_stack[-1].piece_to_id(token)
+            if token_id == extra_tokenizer_stack[-1].unk_id():
+                return self.encoder.get(token, self.encoder.get(self._unk_token))
+            else:
+                return token_id + extra_tokenizer_stack[-1].offset
+        else:
+            return self.encoder.get(token, self.encoder.get(self._unk_token))
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_id_to_token
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index)
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = text.replace(
+            "▁", "Ġ"
+        )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.
+        text = text.replace("\n", "Ċ")
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+    def decode(
+        self,
+        token_ids,
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
+        # and cannot be configured elsewhere, but it should default to False for InternS1Tokenizer
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        )
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
+        """
+        Modified from `transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary` to support saving custom extension.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
+        )
+        sp_model_smiles = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_SMILES"]
+        )
+        sp_model_prot = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_PROT"]
+        )
+        sp_model_xna = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_XNA"]
+        )
+
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+
+        index = 0
+        with open(merge_file, "w", encoding="utf-8") as writer:
+            writer.write("#version: 0.2\n")
+            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
+                        " Please check that the tokenizer is not corrupted!"
+                    )
+                    index = token_index
+                writer.write(" ".join(bpe_tokens) + "\n")
+                index += 1
+
+        with open(sp_model_smiles, "wb") as f:
+            f.write(self.sp_model_SMILES.serialized_model_proto())
+
+        with open(sp_model_prot, "wb") as f:
+            f.write(self.sp_model_PROT.serialized_model_proto())
+
+        with open(sp_model_xna, "wb") as f:
+            f.write(self.sp_model_XNA.serialized_model_proto())
+
+        return vocab_file, merge_file
+
+    def prepare_for_tokenization(self, text, **kwargs):
+        text = unicodedata.normalize("NFC", text)
+        return (text, kwargs)
+
+
+__all__ = ["InternS1Tokenizer"]

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42
+size 12807982

tokenizer_PROT.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f50d0b02ee7d4d8d563136d55dd94241325e5f992a5583ef67ddeeed77d734
+size 11977

tokenizer_SMILES.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98ac5c860995bf9c127f9953d830f3ffd4c790ed36183e6ddabf7e2d9b5dd89
+size 14640

tokenizer_XNA.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15e56771a919f946fbd3fb762f00333dfa8b91e08fbdb4e89dc98977d8e210be
+size 15309

tokenizer_config.json

@@ -0,0 +1,508 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "248044": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248045": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248046": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248047": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248048": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248049": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248050": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248051": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248052": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248053": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248054": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248055": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248056": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248057": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248058": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248059": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248060": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248061": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248062": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248063": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248064": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248065": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248066": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248067": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248068": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248069": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248070": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248071": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248072": {
+      "content": "<tts_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248073": {
+      "content": "<tts_text_bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248074": {
+      "content": "<tts_text_eod>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248075": {
+      "content": "<tts_text_bos_single>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248076": {
+      "content": "<|audio_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248077": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248078": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248079": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248080": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248081": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248082": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248083": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248084": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248085": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248086": {
+      "content": "<|action_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248087": {
+      "content": "<|action_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248088": {
+      "content": "<|interpreter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248089": {
+      "content": "<|plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248090": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248091": {
+      "content": "<|ts|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248092": {
+      "content": "<|/ts|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248093": {
+      "content": "<TS_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248094": {
+      "content": "<SMILES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248095": {
+      "content": "</SMILES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248096": {
+      "content": "<protein>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248097": {
+      "content": "</protein>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248098": {
+      "content": "<dna>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248099": {
+      "content": "</dna>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248100": {
+      "content": "<rna>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248101": {
+      "content": "</rna>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_interns1.InternS1Tokenizer",
+      null
+    ]
+  },
+  "backend": "custom",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "offset_PROT": 249126,
+  "offset_SMILES": 248102,
+  "offset_XNA": 250150,
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "InternS1Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

video_preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+  "size": {
+    "longest_edge": 25165824,
+    "shortest_edge": 4096
+  },
+  "patch_size": 16,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor_type": "Qwen3VLVideoProcessor"
+}