Instructions to use bigai/GS1-9B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use bigai/GS1-9B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("feature-extraction", model="bigai/GS1-9B", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("bigai/GS1-9B", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Upload GS1-9B checkpoint and inference files
Browse files- .gitattributes +1 -0
- LICENSE +202 -0
- NOTICE +6 -0
- README.md +75 -199
- README.zh-CN.md +105 -0
- chat_template.jinja +154 -0
- compiler.py +395 -0
- config.json +148 -0
- config.py +46 -0
- model-00001-of-00009.safetensors +3 -0
- model-00002-of-00009.safetensors +3 -0
- model-00003-of-00009.safetensors +3 -0
- model-00004-of-00009.safetensors +3 -0
- model-00005-of-00009.safetensors +3 -0
- model-00006-of-00009.safetensors +3 -0
- model-00007-of-00009.safetensors +3 -0
- model-00008-of-00009.safetensors +3 -0
- model-00009-of-00009.safetensors +3 -0
- model.safetensors.index.json +828 -0
- modular_gs1.py +971 -0
- preprocessor_config.json +21 -0
- requirements.txt +10 -0
- schema.py +266 -0
- tokenizer.json +3 -0
- tokenizer_config.json +40 -0
- util.py +202 -0
- video_preprocessor_config.json +21 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
GS1
|
| 2 |
+
Copyright 2026 BIGAI.
|
| 3 |
+
|
| 4 |
+
GS1 uses Qwen3.5 components from Hugging Face Transformers under Apache-2.0.
|
| 5 |
+
Third-party libraries retain their respective copyright and licenses.
|
| 6 |
+
Model weights are distributed under the license stated in each model card.
|
README.md
CHANGED
|
@@ -1,240 +1,116 @@
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
-
language:
|
| 4 |
-
- en
|
| 5 |
-
base_model:
|
| 6 |
-
- Qwen/Qwen3.5-9B
|
| 7 |
-
base_model_relation: finetune
|
| 8 |
-
model_name: GS1-9B
|
| 9 |
-
thumbnail: https://huggingface.co/bigai/GS1-9B/resolve/main/assets/banner.svg
|
| 10 |
-
tags:
|
| 11 |
-
- decision-making
|
| 12 |
-
- typed-decisions
|
| 13 |
-
- agent-routing
|
| 14 |
-
- verification
|
| 15 |
-
- uncertainty-estimation
|
| 16 |
-
- classification
|
| 17 |
-
- qwen3.5
|
| 18 |
library_name: transformers
|
| 19 |
-
|
| 20 |
-
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
dataset:
|
| 26 |
-
type: decision-index
|
| 27 |
-
name: Decision Index 0.3 (Public)
|
| 28 |
-
metrics:
|
| 29 |
-
- type: decision-index
|
| 30 |
-
name: Decision Index Score
|
| 31 |
-
value: 70.47
|
| 32 |
-
source:
|
| 33 |
-
name: GS1 reported comparison snapshot
|
| 34 |
-
url: https://github.com/bigai-nlco/GS1/blob/main/README.md
|
| 35 |
---
|
| 36 |
-
# GS1-9B
|
| 37 |
-
|
| 38 |
-
> **Fast, Faithful, Frontier System One Intelligence.**
|
| 39 |
|
| 40 |
-
|
| 41 |
|
| 42 |
-
[
|
| 43 |
|
| 44 |
-
|
| 45 |
|
| 46 |
-
|
| 47 |
-
> Load this model directly with Transformers `AutoModel` and call `model.decide()`. The GS1 repository is optional for default inference; install it for optimized execution, serving, and evaluation.
|
| 48 |
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
-
|
| 52 |
|
| 53 |
-
* **
|
| 54 |
-
* **Full comparisons, reusable prefixes:** Causal Gated DeltaNet and segmented bidirectional full attention combine candidate comparison with immutable shared prefixes.
|
| 55 |
-
* **Carry-preserving Attention Residuals:** Sparse full-attention receivers retrieve earlier representations while retaining the explicit skip-connection anchor.
|
| 56 |
-
* **Typed uncertainty:** Choice returns candidate probabilities, Noul returns `p(true)`, and Score returns the expected zero-based ordinal level.
|
| 57 |
|
| 58 |
-
|
| 59 |
|
| 60 |
-
##
|
| 61 |
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
**Python 3.11+**. Install the model folder's `requirements.txt`, or:
|
| 65 |
|
| 66 |
```bash
|
| 67 |
-
|
| 68 |
-
|
|
|
|
| 69 |
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
"
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
"safe": {
|
| 88 |
-
"type": "noul",
|
| 89 |
-
"instructions": "Can delivery finish with at least the required battery reserve?",
|
| 90 |
-
},
|
| 91 |
-
"risk": {
|
| 92 |
-
"type": "score",
|
| 93 |
-
"instructions": "Rate the risk of delivering now against the reserve requirement.",
|
| 94 |
-
"criteria": ["No risk.", "Uncertain.", "Reserve would be violated."],
|
| 95 |
},
|
| 96 |
},
|
| 97 |
-
|
| 98 |
)
|
| 99 |
-
print(
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
The response contains `model`, `answers`, and `usage`; `usage.output_tokens` is **0**.
|
| 103 |
-
|
| 104 |
-
| Type | Answer fields | Input limits |
|
| 105 |
-
| :--- | :--- | :--- |
|
| 106 |
-
| Choice | `choice`, `probabilities`, `confidence` | 1–255 candidates |
|
| 107 |
-
| Noul | `noul` = `p(true)` | True/false verification |
|
| 108 |
-
| Score | `score`, `probabilities`, `legend`, `confidence` | 2–10 ordered levels, starting at 0 |
|
| 109 |
-
|
| 110 |
-
> [!Tip]
|
| 111 |
-
> Put shared facts in `state` and related questions in one request. Include an explicit `unknown` candidate when evidence may be insufficient. Confidence is a distribution summary, not a guarantee of correctness.
|
| 112 |
-
|
| 113 |
-
> [!Important]
|
| 114 |
-
> Preserve the checkpoint's stored dtypes; use `precision` to select computation precision rather than converting all weights to BF16. For CPU reference inference, use `.to("cpu")` and `precision="fp32"`.
|
| 115 |
-
|
| 116 |
-
For reproducibility, pass a saved model commit SHA as `revision` to `from_pretrained`. Keep the complete model folder, including AutoClass code and tokenizer/processor assets. No `artifact.json` is required.
|
| 117 |
-
|
| 118 |
-
### Optimized runtime and HTTP serving
|
| 119 |
-
|
| 120 |
-
From a Linux/CUDA environment:
|
| 121 |
-
|
| 122 |
-
```bash
|
| 123 |
-
git clone https://github.com/bigai-nlco/GS1.git
|
| 124 |
-
cd GS1
|
| 125 |
-
uv sync --locked --extra serve --extra cuda
|
| 126 |
-
uv run gs1 serve --model bigai/GS1-9B --model-name GS1
|
| 127 |
```
|
| 128 |
|
| 129 |
-
|
| 130 |
|
| 131 |
```bash
|
| 132 |
-
|
| 133 |
-
-
|
| 134 |
-
-
|
| 135 |
-
"model": "GS1",
|
| 136 |
-
"state": {"battery": 18, "delivery_cost": 12, "required_reserve": 10},
|
| 137 |
-
"questions": {
|
| 138 |
-
"action": {
|
| 139 |
-
"type": "choice",
|
| 140 |
-
"instructions": "Choose an action that preserves the required battery reserve.",
|
| 141 |
-
"criteria": {"deliver": "Deliver now.", "recharge": "Recharge first."}
|
| 142 |
-
}
|
| 143 |
-
}
|
| 144 |
-
}'
|
| 145 |
```
|
| 146 |
|
| 147 |
-
The
|
| 148 |
|
| 149 |
-
##
|
| 150 |
|
| 151 |
-
|
| 152 |
|
| 153 |
```python
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
with TypeSafeClient(
|
| 157 |
-
base_url="http://127.0.0.1:8000", api_key="local", model="GS1"
|
| 158 |
-
) as client:
|
| 159 |
-
result = client.system_one(
|
| 160 |
-
state="A customer was charged twice and requests a refund.",
|
| 161 |
-
questions={
|
| 162 |
-
"team": Choice(
|
| 163 |
-
instructions="Which team should handle this ticket?",
|
| 164 |
-
criteria={"billing": "Payments and refunds", "technical": "Software problems"},
|
| 165 |
-
),
|
| 166 |
-
"refund": Noul(instructions="Does the customer request a refund?"),
|
| 167 |
-
"urgency": Score(
|
| 168 |
-
instructions="Rate urgency.", criteria=["Routine", "Soon", "Immediate"]
|
| 169 |
-
),
|
| 170 |
-
},
|
| 171 |
-
)
|
| 172 |
-
|
| 173 |
-
print(result.choices["team"].probabilities)
|
| 174 |
-
print(result.nouls["refund"].noul)
|
| 175 |
-
print(result.scores["urgency"].score)
|
| 176 |
-
```
|
| 177 |
-
|
| 178 |
-
`api_key="local"` satisfies the SDK argument. Use the API root as `base_url` and match `model` to the server alias. [API and confidence definitions](https://github.com/bigai-nlco/GS1/blob/main/docs/api.md).
|
| 179 |
-
|
| 180 |
-
### Interactive Playground
|
| 181 |
-
|
| 182 |
-
With GS1 serving on port 8000, run from the source checkout (**Node.js 22+**):
|
| 183 |
-
|
| 184 |
-
```bash
|
| 185 |
-
cd playground
|
| 186 |
-
npm ci
|
| 187 |
-
npm run build
|
| 188 |
-
npm start
|
| 189 |
-
```
|
| 190 |
-
|
| 191 |
-
Open [localhost:3000](http://127.0.0.1:3000). GS1 is the default backend. Explore **Query, Go, Maze, and Snake**, with probabilities and raw JSON. [Playground guide](https://github.com/bigai-nlco/GS1/blob/main/playground/README.md).
|
| 192 |
-
|
| 193 |
-
## Evaluation
|
| 194 |
-
|
| 195 |
-
**Decision Index 0.3 (Public): 70.47**, as reported in the repository's model comparison and 6 November 2026 technical-report snapshot. This is a chance-corrected composite score, not accuracy.
|
| 196 |
-
|
| 197 |
-
[](assets/gs1_pareto_frontier.svg)
|
| 198 |
-
|
| 199 |
-
> [!Note]
|
| 200 |
-
> The repository keeps the 9B report score separate pending checkpoint validation. Its complete public evaluation and regular JevBench, AG News, and ToolACE measurements currently cover 0.8B, 2B, and 4B only; checkpoint-validated 9B accuracy, Brier, and ECE results are not provided. [Evaluation status](https://github.com/bigai-nlco/GS1/blob/main/benchmark/results.md).
|
| 201 |
|
| 202 |
-
|
|
|
|
|
|
|
| 203 |
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
|
|
|
| 209 |
```
|
| 210 |
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
## Inference Performance
|
| 214 |
-
|
| 215 |
-
Checkpoint-specific GS1-9B latency, speedup, and GPU memory measurements are not provided in the current repository. The published complete-call measurements cover 0.8B, 2B, and 4B. [Measurement protocol and available results](https://github.com/bigai-nlco/GS1/blob/main/benchmark/latency.md).
|
| 216 |
|
| 217 |
-
|
| 218 |
-
> Default loading uses eager inference. CUDA Graph and shared execution require a matching runtime qualification; gains depend on request shape. Follow the [CUDA Graph setup](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.md#cuda-graph-inference) before comparing these timings.
|
| 219 |
|
| 220 |
-
|
| 221 |
|
| 222 |
-
The
|
| 223 |
|
| 224 |
-
|
| 225 |
-
* Probabilities may be miscalibrated under distribution shift. Validate accuracy, calibration, and abstention thresholds on your workload.
|
| 226 |
-
* Base-model and training-data biases may affect decisions; use human review for consequential applications. Reported benchmarks are text-only and do not establish multimodal or safety performance.
|
| 227 |
-
* Requests exceeding configured context or admission budgets fail explicitly; state and candidates are not silently truncated.
|
| 228 |
|
| 229 |
-
|
| 230 |
|
| 231 |
-
|
| 232 |
|
| 233 |
-
|
| 234 |
-
@techreport{zheng2026gs1,
|
| 235 |
-
title={GS1: Fast, Faithful, Frontier System One Intelligence},
|
| 236 |
-
author={Zheng, Zilong and Liu, Yang and Qin, Shuhan and Lin, Ziyong and Li, Jiaqi and Jia, Zixia and Zhu, Song-Chun},
|
| 237 |
-
institution={BIGAI},
|
| 238 |
-
year={2026}
|
| 239 |
-
}
|
| 240 |
-
```
|
|
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
library_name: transformers
|
| 4 |
+
tags:
|
| 5 |
+
- gs1
|
| 6 |
+
- decision-model
|
| 7 |
+
- custom_code
|
| 8 |
+
- system-one
|
| 9 |
+
- safetensors
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
---
|
| 11 |
+
# GS1-9B
|
|
|
|
|
|
|
| 12 |
|
| 13 |
+
[English](README.md) · [简体中文](README.zh-CN.md)
|
| 14 |
|
| 15 |
+

|
| 16 |
|
| 17 |
+
**Fast, Faithful, Frontier.** GS1-9B is a native typed decision model. Supply state, instructions and runtime candidate meanings to obtain **Choice**, **Noul** and **Score** distributions with zero generated output tokens.
|
| 18 |
|
| 19 |
+
[Code and optimized runtime](https://github.com/bigai-nlco/GS1) · [Model family](https://huggingface.co/collections/bigai/gs1)
|
|
|
|
| 20 |
|
| 21 |
+
| Measurement | GS1-9B |
|
| 22 |
+
|---|---:|
|
| 23 |
+
| Stored parameters | 8,829,078,768 |
|
| 24 |
+
| Hidden width / decoder layers | 4,096 / 32 |
|
| 25 |
+
| Decision attention heads / routing layers | 16 / 2 |
|
| 26 |
+
| Decision projection size | 2,048 |
|
| 27 |
+
| Maximum branch context | 32,768 tokens |
|
| 28 |
+
| Checkpoint optimizer step | 1,900 |
|
| 29 |
+
| Public Decision Index 0.3, 37 benchmarks | **70.45** |
|
| 30 |
|
| 31 |
+
## Decision Index 0.3
|
| 32 |
|
| 33 |
+
Complete public-panel evaluation on 10 November 2026: **70.45 / 100** after chance correction; raw index **77.46**. All **110,087 requests / 272,229 decisions** across 37 benchmarks completed successfully, with **100% coverage** and zero failed requests. The pinned official scorer (`9eb2dbe2a358004c8782c66e40a83ac07b953fec`) and the local scorer agree exactly. This is the public index, separate from the private Full score.
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
+
BF16 native reference computation preserves the original FP32 text/head and BF16 vision weights. Context admission was raised to 32,768 tokens; the longest frozen branch is 22,650 tokens. Inputs are unchanged and untruncated. [Full results, per-benchmark scores and provenance](assets/results/decision-index-0.3.json).
|
| 36 |
|
| 37 |
+
## GS1 framework inference
|
| 38 |
|
| 39 |
+
Use the existing GS1 environment and this local model folder:
|
|
|
|
|
|
|
| 40 |
|
| 41 |
```bash
|
| 42 |
+
cd ~/GS1
|
| 43 |
+
.venv/bin/python - <<'PYTHON'
|
| 44 |
+
from gs1 import Engine
|
| 45 |
|
| 46 |
+
engine = Engine.from_pretrained(
|
| 47 |
+
"/data/shared/GS1/GS1-9B", device="cuda:0", precision="bf16", local_files_only=True,
|
| 48 |
+
)
|
| 49 |
+
result = engine.evaluate(
|
| 50 |
+
{
|
| 51 |
+
"state": "A customer was charged twice and asks for a refund.",
|
| 52 |
+
"questions": {
|
| 53 |
+
"team": {
|
| 54 |
+
"type": "choice",
|
| 55 |
+
"instructions": "Choose the team that should handle this ticket.",
|
| 56 |
+
"criteria": {"billing": "Payments and refunds", "technical": "Software problems"},
|
| 57 |
+
},
|
| 58 |
+
"refund": {"type": "noul", "instructions": "Does the customer request a refund?"},
|
| 59 |
+
"urgency": {
|
| 60 |
+
"type": "score", "instructions": "Rate the urgency.",
|
| 61 |
+
"criteria": ["Routine", "Soon", "Immediate"],
|
| 62 |
+
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
},
|
| 64 |
},
|
| 65 |
+
resolved_model="GS1-9B",
|
| 66 |
)
|
| 67 |
+
print(result.model_dump(mode="json"))
|
| 68 |
+
engine.close()
|
| 69 |
+
PYTHON
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
```
|
| 71 |
|
| 72 |
+
For the HTTP service, run from `~/GS1`:
|
| 73 |
|
| 74 |
```bash
|
| 75 |
+
.venv/bin/gs1 serve --model /data/shared/GS1/GS1-9B --model-name GS1-9B --device cuda:0 \
|
| 76 |
+
--max-context 32768 --batch-tokens 32768 \
|
| 77 |
+
--max-attention-bytes 549755813888 --deadline-seconds 120
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
```
|
| 79 |
|
| 80 |
+
The reference engine works directly from this directory. Generate a new model/hardware/source-specific runtime profile before enabling shared inference or CUDA Graphs; profiles for other checkpoints do not apply. See the [inference guide](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.md).
|
| 81 |
|
| 82 |
+
## Standalone Hugging Face inference
|
| 83 |
|
| 84 |
+
Python 3.11+. Install this folder's `requirements.txt`. The GS1 source repository is optional for this route.
|
| 85 |
|
| 86 |
```python
|
| 87 |
+
import torch
|
| 88 |
+
from transformers import AutoModel
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
+
model = AutoModel.from_pretrained(
|
| 91 |
+
"/data/shared/GS1/GS1-9B", trust_remote_code=True, local_files_only=True,
|
| 92 |
+
).eval().to("cuda:0")
|
| 93 |
|
| 94 |
+
with torch.autocast("cuda", dtype=torch.bfloat16, cache_enabled=False):
|
| 95 |
+
result = model.decide(
|
| 96 |
+
state="A customer asks for a refund.",
|
| 97 |
+
questions={"refund": {"type": "noul", "instructions": "Does the customer request a refund?"}},
|
| 98 |
+
)
|
| 99 |
+
print(result["answers"])
|
| 100 |
```
|
| 101 |
|
| 102 |
+
Disabling the autocast weight cache bounds temporary GPU memory while preserving the original mixed storage dtypes. Default loading keeps FP32 text/decision tensors and BF16 vision tensors; final logits and probability reductions use FP32. The stored weight payload is approximately 32.04 GiB; inference also requires working memory.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
+
## Architecture and checkpoint provenance
|
|
|
|
| 105 |
|
| 106 |
+

|
| 107 |
|
| 108 |
+
The model combines a Qwen3.5 hybrid GDN/attention backbone, segmented bidirectional full attention, Attention Residuals, a cross-attention decision head and the retained vision tower. Candidate meanings are supplied at inference time. It has no language-generation output head.
|
| 109 |
|
| 110 |
+
This package retains the supplied checkpoint at optimizer step 1,900, with 3,890,593 questions seen. All 820 tensor names, shapes, values and storage dtypes are unchanged. The 2,048-dimensional projection and original tensor precision are retained. Context admission was extended from 16,384 to 32,768 tokens to cover all frozen Decision Index 0.3 inputs without truncation. Structural embedding offsets were already merged in the supplied checkpoint and are not applied again.
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
+
The directory follows the same standalone Hugging Face layout as GS1-0.8B, GS1-2B and GS1-4B: indexed safetensors shards, `config.json` with `auto_map`, structural token IDs and `inference_limits`, tokenizer/processor assets and native model code. The embedding tensor exceeds 4 GB and occupies its own shard. The supplied directory is overwritten without a backup. Obsolete conversion scripts and manifests are removed; local verification results are stored separately under `../release-validation-20261110-gs1-9b`.
|
| 113 |
|
| 114 |
+
## License
|
| 115 |
|
| 116 |
+
Released code and weights follow [Apache-2.0](LICENSE); [upstream attribution](NOTICE) is included. No Hub upload is performed by this local refactor.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.zh-CN.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GS1-9B
|
| 2 |
+
|
| 3 |
+
[English](README.md) · [简体中文](README.zh-CN.md)
|
| 4 |
+
|
| 5 |
+

|
| 6 |
+
|
| 7 |
+
**Fast, Faithful, Frontier.** GS1-9B 是原生类型化决策模型。输入状态、问题和运行时定义的候选含义,直接获得 **Choice、Noul、Score** 分布,生成输出 token 数为零。
|
| 8 |
+
|
| 9 |
+
[代码与优化推理](https://github.com/bigai-nlco/GS1) · [模型合集](https://huggingface.co/collections/bigai/gs1)
|
| 10 |
+
|
| 11 |
+
| 指标 | GS1-9B |
|
| 12 |
+
|---|---:|
|
| 13 |
+
| 实际存储参数量 | 8,829,078,768 |
|
| 14 |
+
| 隐藏宽度 / 解码层数 | 4,096 / 32 |
|
| 15 |
+
| 决策注意力头数 / 路由层数 | 16 / 2 |
|
| 16 |
+
| 决策投影维度 | 2,048 |
|
| 17 |
+
| 完整分支上下文上限 | 32,768 token |
|
| 18 |
+
| 检查点优化步数 | 1,900 |
|
| 19 |
+
| 公开 Decision Index 0.3,37 项基准 | **70.45** |
|
| 20 |
+
|
| 21 |
+
## Decision Index 0.3
|
| 22 |
+
|
| 23 |
+
2026 年 11 月 10 日完成完整公开面板:随机校正分数 **70.45 / 100**,原始指数 **77.46**。37 项基准的 **110,087 个请求、272,229 个决策**全部完成,覆盖率 **100%**,最终失败请求为零。固定 revision `9eb2dbe2a358004c8782c66e40a83ac07b953fec` 的官方计分器与本地计分器结果完全一致。该分数为公共指数,与私有 Full 分数分开。
|
| 24 |
+
|
| 25 |
+
使用 BF16 原生参考计算,保留原始 FP32 文本/决策头权重和 BF16 视觉权重。上下文准入提高至 32,768 token;冻结输入的最长分支为 22,650 token,输入内容未改变、未截断。[完整结果、各基准分数与来源记录](assets/results/decision-index-0.3.json)。
|
| 26 |
+
|
| 27 |
+
## 通过 GS1 框架调用
|
| 28 |
+
|
| 29 |
+
使用 `~/GS1` 下的现有环境和本地目录:
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
cd ~/GS1
|
| 33 |
+
.venv/bin/python - <<'PYTHON'
|
| 34 |
+
from gs1 import Engine
|
| 35 |
+
|
| 36 |
+
engine = Engine.from_pretrained(
|
| 37 |
+
"/data/shared/GS1/GS1-9B", device="cuda:0", precision="bf16", local_files_only=True,
|
| 38 |
+
)
|
| 39 |
+
result = engine.evaluate(
|
| 40 |
+
{
|
| 41 |
+
"state": "客户被重复扣费,要求退款。",
|
| 42 |
+
"questions": {
|
| 43 |
+
"team": {
|
| 44 |
+
"type": "choice", "instructions": "选择处理工单的团队。",
|
| 45 |
+
"criteria": {"billing": "支付和退款", "technical": "软件故障"},
|
| 46 |
+
},
|
| 47 |
+
"refund": {"type": "noul", "instructions": "客户是否要求退款?"},
|
| 48 |
+
"urgency": {
|
| 49 |
+
"type": "score", "instructions": "评估处理紧急程度。",
|
| 50 |
+
"criteria": ["常规", "尽快", "立即"],
|
| 51 |
+
},
|
| 52 |
+
},
|
| 53 |
+
},
|
| 54 |
+
resolved_model="GS1-9B",
|
| 55 |
+
)
|
| 56 |
+
print(result.model_dump(mode="json"))
|
| 57 |
+
engine.close()
|
| 58 |
+
PYTHON
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
HTTP 服务可在 `~/GS1` 下启动:
|
| 62 |
+
|
| 63 |
+
```bash
|
| 64 |
+
.venv/bin/gs1 serve --model /data/shared/GS1/GS1-9B --model-name GS1-9B --device cuda:0 \
|
| 65 |
+
--max-context 32768 --batch-tokens 32768 \
|
| 66 |
+
--max-attention-bytes 549755813888 --deadline-seconds 120
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
参考引擎可以直接加载。启用状态共享或 CUDA Graph 前,应为当前模型、源码和硬件重新生成运行时资格文件;其他检查点的资格文件不能复用。详见[推理指南](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.zh-CN.md)。
|
| 70 |
+
|
| 71 |
+
## Hugging Face 独立加载
|
| 72 |
+
|
| 73 |
+
使用 Python 3.11+,安装本目录 `requirements.txt` 的依赖。此路径无需安装 GS1 源码仓库。
|
| 74 |
+
|
| 75 |
+
```python
|
| 76 |
+
import torch
|
| 77 |
+
from transformers import AutoModel
|
| 78 |
+
|
| 79 |
+
model = AutoModel.from_pretrained(
|
| 80 |
+
"/data/shared/GS1/GS1-9B", trust_remote_code=True, local_files_only=True,
|
| 81 |
+
).eval().to("cuda:0")
|
| 82 |
+
|
| 83 |
+
with torch.autocast("cuda", dtype=torch.bfloat16, cache_enabled=False):
|
| 84 |
+
result = model.decide(
|
| 85 |
+
state="客户要求退款。",
|
| 86 |
+
questions={"refund": {"type": "noul", "instructions": "客户是否要求退款?"}},
|
| 87 |
+
)
|
| 88 |
+
print(result["answers"])
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
关闭 autocast 权重缓存可控制临时显存占用,同时保留原始混合存储精度。文本和决策头权重保持 FP32,视觉塔保持 BF16,最终 logits 和概率归约使用 FP32。持久权重约 32.04 GiB,推理另需工作显存。
|
| 92 |
+
|
| 93 |
+
## 架构与检查点来源
|
| 94 |
+
|
| 95 |
+

|
| 96 |
+
|
| 97 |
+
模型包含 Qwen3.5 混合 GDN/注意力骨干、分段双向全注意力、Attention Residuals、原生交叉注意力决策头及保留的视觉塔。候选含义在推理时提供,不包含语言生成输出头。
|
| 98 |
+
|
| 99 |
+
本目录保留所提供的第 1,900 步检查点,训练记录中的已见问题数为 3,890,593。820 个张量的名称、形状、数值和存储 dtype 完整保留;决策投影维度与原始张量精度保持不变;为完整兼容 Decision Index 0.3 输入,上下文准入从 16,384 提高至 32,768 token,不截断输入。结构 token 的 embedding 偏移已在原检查点中合并,本次不重复合并。
|
| 100 |
+
|
| 101 |
+
目录采用与 GS1-0.8B、GS1-2B、GS1-4B 相同的独立 Hugging Face 格式:标准 safetensors 分片与索引、包含 `auto_map`、结构 token ID 和 `inference_limits` 的 `config.json`,以及 tokenizer、processor 和模型代码。Embedding 单个张量超过 4 GB,独占一个分片。按要求直接覆盖原目录,不保留备份,并删除过时的转换脚本和 manifest。验证记录单独保存在 `../release-validation-20261110-gs1-9b`。
|
| 102 |
+
|
| 103 |
+
## 许可
|
| 104 |
+
|
| 105 |
+
代码与发布权重采用 [Apache-2.0](LICENSE),保留[上游署名](NOTICE)。本次仅在本地重构目录,未上传至 Hub。
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 150 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
compiler.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compile typed decisions into XML prompts and execute native decision branches."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
from html import escape
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from .config import INFERENCE_LIMITS, DecisionConfig
|
| 10 |
+
from .schema import DecisionRequest, candidate_ids, noul_descriptions
|
| 11 |
+
from .util import canonical
|
| 12 |
+
|
| 13 |
+
XML_STATE, XML_STATE_END = "<state>", "</state>"
|
| 14 |
+
XML_QUESTION, XML_QUESTION_END = "<question>", "</question>"
|
| 15 |
+
XML_OPTION, XML_OPTION_END = "<option>", "</option>"
|
| 16 |
+
XML_TAGS = (XML_STATE, XML_STATE_END, XML_QUESTION, XML_QUESTION_END, XML_OPTION, XML_OPTION_END)
|
| 17 |
+
DECISION_TOKEN = "<|decide|>"
|
| 18 |
+
NO_THINKING_PREFIX = "<think>\n\n</think>\n\n"
|
| 19 |
+
COMPILER = "xml_decision"
|
| 20 |
+
POSITIONS = "branch_local"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def structural_tokens():
|
| 24 |
+
return [*XML_TAGS, DECISION_TOKEN]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def no_thinking_token_ids(tokenizer):
|
| 28 |
+
ids = tuple(tokenizer.encode(NO_THINKING_PREFIX, add_special_tokens=False))
|
| 29 |
+
if (
|
| 30 |
+
not ids
|
| 31 |
+
or tokenizer.decode(ids, skip_special_tokens=False, clean_up_tokenization_spaces=False)
|
| 32 |
+
!= NO_THINKING_PREFIX
|
| 33 |
+
):
|
| 34 |
+
raise ValueError("Tokenizer cannot encode the no-thinking prefix exactly")
|
| 35 |
+
return ids
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass(frozen=True)
|
| 39 |
+
class Branch:
|
| 40 |
+
question_id: str
|
| 41 |
+
question_type: str
|
| 42 |
+
branch_id: int
|
| 43 |
+
suffix: tuple[int, ...]
|
| 44 |
+
token_roles: tuple[str, ...]
|
| 45 |
+
logical_positions: tuple[int, ...]
|
| 46 |
+
option_ends: tuple[int, ...]
|
| 47 |
+
candidates: tuple[str, ...]
|
| 48 |
+
decision_position: int
|
| 49 |
+
option_spans: tuple[tuple[int, int], ...]
|
| 50 |
+
option_starts: tuple[int, ...]
|
| 51 |
+
question_end_position: int
|
| 52 |
+
question_start_position: int
|
| 53 |
+
question_span: tuple[int, int]
|
| 54 |
+
state_evidence_start: int = 0
|
| 55 |
+
visual_inputs: dict | None = field(default=None, compare=False, repr=False)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass(frozen=True)
|
| 59 |
+
class CompiledRequest:
|
| 60 |
+
state: tuple[int, ...]
|
| 61 |
+
branches: tuple[Branch, ...]
|
| 62 |
+
question_order: tuple[str, ...]
|
| 63 |
+
candidate_order: dict[str, tuple[str, ...]]
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def input_tokens(self):
|
| 67 |
+
return len(self.state) + sum(len(b.suffix) for b in self.branches)
|
| 68 |
+
|
| 69 |
+
@property
|
| 70 |
+
def longest_path(self):
|
| 71 |
+
return max(len(self.state) + len(b.suffix) for b in self.branches)
|
| 72 |
+
|
| 73 |
+
@property
|
| 74 |
+
def query_tokens(self):
|
| 75 |
+
return len(self.branches)
|
| 76 |
+
|
| 77 |
+
@property
|
| 78 |
+
def candidate_valid_mask(self):
|
| 79 |
+
width = max(map(len, self.candidate_order.values()))
|
| 80 |
+
return tuple(
|
| 81 |
+
tuple(k < len(self.candidate_order[q]) for k in range(width)) for q in self.question_order
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class Compiler:
|
| 86 |
+
def __init__(
|
| 87 |
+
self,
|
| 88 |
+
tokenizer,
|
| 89 |
+
max_context=INFERENCE_LIMITS["max_context"],
|
| 90 |
+
max_questions=512,
|
| 91 |
+
max_request_tokens=131072,
|
| 92 |
+
decision=None,
|
| 93 |
+
max_branches=5120,
|
| 94 |
+
processor=None,
|
| 95 |
+
):
|
| 96 |
+
self.tokenizer = tokenizer
|
| 97 |
+
self.decision = DecisionConfig.model_validate(decision or {})
|
| 98 |
+
self.max_context = max_context
|
| 99 |
+
self.max_questions = max_questions
|
| 100 |
+
self.max_request_tokens = max_request_tokens
|
| 101 |
+
self.max_branches = max_branches
|
| 102 |
+
self.processor = processor
|
| 103 |
+
tokens = structural_tokens()
|
| 104 |
+
self.controls = dict(zip(tokens, tokenizer.convert_tokens_to_ids(tokens), strict=True))
|
| 105 |
+
if len(set(self.controls.values())) != len(tokens) or any(
|
| 106 |
+
type(i) is not int or i < 0 or i == tokenizer.unk_token_id or i not in tokenizer.all_special_ids
|
| 107 |
+
for i in self.controls.values()
|
| 108 |
+
):
|
| 109 |
+
raise ValueError("Tokenizer requires the seven distinct GS1 special tokens")
|
| 110 |
+
self.xml_tags = {tag: (self.controls[tag],) for tag in XML_TAGS}
|
| 111 |
+
self.xml_newline = tuple(tokenizer.encode("\n", add_special_tokens=False))
|
| 112 |
+
self.no_thinking_ids = no_thinking_token_ids(tokenizer)
|
| 113 |
+
|
| 114 |
+
@staticmethod
|
| 115 |
+
def _xml_text(value):
|
| 116 |
+
return escape(value if isinstance(value, str) else canonical(value), quote=False)
|
| 117 |
+
|
| 118 |
+
@staticmethod
|
| 119 |
+
def _visual_state(state):
|
| 120 |
+
return isinstance(state, list) and any(
|
| 121 |
+
isinstance(item, dict) and item.get("type") == "image" for item in state
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
@staticmethod
|
| 125 |
+
def choice_option_text(name, description):
|
| 126 |
+
value = name if description is None else description
|
| 127 |
+
return f"{name}: {value if isinstance(value, str) else canonical(value)}"
|
| 128 |
+
|
| 129 |
+
def _xml_positions(self, token_ids, option_count):
|
| 130 |
+
found = {
|
| 131 |
+
tag: [i for i, value in enumerate(token_ids) if value == self.controls[tag]] for tag in XML_TAGS
|
| 132 |
+
}
|
| 133 |
+
if any(len(found[tag]) != 1 for tag in XML_TAGS[:4]) or any(
|
| 134 |
+
len(found[tag]) != option_count for tag in XML_TAGS[4:]
|
| 135 |
+
):
|
| 136 |
+
raise ValueError("Tokenizer changed the XML prompt tag positions")
|
| 137 |
+
ordered = [found[tag][0] for tag in XML_TAGS[:4]]
|
| 138 |
+
ordered.extend(i for pair in zip(found[XML_OPTION], found[XML_OPTION_END], strict=True) for i in pair)
|
| 139 |
+
if ordered != sorted(set(ordered)):
|
| 140 |
+
raise ValueError("XML prompt tags are out of order")
|
| 141 |
+
prefix = found[XML_QUESTION][0]
|
| 142 |
+
|
| 143 |
+
def span(opening, closing):
|
| 144 |
+
begin, end = opening + 1, closing
|
| 145 |
+
width = len(self.xml_newline)
|
| 146 |
+
if width and tuple(token_ids[begin : begin + width]) == self.xml_newline:
|
| 147 |
+
begin += width
|
| 148 |
+
if width and tuple(token_ids[end - width : end]) == self.xml_newline:
|
| 149 |
+
end -= width
|
| 150 |
+
if begin >= end:
|
| 151 |
+
begin, end = closing, closing + 1
|
| 152 |
+
return begin - prefix, end - prefix
|
| 153 |
+
|
| 154 |
+
return dict(
|
| 155 |
+
prefix_end=prefix,
|
| 156 |
+
state_evidence_start=found[XML_STATE][0],
|
| 157 |
+
question_start_position=0,
|
| 158 |
+
question_end_position=found[XML_QUESTION_END][0] - prefix,
|
| 159 |
+
question_span=span(found[XML_QUESTION][0], found[XML_QUESTION_END][0]),
|
| 160 |
+
option_starts=tuple(i - prefix for i in found[XML_OPTION]),
|
| 161 |
+
option_ends=tuple(i - prefix for i in found[XML_OPTION_END]),
|
| 162 |
+
option_spans=tuple(
|
| 163 |
+
span(a, b) for a, b in zip(found[XML_OPTION], found[XML_OPTION_END], strict=True)
|
| 164 |
+
),
|
| 165 |
+
first_option=found[XML_OPTION][0] - prefix,
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
def _processor(self):
|
| 169 |
+
if self.processor is None:
|
| 170 |
+
raise ValueError("Visual input requires the GS1 processor")
|
| 171 |
+
if isinstance(self.processor, (str, Path)):
|
| 172 |
+
from transformers import AutoProcessor
|
| 173 |
+
|
| 174 |
+
self.processor = AutoProcessor.from_pretrained(
|
| 175 |
+
self.processor, local_files_only=True, trust_remote_code=True
|
| 176 |
+
)
|
| 177 |
+
# Fresh Qwen processor assets precede the seven GS1 structural tokens.
|
| 178 |
+
self.processor.tokenizer = self.tokenizer
|
| 179 |
+
return self.processor
|
| 180 |
+
|
| 181 |
+
def _contents(self, state):
|
| 182 |
+
if not self._visual_state(state):
|
| 183 |
+
return f"{XML_STATE}\n{self._xml_text(state)}\n{XML_STATE_END}"
|
| 184 |
+
contents = [{"type": "text", "text": f"{XML_STATE}\n"}]
|
| 185 |
+
for item in state:
|
| 186 |
+
if item.get("type") == "text" and isinstance(item.get("text"), str):
|
| 187 |
+
contents.append({"type": "text", "text": self._xml_text(item["text"])})
|
| 188 |
+
elif (
|
| 189 |
+
set(item) == {"type", "image"}
|
| 190 |
+
and isinstance(item.get("image"), str)
|
| 191 |
+
and item["image"].startswith("data:image/")
|
| 192 |
+
and ";base64," in item["image"]
|
| 193 |
+
):
|
| 194 |
+
contents.append(item)
|
| 195 |
+
else:
|
| 196 |
+
raise ValueError("Vision state requires text and inline base64 images")
|
| 197 |
+
return [*contents, {"type": "text", "text": f"\n{XML_STATE_END}"}]
|
| 198 |
+
|
| 199 |
+
def compile(self, request: DecisionRequest):
|
| 200 |
+
if len(request.questions) > self.max_questions:
|
| 201 |
+
raise ValueError(f"questions exceeds {self.max_questions}")
|
| 202 |
+
visual = self._visual_state(request.state)
|
| 203 |
+
contents = self._contents(request.state)
|
| 204 |
+
template = self._processor() if visual else self.tokenizer
|
| 205 |
+
branches, candidates, state = [], {}, None
|
| 206 |
+
for qid, question in request.questions.items():
|
| 207 |
+
ids = tuple(candidate_ids(question))
|
| 208 |
+
candidates[qid] = ids
|
| 209 |
+
if question.type == "choice":
|
| 210 |
+
options = [self.choice_option_text(k, d) for k, d in question.criteria.items()]
|
| 211 |
+
elif question.type == "noul":
|
| 212 |
+
options = list(noul_descriptions(question))
|
| 213 |
+
else:
|
| 214 |
+
options = list(question.criteria)
|
| 215 |
+
prompt = (
|
| 216 |
+
f"\n{XML_QUESTION}\n{self._xml_text(question.instructions)}\n{XML_QUESTION_END}"
|
| 217 |
+
+ "".join(f"\n{XML_OPTION}\n{self._xml_text(o)}\n{XML_OPTION_END}" for o in options)
|
| 218 |
+
)
|
| 219 |
+
content = [*contents, {"type": "text", "text": prompt}] if visual else contents + prompt
|
| 220 |
+
rendered = template.apply_chat_template(
|
| 221 |
+
[{"role": "user", "content": content}],
|
| 222 |
+
add_generation_prompt=True,
|
| 223 |
+
enable_thinking=False,
|
| 224 |
+
tokenize=False,
|
| 225 |
+
)
|
| 226 |
+
payload = None
|
| 227 |
+
if visual:
|
| 228 |
+
encoded = template(
|
| 229 |
+
text=rendered,
|
| 230 |
+
images=[[i["image"] for i in request.state if i["type"] == "image"]],
|
| 231 |
+
return_tensors="pt",
|
| 232 |
+
)
|
| 233 |
+
token_ids = encoded["input_ids"][0].tolist()
|
| 234 |
+
else:
|
| 235 |
+
token_ids = self.tokenizer.encode(rendered, add_special_tokens=False)
|
| 236 |
+
if tuple(token_ids[-len(self.no_thinking_ids) :]) != self.no_thinking_ids:
|
| 237 |
+
raise ValueError("Chat template changed the no-thinking assistant prefix")
|
| 238 |
+
token_ids.append(self.controls[DECISION_TOKEN])
|
| 239 |
+
markers = self._xml_positions(token_ids, len(ids))
|
| 240 |
+
prefix = markers.pop("prefix_end")
|
| 241 |
+
first_option = markers.pop("first_option")
|
| 242 |
+
current_state = tuple(token_ids[:prefix])
|
| 243 |
+
if state is None:
|
| 244 |
+
state = current_state
|
| 245 |
+
elif state != current_state:
|
| 246 |
+
raise ValueError("State prefix differs across question branches")
|
| 247 |
+
suffix = tuple(token_ids[prefix:])
|
| 248 |
+
if visual:
|
| 249 |
+
payload = {
|
| 250 |
+
k: v
|
| 251 |
+
for k, v in encoded.items()
|
| 252 |
+
if k
|
| 253 |
+
in {"input_ids", "attention_mask", "pixel_values", "image_grid_thw", "mm_token_type_ids"}
|
| 254 |
+
}
|
| 255 |
+
payload["input_ids"] = torch.tensor([token_ids], dtype=torch.long)
|
| 256 |
+
for key, fill in (("attention_mask", 1), ("mm_token_type_ids", 0)):
|
| 257 |
+
if key in payload:
|
| 258 |
+
value = payload[key]
|
| 259 |
+
payload[key] = torch.cat((value, torch.full((1, 1), fill, dtype=value.dtype)), dim=-1)
|
| 260 |
+
branches.append(
|
| 261 |
+
Branch(
|
| 262 |
+
qid,
|
| 263 |
+
question.type,
|
| 264 |
+
len(branches),
|
| 265 |
+
suffix,
|
| 266 |
+
tuple(
|
| 267 |
+
["question_context"] * first_option
|
| 268 |
+
+ ["candidate_context"] * (len(suffix) - first_option - 1)
|
| 269 |
+
+ ["decision_head_query"]
|
| 270 |
+
),
|
| 271 |
+
tuple(range(prefix, len(token_ids))),
|
| 272 |
+
markers.pop("option_ends"),
|
| 273 |
+
ids,
|
| 274 |
+
len(suffix) - 1,
|
| 275 |
+
**markers,
|
| 276 |
+
visual_inputs=payload,
|
| 277 |
+
)
|
| 278 |
+
)
|
| 279 |
+
result = CompiledRequest(state, tuple(branches), tuple(request.questions), candidates)
|
| 280 |
+
if result.longest_path > self.max_context:
|
| 281 |
+
raise ValueError(f"Branch exceeds {self.max_context} tokens; no truncation")
|
| 282 |
+
if result.input_tokens > self.max_request_tokens:
|
| 283 |
+
raise ValueError(f"Request exceeds {self.max_request_tokens} tokens; no truncation")
|
| 284 |
+
if len(branches) > self.max_branches:
|
| 285 |
+
raise ValueError(f"Physical branches exceed {self.max_branches}")
|
| 286 |
+
return result
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
@dataclass
|
| 290 |
+
class PreparedBatch:
|
| 291 |
+
ids: torch.Tensor
|
| 292 |
+
positions: torch.Tensor
|
| 293 |
+
valid: torch.Tensor
|
| 294 |
+
roles: torch.Tensor
|
| 295 |
+
branches: tuple
|
| 296 |
+
offsets: tuple
|
| 297 |
+
starts: tuple
|
| 298 |
+
layout: tuple
|
| 299 |
+
|
| 300 |
+
def forward(self, model, *, readout=None, execution_policy=None):
|
| 301 |
+
hidden = model.language_model(
|
| 302 |
+
inputs_embeds=model.get_input_embeddings()(self.ids),
|
| 303 |
+
position_ids=self.positions,
|
| 304 |
+
terminal_query=True,
|
| 305 |
+
prepared_layout=self.layout,
|
| 306 |
+
execution_policy=execution_policy,
|
| 307 |
+
).last_hidden_state
|
| 308 |
+
values = [
|
| 309 |
+
(readout or model.readout)(hidden[row, begin:], branch, offset)
|
| 310 |
+
for row, (begin, branch, offset) in enumerate(
|
| 311 |
+
zip(self.starts, self.branches, self.offsets, strict=True)
|
| 312 |
+
)
|
| 313 |
+
]
|
| 314 |
+
return values
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def prepare_batch(model, rows, pad_token_id=0):
|
| 318 |
+
"""Collate text once; no scalar device reads remain in PreparedBatch.forward."""
|
| 319 |
+
if not rows or any(branch.visual_inputs is not None for _, branch in rows):
|
| 320 |
+
raise ValueError("Prepared GS1 batches require nonempty text branches")
|
| 321 |
+
device = model.get_input_embeddings().weight.device
|
| 322 |
+
lengths = [len(state) + len(branch.suffix) for state, branch in rows]
|
| 323 |
+
width = max(lengths)
|
| 324 |
+
ids = torch.full((len(rows), width), pad_token_id, dtype=torch.long, device=device)
|
| 325 |
+
valid = torch.zeros_like(ids, dtype=torch.bool)
|
| 326 |
+
positions = torch.zeros_like(ids)
|
| 327 |
+
roles = torch.full_like(ids, -1)
|
| 328 |
+
starts = []
|
| 329 |
+
for row, ((state, branch), length) in enumerate(zip(rows, lengths, strict=True)):
|
| 330 |
+
begin, offset = width - length, len(state)
|
| 331 |
+
starts.append(begin)
|
| 332 |
+
ids[row, begin:] = torch.tensor(state + branch.suffix, device=device)
|
| 333 |
+
valid[row, begin:] = True
|
| 334 |
+
positions[row, begin:] = torch.arange(length, device=device)
|
| 335 |
+
roles[row, begin : begin + offset] = 0
|
| 336 |
+
roles[row, begin + offset :] = torch.tensor(
|
| 337 |
+
[
|
| 338 |
+
1 if role == "question_context" else 2 if role == "candidate_context" else 3
|
| 339 |
+
for role in branch.token_roles
|
| 340 |
+
],
|
| 341 |
+
device=device,
|
| 342 |
+
)
|
| 343 |
+
hidden = model.get_input_embeddings()(ids)
|
| 344 |
+
layout = model.language_model.prepare_encode(
|
| 345 |
+
hidden, positions, roles, terminal_query=True, attention_mask=valid
|
| 346 |
+
)
|
| 347 |
+
return PreparedBatch(
|
| 348 |
+
ids,
|
| 349 |
+
positions,
|
| 350 |
+
valid,
|
| 351 |
+
roles,
|
| 352 |
+
tuple(b for _, b in rows),
|
| 353 |
+
tuple(len(s) for s, _ in rows),
|
| 354 |
+
tuple(starts),
|
| 355 |
+
layout,
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def branch_logits(model, rows, *, pad_token_id=0, readout=None, execution_policy=None):
|
| 360 |
+
"""Execute text branches in independent, genuinely batched left-padded rows."""
|
| 361 |
+
if not rows:
|
| 362 |
+
return []
|
| 363 |
+
device = model.get_input_embeddings().weight.device
|
| 364 |
+
outputs = [None] * len(rows)
|
| 365 |
+
text = [
|
| 366 |
+
(index, state, branch) for index, (state, branch) in enumerate(rows) if branch.visual_inputs is None
|
| 367 |
+
]
|
| 368 |
+
if text:
|
| 369 |
+
prepared = prepare_batch(model, [(state, branch) for _, state, branch in text], pad_token_id)
|
| 370 |
+
logits = prepared.forward(model, readout=readout, execution_policy=execution_policy)
|
| 371 |
+
for row, (index, _, branch) in enumerate(text):
|
| 372 |
+
outputs[index] = logits[row]
|
| 373 |
+
for index, (state, branch) in enumerate(rows):
|
| 374 |
+
if branch.visual_inputs is not None:
|
| 375 |
+
payload = {
|
| 376 |
+
key: value.to(device) if isinstance(value, torch.Tensor) else value
|
| 377 |
+
for key, value in branch.visual_inputs.items()
|
| 378 |
+
}
|
| 379 |
+
outputs[index] = model(
|
| 380 |
+
**payload, branch=branch, state_length=len(state), readout_callback=readout
|
| 381 |
+
).logits
|
| 382 |
+
return outputs
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def assemble_logits(compiled, outputs):
|
| 386 |
+
if len(outputs) != len(compiled.branches):
|
| 387 |
+
raise ValueError("Missing physical branch results")
|
| 388 |
+
result = {}
|
| 389 |
+
for branch, output in zip(compiled.branches, outputs, strict=True):
|
| 390 |
+
if output.shape != (len(branch.candidates),) or branch.question_id in result:
|
| 391 |
+
raise ValueError("Malformed or duplicate single-decision result")
|
| 392 |
+
result[branch.question_id] = output
|
| 393 |
+
if set(result) != set(compiled.question_order):
|
| 394 |
+
raise ValueError("Missing logical question results")
|
| 395 |
+
return {key: result[key] for key in compiled.question_order}
|
config.json
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GS1Model"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "modular_gs1.GS1Config",
|
| 7 |
+
"AutoModel": "modular_gs1.GS1Model"
|
| 8 |
+
},
|
| 9 |
+
"calibration": {
|
| 10 |
+
"log_temperature_bound": 2.995732273553991,
|
| 11 |
+
"mode": "identity"
|
| 12 |
+
},
|
| 13 |
+
"decision": {
|
| 14 |
+
"attnres": true,
|
| 15 |
+
"attnres_eps": 1e-06,
|
| 16 |
+
"attnres_layers": null,
|
| 17 |
+
"bidirectional": true,
|
| 18 |
+
"bidirectional_layers": null,
|
| 19 |
+
"cross_attention_heads": 16,
|
| 20 |
+
"cross_attention_layers": 2,
|
| 21 |
+
"norm_eps": 1e-05,
|
| 22 |
+
"projection_size": 2048
|
| 23 |
+
},
|
| 24 |
+
"dtype": "float32",
|
| 25 |
+
"head_compute_dtype": "bfloat16",
|
| 26 |
+
"image_token_id": 248056,
|
| 27 |
+
"inference_limits": {
|
| 28 |
+
"batch_tokens": 32768,
|
| 29 |
+
"max_attention_bytes": 549755813888,
|
| 30 |
+
"max_branches": 5120,
|
| 31 |
+
"max_context": 32768,
|
| 32 |
+
"max_logit_elements": 16777216,
|
| 33 |
+
"max_questions": 512,
|
| 34 |
+
"max_request_tokens": 131072
|
| 35 |
+
},
|
| 36 |
+
"initialization_seed": 41,
|
| 37 |
+
"model_type": "gs1",
|
| 38 |
+
"release_date": "2026-11-10",
|
| 39 |
+
"structural_token_ids": {
|
| 40 |
+
"</option>": 248082,
|
| 41 |
+
"</question>": 248080,
|
| 42 |
+
"</state>": 248078,
|
| 43 |
+
"<option>": 248081,
|
| 44 |
+
"<question>": 248079,
|
| 45 |
+
"<state>": 248077,
|
| 46 |
+
"<|decide|>": 248083
|
| 47 |
+
},
|
| 48 |
+
"text_config": {
|
| 49 |
+
"attention_bias": false,
|
| 50 |
+
"attention_dropout": 0.0,
|
| 51 |
+
"attn_output_gate": true,
|
| 52 |
+
"bos_token_id": null,
|
| 53 |
+
"dtype": "bfloat16",
|
| 54 |
+
"eos_token_id": 248044,
|
| 55 |
+
"full_attention_interval": 4,
|
| 56 |
+
"head_dim": 256,
|
| 57 |
+
"hidden_act": "silu",
|
| 58 |
+
"hidden_size": 4096,
|
| 59 |
+
"initializer_range": 0.02,
|
| 60 |
+
"intermediate_size": 12288,
|
| 61 |
+
"layer_types": [
|
| 62 |
+
"linear_attention",
|
| 63 |
+
"linear_attention",
|
| 64 |
+
"linear_attention",
|
| 65 |
+
"full_attention",
|
| 66 |
+
"linear_attention",
|
| 67 |
+
"linear_attention",
|
| 68 |
+
"linear_attention",
|
| 69 |
+
"full_attention",
|
| 70 |
+
"linear_attention",
|
| 71 |
+
"linear_attention",
|
| 72 |
+
"linear_attention",
|
| 73 |
+
"full_attention",
|
| 74 |
+
"linear_attention",
|
| 75 |
+
"linear_attention",
|
| 76 |
+
"linear_attention",
|
| 77 |
+
"full_attention",
|
| 78 |
+
"linear_attention",
|
| 79 |
+
"linear_attention",
|
| 80 |
+
"linear_attention",
|
| 81 |
+
"full_attention",
|
| 82 |
+
"linear_attention",
|
| 83 |
+
"linear_attention",
|
| 84 |
+
"linear_attention",
|
| 85 |
+
"full_attention",
|
| 86 |
+
"linear_attention",
|
| 87 |
+
"linear_attention",
|
| 88 |
+
"linear_attention",
|
| 89 |
+
"full_attention",
|
| 90 |
+
"linear_attention",
|
| 91 |
+
"linear_attention",
|
| 92 |
+
"linear_attention",
|
| 93 |
+
"full_attention"
|
| 94 |
+
],
|
| 95 |
+
"linear_conv_kernel_dim": 4,
|
| 96 |
+
"linear_key_head_dim": 128,
|
| 97 |
+
"linear_num_key_heads": 16,
|
| 98 |
+
"linear_num_value_heads": 32,
|
| 99 |
+
"linear_value_head_dim": 128,
|
| 100 |
+
"mamba_ssm_dtype": "float32",
|
| 101 |
+
"max_position_embeddings": 262144,
|
| 102 |
+
"mlp_only_layers": [],
|
| 103 |
+
"model_type": "qwen3_5_text",
|
| 104 |
+
"mtp_num_hidden_layers": 1,
|
| 105 |
+
"mtp_use_dedicated_embeddings": false,
|
| 106 |
+
"num_attention_heads": 16,
|
| 107 |
+
"num_hidden_layers": 32,
|
| 108 |
+
"num_key_value_heads": 4,
|
| 109 |
+
"pad_token_id": null,
|
| 110 |
+
"partial_rotary_factor": 0.25,
|
| 111 |
+
"rms_norm_eps": 1e-06,
|
| 112 |
+
"rope_parameters": {
|
| 113 |
+
"mrope_interleaved": true,
|
| 114 |
+
"mrope_section": [
|
| 115 |
+
11,
|
| 116 |
+
11,
|
| 117 |
+
10
|
| 118 |
+
],
|
| 119 |
+
"partial_rotary_factor": 0.25,
|
| 120 |
+
"rope_theta": 10000000,
|
| 121 |
+
"rope_type": "default"
|
| 122 |
+
},
|
| 123 |
+
"tie_word_embeddings": false,
|
| 124 |
+
"use_cache": false,
|
| 125 |
+
"vocab_size": 248320
|
| 126 |
+
},
|
| 127 |
+
"tie_word_embeddings": false,
|
| 128 |
+
"transformers_version": "5.3.0",
|
| 129 |
+
"video_token_id": 248057,
|
| 130 |
+
"vision_config": {
|
| 131 |
+
"deepstack_visual_indexes": [],
|
| 132 |
+
"depth": 27,
|
| 133 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 134 |
+
"hidden_size": 1152,
|
| 135 |
+
"in_channels": 3,
|
| 136 |
+
"initializer_range": 0.02,
|
| 137 |
+
"intermediate_size": 4304,
|
| 138 |
+
"model_type": "qwen3_5",
|
| 139 |
+
"num_heads": 16,
|
| 140 |
+
"num_position_embeddings": 2304,
|
| 141 |
+
"out_hidden_size": 4096,
|
| 142 |
+
"patch_size": 16,
|
| 143 |
+
"spatial_merge_size": 2,
|
| 144 |
+
"temporal_patch_size": 2
|
| 145 |
+
},
|
| 146 |
+
"vision_end_token_id": 248054,
|
| 147 |
+
"vision_start_token_id": 248053
|
| 148 |
+
}
|
config.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Semantic architecture and scalar calibration settings for GS1."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from typing import Literal
|
| 5 |
+
|
| 6 |
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
| 7 |
+
|
| 8 |
+
INFERENCE_LIMITS = dict(
|
| 9 |
+
max_context=65536,
|
| 10 |
+
max_questions=512,
|
| 11 |
+
max_request_tokens=131072,
|
| 12 |
+
max_branches=5120,
|
| 13 |
+
batch_tokens=16384,
|
| 14 |
+
max_attention_bytes=1073741824,
|
| 15 |
+
max_logit_elements=16777216,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class ModelConfig(BaseModel):
|
| 20 |
+
model_config = ConfigDict(extra="forbid", strict=True, allow_inf_nan=False)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class DecisionConfig(ModelConfig):
|
| 24 |
+
attnres: bool = True
|
| 25 |
+
attnres_layers: list[int] | None = None
|
| 26 |
+
attnres_eps: float = Field(default=1e-6, gt=0)
|
| 27 |
+
bidirectional: bool = True
|
| 28 |
+
bidirectional_layers: list[int] | None = None
|
| 29 |
+
cross_attention_heads: int = Field(default=8, ge=1)
|
| 30 |
+
cross_attention_layers: int = Field(default=2, ge=1)
|
| 31 |
+
projection_size: int = Field(default=256, ge=2)
|
| 32 |
+
norm_eps: float = Field(default=1e-5, gt=0)
|
| 33 |
+
|
| 34 |
+
@field_validator("attnres_layers", "bidirectional_layers")
|
| 35 |
+
@classmethod
|
| 36 |
+
def selected_layers(cls, value):
|
| 37 |
+
if value is not None:
|
| 38 |
+
if not value or any(i < 0 for i in value) or len(set(value)) != len(value):
|
| 39 |
+
raise ValueError("Layer selection must contain distinct nonnegative indices")
|
| 40 |
+
return sorted(value)
|
| 41 |
+
return value
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class CalibrationConfig(ModelConfig):
|
| 45 |
+
mode: Literal["identity", "scalar_temperature"] = "identity"
|
| 46 |
+
log_temperature_bound: float = Field(default=math.log(20), gt=0)
|
model-00001-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:798e888e74acd113bad08993f33b0d57182d1867b440c3efc7ec0ca51c332d78
|
| 3 |
+
size 4068475032
|
model-00002-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f34e25acdb08b7dd7aefb711a0980b597d909ea89d88efc467b9aaa78e90b5eb
|
| 3 |
+
size 3930731072
|
model-00003-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d7f3cb7617ac72af8fce4b2f80805de9bb34a5f74fc034a24557b8ec15dc0d3
|
| 3 |
+
size 3929715152
|
model-00004-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3bee749523dcc2a0b9e9dc4f87b499cf7a106d7588710bf429c17e33dd994e39
|
| 3 |
+
size 3863489832
|
model-00005-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ede907fad0346e5913c30d84368ff2d105d24156d8058868dd0b07b314e00701
|
| 3 |
+
size 3929715216
|
model-00006-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17aeb5de206f5d110f150e73137fbfae159bb8094bacf01889a1401a66e48d61
|
| 3 |
+
size 3863489864
|
model-00007-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ea0ae8c0d21c860f13a844003e33135bb063c7cdc0be11fcf6678d8d8da9236f
|
| 3 |
+
size 3996691776
|
model-00008-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d16bf1af5e63548ad9c2192ecca1821d3fd2b4654a87ec338a1ca4d4c634777d
|
| 3 |
+
size 3963106840
|
model-00009-of-00009.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:192e5897081c8570e30a9b9f18484d99cd1d4792bf391f8698cab344fe78b77a
|
| 3 |
+
size 2858974760
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,828 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_parameters": 8829078768,
|
| 4 |
+
"total_size": 34404294112
|
| 5 |
+
},
|
| 6 |
+
"weight_map": {
|
| 7 |
+
"decision_head.condition_projection.weight": "model-00009-of-00009.safetensors",
|
| 8 |
+
"decision_head.context_norm.bias": "model-00009-of-00009.safetensors",
|
| 9 |
+
"decision_head.context_norm.weight": "model-00009-of-00009.safetensors",
|
| 10 |
+
"decision_head.decision_projection.weight": "model-00009-of-00009.safetensors",
|
| 11 |
+
"decision_head.key_mlp.0.bias": "model-00009-of-00009.safetensors",
|
| 12 |
+
"decision_head.key_mlp.0.weight": "model-00009-of-00009.safetensors",
|
| 13 |
+
"decision_head.key_mlp.2.bias": "model-00009-of-00009.safetensors",
|
| 14 |
+
"decision_head.key_mlp.2.weight": "model-00009-of-00009.safetensors",
|
| 15 |
+
"decision_head.key_norm.bias": "model-00009-of-00009.safetensors",
|
| 16 |
+
"decision_head.key_norm.weight": "model-00009-of-00009.safetensors",
|
| 17 |
+
"decision_head.memory_norm.bias": "model-00009-of-00009.safetensors",
|
| 18 |
+
"decision_head.memory_norm.weight": "model-00009-of-00009.safetensors",
|
| 19 |
+
"decision_head.memory_projection.weight": "model-00009-of-00009.safetensors",
|
| 20 |
+
"decision_head.option_norm.bias": "model-00009-of-00009.safetensors",
|
| 21 |
+
"decision_head.option_norm.weight": "model-00009-of-00009.safetensors",
|
| 22 |
+
"decision_head.option_projection.weight": "model-00009-of-00009.safetensors",
|
| 23 |
+
"decision_head.query_mlp.0.bias": "model-00009-of-00009.safetensors",
|
| 24 |
+
"decision_head.query_mlp.0.weight": "model-00009-of-00009.safetensors",
|
| 25 |
+
"decision_head.query_mlp.2.bias": "model-00009-of-00009.safetensors",
|
| 26 |
+
"decision_head.query_mlp.2.weight": "model-00009-of-00009.safetensors",
|
| 27 |
+
"decision_head.query_norm.bias": "model-00009-of-00009.safetensors",
|
| 28 |
+
"decision_head.query_norm.weight": "model-00009-of-00009.safetensors",
|
| 29 |
+
"decision_head.question_attention.in_proj_bias": "model-00009-of-00009.safetensors",
|
| 30 |
+
"decision_head.question_attention.in_proj_weight": "model-00009-of-00009.safetensors",
|
| 31 |
+
"decision_head.question_attention.out_proj.bias": "model-00009-of-00009.safetensors",
|
| 32 |
+
"decision_head.question_attention.out_proj.weight": "model-00009-of-00009.safetensors",
|
| 33 |
+
"decision_head.question_evidence_projection.weight": "model-00009-of-00009.safetensors",
|
| 34 |
+
"decision_head.question_projection.weight": "model-00009-of-00009.safetensors",
|
| 35 |
+
"decision_head.question_seed.weight": "model-00009-of-00009.safetensors",
|
| 36 |
+
"decision_head.routing_attention.0.in_proj_bias": "model-00009-of-00009.safetensors",
|
| 37 |
+
"decision_head.routing_attention.0.in_proj_weight": "model-00009-of-00009.safetensors",
|
| 38 |
+
"decision_head.routing_attention.0.out_proj.bias": "model-00009-of-00009.safetensors",
|
| 39 |
+
"decision_head.routing_attention.0.out_proj.weight": "model-00009-of-00009.safetensors",
|
| 40 |
+
"decision_head.routing_attention.1.in_proj_bias": "model-00009-of-00009.safetensors",
|
| 41 |
+
"decision_head.routing_attention.1.in_proj_weight": "model-00009-of-00009.safetensors",
|
| 42 |
+
"decision_head.routing_attention.1.out_proj.bias": "model-00009-of-00009.safetensors",
|
| 43 |
+
"decision_head.routing_attention.1.out_proj.weight": "model-00009-of-00009.safetensors",
|
| 44 |
+
"decision_head.routing_attention_norm.0.bias": "model-00009-of-00009.safetensors",
|
| 45 |
+
"decision_head.routing_attention_norm.0.weight": "model-00009-of-00009.safetensors",
|
| 46 |
+
"decision_head.routing_attention_norm.1.bias": "model-00009-of-00009.safetensors",
|
| 47 |
+
"decision_head.routing_attention_norm.1.weight": "model-00009-of-00009.safetensors",
|
| 48 |
+
"decision_head.routing_ffn.0.0.bias": "model-00009-of-00009.safetensors",
|
| 49 |
+
"decision_head.routing_ffn.0.0.weight": "model-00009-of-00009.safetensors",
|
| 50 |
+
"decision_head.routing_ffn.0.2.bias": "model-00009-of-00009.safetensors",
|
| 51 |
+
"decision_head.routing_ffn.0.2.weight": "model-00009-of-00009.safetensors",
|
| 52 |
+
"decision_head.routing_ffn.1.0.bias": "model-00009-of-00009.safetensors",
|
| 53 |
+
"decision_head.routing_ffn.1.0.weight": "model-00009-of-00009.safetensors",
|
| 54 |
+
"decision_head.routing_ffn.1.2.bias": "model-00009-of-00009.safetensors",
|
| 55 |
+
"decision_head.routing_ffn.1.2.weight": "model-00009-of-00009.safetensors",
|
| 56 |
+
"decision_head.routing_output_norm.0.bias": "model-00009-of-00009.safetensors",
|
| 57 |
+
"decision_head.routing_output_norm.0.weight": "model-00009-of-00009.safetensors",
|
| 58 |
+
"decision_head.routing_output_norm.1.bias": "model-00009-of-00009.safetensors",
|
| 59 |
+
"decision_head.routing_output_norm.1.weight": "model-00009-of-00009.safetensors",
|
| 60 |
+
"language_model.attnres_routers.11.query": "model-00009-of-00009.safetensors",
|
| 61 |
+
"language_model.attnres_routers.15.query": "model-00009-of-00009.safetensors",
|
| 62 |
+
"language_model.attnres_routers.19.query": "model-00009-of-00009.safetensors",
|
| 63 |
+
"language_model.attnres_routers.23.query": "model-00009-of-00009.safetensors",
|
| 64 |
+
"language_model.attnres_routers.27.query": "model-00009-of-00009.safetensors",
|
| 65 |
+
"language_model.attnres_routers.3.query": "model-00009-of-00009.safetensors",
|
| 66 |
+
"language_model.attnres_routers.31.query": "model-00009-of-00009.safetensors",
|
| 67 |
+
"language_model.attnres_routers.7.query": "model-00009-of-00009.safetensors",
|
| 68 |
+
"language_model.embed_tokens.weight": "model-00001-of-00009.safetensors",
|
| 69 |
+
"language_model.layers.0.input_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 70 |
+
"language_model.layers.0.linear_attn.A_log": "model-00002-of-00009.safetensors",
|
| 71 |
+
"language_model.layers.0.linear_attn.conv1d.weight": "model-00002-of-00009.safetensors",
|
| 72 |
+
"language_model.layers.0.linear_attn.dt_bias": "model-00002-of-00009.safetensors",
|
| 73 |
+
"language_model.layers.0.linear_attn.in_proj_a.weight": "model-00002-of-00009.safetensors",
|
| 74 |
+
"language_model.layers.0.linear_attn.in_proj_b.weight": "model-00002-of-00009.safetensors",
|
| 75 |
+
"language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00002-of-00009.safetensors",
|
| 76 |
+
"language_model.layers.0.linear_attn.in_proj_z.weight": "model-00002-of-00009.safetensors",
|
| 77 |
+
"language_model.layers.0.linear_attn.norm.weight": "model-00002-of-00009.safetensors",
|
| 78 |
+
"language_model.layers.0.linear_attn.out_proj.weight": "model-00002-of-00009.safetensors",
|
| 79 |
+
"language_model.layers.0.mlp.down_proj.weight": "model-00002-of-00009.safetensors",
|
| 80 |
+
"language_model.layers.0.mlp.gate_proj.weight": "model-00002-of-00009.safetensors",
|
| 81 |
+
"language_model.layers.0.mlp.up_proj.weight": "model-00002-of-00009.safetensors",
|
| 82 |
+
"language_model.layers.0.post_attention_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 83 |
+
"language_model.layers.1.input_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 84 |
+
"language_model.layers.1.linear_attn.A_log": "model-00002-of-00009.safetensors",
|
| 85 |
+
"language_model.layers.1.linear_attn.conv1d.weight": "model-00002-of-00009.safetensors",
|
| 86 |
+
"language_model.layers.1.linear_attn.dt_bias": "model-00002-of-00009.safetensors",
|
| 87 |
+
"language_model.layers.1.linear_attn.in_proj_a.weight": "model-00002-of-00009.safetensors",
|
| 88 |
+
"language_model.layers.1.linear_attn.in_proj_b.weight": "model-00002-of-00009.safetensors",
|
| 89 |
+
"language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00002-of-00009.safetensors",
|
| 90 |
+
"language_model.layers.1.linear_attn.in_proj_z.weight": "model-00002-of-00009.safetensors",
|
| 91 |
+
"language_model.layers.1.linear_attn.norm.weight": "model-00002-of-00009.safetensors",
|
| 92 |
+
"language_model.layers.1.linear_attn.out_proj.weight": "model-00002-of-00009.safetensors",
|
| 93 |
+
"language_model.layers.1.mlp.down_proj.weight": "model-00002-of-00009.safetensors",
|
| 94 |
+
"language_model.layers.1.mlp.gate_proj.weight": "model-00002-of-00009.safetensors",
|
| 95 |
+
"language_model.layers.1.mlp.up_proj.weight": "model-00002-of-00009.safetensors",
|
| 96 |
+
"language_model.layers.1.post_attention_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 97 |
+
"language_model.layers.10.input_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 98 |
+
"language_model.layers.10.linear_attn.A_log": "model-00004-of-00009.safetensors",
|
| 99 |
+
"language_model.layers.10.linear_attn.conv1d.weight": "model-00004-of-00009.safetensors",
|
| 100 |
+
"language_model.layers.10.linear_attn.dt_bias": "model-00004-of-00009.safetensors",
|
| 101 |
+
"language_model.layers.10.linear_attn.in_proj_a.weight": "model-00004-of-00009.safetensors",
|
| 102 |
+
"language_model.layers.10.linear_attn.in_proj_b.weight": "model-00004-of-00009.safetensors",
|
| 103 |
+
"language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00004-of-00009.safetensors",
|
| 104 |
+
"language_model.layers.10.linear_attn.in_proj_z.weight": "model-00004-of-00009.safetensors",
|
| 105 |
+
"language_model.layers.10.linear_attn.norm.weight": "model-00004-of-00009.safetensors",
|
| 106 |
+
"language_model.layers.10.linear_attn.out_proj.weight": "model-00004-of-00009.safetensors",
|
| 107 |
+
"language_model.layers.10.mlp.down_proj.weight": "model-00004-of-00009.safetensors",
|
| 108 |
+
"language_model.layers.10.mlp.gate_proj.weight": "model-00004-of-00009.safetensors",
|
| 109 |
+
"language_model.layers.10.mlp.up_proj.weight": "model-00004-of-00009.safetensors",
|
| 110 |
+
"language_model.layers.10.post_attention_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 111 |
+
"language_model.layers.11.input_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 112 |
+
"language_model.layers.11.mlp.down_proj.weight": "model-00004-of-00009.safetensors",
|
| 113 |
+
"language_model.layers.11.mlp.gate_proj.weight": "model-00004-of-00009.safetensors",
|
| 114 |
+
"language_model.layers.11.mlp.up_proj.weight": "model-00004-of-00009.safetensors",
|
| 115 |
+
"language_model.layers.11.post_attention_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 116 |
+
"language_model.layers.11.self_attn.k_norm.weight": "model-00004-of-00009.safetensors",
|
| 117 |
+
"language_model.layers.11.self_attn.k_proj.weight": "model-00004-of-00009.safetensors",
|
| 118 |
+
"language_model.layers.11.self_attn.o_proj.weight": "model-00004-of-00009.safetensors",
|
| 119 |
+
"language_model.layers.11.self_attn.q_norm.weight": "model-00004-of-00009.safetensors",
|
| 120 |
+
"language_model.layers.11.self_attn.q_proj.weight": "model-00004-of-00009.safetensors",
|
| 121 |
+
"language_model.layers.11.self_attn.v_proj.weight": "model-00004-of-00009.safetensors",
|
| 122 |
+
"language_model.layers.12.input_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 123 |
+
"language_model.layers.12.linear_attn.A_log": "model-00004-of-00009.safetensors",
|
| 124 |
+
"language_model.layers.12.linear_attn.conv1d.weight": "model-00004-of-00009.safetensors",
|
| 125 |
+
"language_model.layers.12.linear_attn.dt_bias": "model-00004-of-00009.safetensors",
|
| 126 |
+
"language_model.layers.12.linear_attn.in_proj_a.weight": "model-00004-of-00009.safetensors",
|
| 127 |
+
"language_model.layers.12.linear_attn.in_proj_b.weight": "model-00004-of-00009.safetensors",
|
| 128 |
+
"language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00004-of-00009.safetensors",
|
| 129 |
+
"language_model.layers.12.linear_attn.in_proj_z.weight": "model-00004-of-00009.safetensors",
|
| 130 |
+
"language_model.layers.12.linear_attn.norm.weight": "model-00004-of-00009.safetensors",
|
| 131 |
+
"language_model.layers.12.linear_attn.out_proj.weight": "model-00004-of-00009.safetensors",
|
| 132 |
+
"language_model.layers.12.mlp.down_proj.weight": "model-00004-of-00009.safetensors",
|
| 133 |
+
"language_model.layers.12.mlp.gate_proj.weight": "model-00004-of-00009.safetensors",
|
| 134 |
+
"language_model.layers.12.mlp.up_proj.weight": "model-00004-of-00009.safetensors",
|
| 135 |
+
"language_model.layers.12.post_attention_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 136 |
+
"language_model.layers.13.input_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 137 |
+
"language_model.layers.13.linear_attn.A_log": "model-00004-of-00009.safetensors",
|
| 138 |
+
"language_model.layers.13.linear_attn.conv1d.weight": "model-00004-of-00009.safetensors",
|
| 139 |
+
"language_model.layers.13.linear_attn.dt_bias": "model-00004-of-00009.safetensors",
|
| 140 |
+
"language_model.layers.13.linear_attn.in_proj_a.weight": "model-00004-of-00009.safetensors",
|
| 141 |
+
"language_model.layers.13.linear_attn.in_proj_b.weight": "model-00004-of-00009.safetensors",
|
| 142 |
+
"language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00004-of-00009.safetensors",
|
| 143 |
+
"language_model.layers.13.linear_attn.in_proj_z.weight": "model-00004-of-00009.safetensors",
|
| 144 |
+
"language_model.layers.13.linear_attn.norm.weight": "model-00004-of-00009.safetensors",
|
| 145 |
+
"language_model.layers.13.linear_attn.out_proj.weight": "model-00004-of-00009.safetensors",
|
| 146 |
+
"language_model.layers.13.mlp.down_proj.weight": "model-00005-of-00009.safetensors",
|
| 147 |
+
"language_model.layers.13.mlp.gate_proj.weight": "model-00004-of-00009.safetensors",
|
| 148 |
+
"language_model.layers.13.mlp.up_proj.weight": "model-00005-of-00009.safetensors",
|
| 149 |
+
"language_model.layers.13.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 150 |
+
"language_model.layers.14.input_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 151 |
+
"language_model.layers.14.linear_attn.A_log": "model-00005-of-00009.safetensors",
|
| 152 |
+
"language_model.layers.14.linear_attn.conv1d.weight": "model-00005-of-00009.safetensors",
|
| 153 |
+
"language_model.layers.14.linear_attn.dt_bias": "model-00005-of-00009.safetensors",
|
| 154 |
+
"language_model.layers.14.linear_attn.in_proj_a.weight": "model-00005-of-00009.safetensors",
|
| 155 |
+
"language_model.layers.14.linear_attn.in_proj_b.weight": "model-00005-of-00009.safetensors",
|
| 156 |
+
"language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00005-of-00009.safetensors",
|
| 157 |
+
"language_model.layers.14.linear_attn.in_proj_z.weight": "model-00005-of-00009.safetensors",
|
| 158 |
+
"language_model.layers.14.linear_attn.norm.weight": "model-00005-of-00009.safetensors",
|
| 159 |
+
"language_model.layers.14.linear_attn.out_proj.weight": "model-00005-of-00009.safetensors",
|
| 160 |
+
"language_model.layers.14.mlp.down_proj.weight": "model-00005-of-00009.safetensors",
|
| 161 |
+
"language_model.layers.14.mlp.gate_proj.weight": "model-00005-of-00009.safetensors",
|
| 162 |
+
"language_model.layers.14.mlp.up_proj.weight": "model-00005-of-00009.safetensors",
|
| 163 |
+
"language_model.layers.14.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 164 |
+
"language_model.layers.15.input_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 165 |
+
"language_model.layers.15.mlp.down_proj.weight": "model-00005-of-00009.safetensors",
|
| 166 |
+
"language_model.layers.15.mlp.gate_proj.weight": "model-00005-of-00009.safetensors",
|
| 167 |
+
"language_model.layers.15.mlp.up_proj.weight": "model-00005-of-00009.safetensors",
|
| 168 |
+
"language_model.layers.15.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 169 |
+
"language_model.layers.15.self_attn.k_norm.weight": "model-00005-of-00009.safetensors",
|
| 170 |
+
"language_model.layers.15.self_attn.k_proj.weight": "model-00005-of-00009.safetensors",
|
| 171 |
+
"language_model.layers.15.self_attn.o_proj.weight": "model-00005-of-00009.safetensors",
|
| 172 |
+
"language_model.layers.15.self_attn.q_norm.weight": "model-00005-of-00009.safetensors",
|
| 173 |
+
"language_model.layers.15.self_attn.q_proj.weight": "model-00005-of-00009.safetensors",
|
| 174 |
+
"language_model.layers.15.self_attn.v_proj.weight": "model-00005-of-00009.safetensors",
|
| 175 |
+
"language_model.layers.16.input_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 176 |
+
"language_model.layers.16.linear_attn.A_log": "model-00005-of-00009.safetensors",
|
| 177 |
+
"language_model.layers.16.linear_attn.conv1d.weight": "model-00005-of-00009.safetensors",
|
| 178 |
+
"language_model.layers.16.linear_attn.dt_bias": "model-00005-of-00009.safetensors",
|
| 179 |
+
"language_model.layers.16.linear_attn.in_proj_a.weight": "model-00005-of-00009.safetensors",
|
| 180 |
+
"language_model.layers.16.linear_attn.in_proj_b.weight": "model-00005-of-00009.safetensors",
|
| 181 |
+
"language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00005-of-00009.safetensors",
|
| 182 |
+
"language_model.layers.16.linear_attn.in_proj_z.weight": "model-00005-of-00009.safetensors",
|
| 183 |
+
"language_model.layers.16.linear_attn.norm.weight": "model-00005-of-00009.safetensors",
|
| 184 |
+
"language_model.layers.16.linear_attn.out_proj.weight": "model-00005-of-00009.safetensors",
|
| 185 |
+
"language_model.layers.16.mlp.down_proj.weight": "model-00005-of-00009.safetensors",
|
| 186 |
+
"language_model.layers.16.mlp.gate_proj.weight": "model-00005-of-00009.safetensors",
|
| 187 |
+
"language_model.layers.16.mlp.up_proj.weight": "model-00005-of-00009.safetensors",
|
| 188 |
+
"language_model.layers.16.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 189 |
+
"language_model.layers.17.input_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 190 |
+
"language_model.layers.17.linear_attn.A_log": "model-00005-of-00009.safetensors",
|
| 191 |
+
"language_model.layers.17.linear_attn.conv1d.weight": "model-00005-of-00009.safetensors",
|
| 192 |
+
"language_model.layers.17.linear_attn.dt_bias": "model-00005-of-00009.safetensors",
|
| 193 |
+
"language_model.layers.17.linear_attn.in_proj_a.weight": "model-00005-of-00009.safetensors",
|
| 194 |
+
"language_model.layers.17.linear_attn.in_proj_b.weight": "model-00005-of-00009.safetensors",
|
| 195 |
+
"language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00005-of-00009.safetensors",
|
| 196 |
+
"language_model.layers.17.linear_attn.in_proj_z.weight": "model-00005-of-00009.safetensors",
|
| 197 |
+
"language_model.layers.17.linear_attn.norm.weight": "model-00005-of-00009.safetensors",
|
| 198 |
+
"language_model.layers.17.linear_attn.out_proj.weight": "model-00005-of-00009.safetensors",
|
| 199 |
+
"language_model.layers.17.mlp.down_proj.weight": "model-00005-of-00009.safetensors",
|
| 200 |
+
"language_model.layers.17.mlp.gate_proj.weight": "model-00005-of-00009.safetensors",
|
| 201 |
+
"language_model.layers.17.mlp.up_proj.weight": "model-00005-of-00009.safetensors",
|
| 202 |
+
"language_model.layers.17.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
|
| 203 |
+
"language_model.layers.18.input_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 204 |
+
"language_model.layers.18.linear_attn.A_log": "model-00005-of-00009.safetensors",
|
| 205 |
+
"language_model.layers.18.linear_attn.conv1d.weight": "model-00005-of-00009.safetensors",
|
| 206 |
+
"language_model.layers.18.linear_attn.dt_bias": "model-00005-of-00009.safetensors",
|
| 207 |
+
"language_model.layers.18.linear_attn.in_proj_a.weight": "model-00006-of-00009.safetensors",
|
| 208 |
+
"language_model.layers.18.linear_attn.in_proj_b.weight": "model-00006-of-00009.safetensors",
|
| 209 |
+
"language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00006-of-00009.safetensors",
|
| 210 |
+
"language_model.layers.18.linear_attn.in_proj_z.weight": "model-00006-of-00009.safetensors",
|
| 211 |
+
"language_model.layers.18.linear_attn.norm.weight": "model-00005-of-00009.safetensors",
|
| 212 |
+
"language_model.layers.18.linear_attn.out_proj.weight": "model-00005-of-00009.safetensors",
|
| 213 |
+
"language_model.layers.18.mlp.down_proj.weight": "model-00006-of-00009.safetensors",
|
| 214 |
+
"language_model.layers.18.mlp.gate_proj.weight": "model-00006-of-00009.safetensors",
|
| 215 |
+
"language_model.layers.18.mlp.up_proj.weight": "model-00006-of-00009.safetensors",
|
| 216 |
+
"language_model.layers.18.post_attention_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 217 |
+
"language_model.layers.19.input_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 218 |
+
"language_model.layers.19.mlp.down_proj.weight": "model-00006-of-00009.safetensors",
|
| 219 |
+
"language_model.layers.19.mlp.gate_proj.weight": "model-00006-of-00009.safetensors",
|
| 220 |
+
"language_model.layers.19.mlp.up_proj.weight": "model-00006-of-00009.safetensors",
|
| 221 |
+
"language_model.layers.19.post_attention_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 222 |
+
"language_model.layers.19.self_attn.k_norm.weight": "model-00006-of-00009.safetensors",
|
| 223 |
+
"language_model.layers.19.self_attn.k_proj.weight": "model-00006-of-00009.safetensors",
|
| 224 |
+
"language_model.layers.19.self_attn.o_proj.weight": "model-00006-of-00009.safetensors",
|
| 225 |
+
"language_model.layers.19.self_attn.q_norm.weight": "model-00006-of-00009.safetensors",
|
| 226 |
+
"language_model.layers.19.self_attn.q_proj.weight": "model-00006-of-00009.safetensors",
|
| 227 |
+
"language_model.layers.19.self_attn.v_proj.weight": "model-00006-of-00009.safetensors",
|
| 228 |
+
"language_model.layers.2.input_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 229 |
+
"language_model.layers.2.linear_attn.A_log": "model-00002-of-00009.safetensors",
|
| 230 |
+
"language_model.layers.2.linear_attn.conv1d.weight": "model-00002-of-00009.safetensors",
|
| 231 |
+
"language_model.layers.2.linear_attn.dt_bias": "model-00002-of-00009.safetensors",
|
| 232 |
+
"language_model.layers.2.linear_attn.in_proj_a.weight": "model-00002-of-00009.safetensors",
|
| 233 |
+
"language_model.layers.2.linear_attn.in_proj_b.weight": "model-00002-of-00009.safetensors",
|
| 234 |
+
"language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00002-of-00009.safetensors",
|
| 235 |
+
"language_model.layers.2.linear_attn.in_proj_z.weight": "model-00002-of-00009.safetensors",
|
| 236 |
+
"language_model.layers.2.linear_attn.norm.weight": "model-00002-of-00009.safetensors",
|
| 237 |
+
"language_model.layers.2.linear_attn.out_proj.weight": "model-00002-of-00009.safetensors",
|
| 238 |
+
"language_model.layers.2.mlp.down_proj.weight": "model-00002-of-00009.safetensors",
|
| 239 |
+
"language_model.layers.2.mlp.gate_proj.weight": "model-00002-of-00009.safetensors",
|
| 240 |
+
"language_model.layers.2.mlp.up_proj.weight": "model-00002-of-00009.safetensors",
|
| 241 |
+
"language_model.layers.2.post_attention_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 242 |
+
"language_model.layers.20.input_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 243 |
+
"language_model.layers.20.linear_attn.A_log": "model-00006-of-00009.safetensors",
|
| 244 |
+
"language_model.layers.20.linear_attn.conv1d.weight": "model-00006-of-00009.safetensors",
|
| 245 |
+
"language_model.layers.20.linear_attn.dt_bias": "model-00006-of-00009.safetensors",
|
| 246 |
+
"language_model.layers.20.linear_attn.in_proj_a.weight": "model-00006-of-00009.safetensors",
|
| 247 |
+
"language_model.layers.20.linear_attn.in_proj_b.weight": "model-00006-of-00009.safetensors",
|
| 248 |
+
"language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00006-of-00009.safetensors",
|
| 249 |
+
"language_model.layers.20.linear_attn.in_proj_z.weight": "model-00006-of-00009.safetensors",
|
| 250 |
+
"language_model.layers.20.linear_attn.norm.weight": "model-00006-of-00009.safetensors",
|
| 251 |
+
"language_model.layers.20.linear_attn.out_proj.weight": "model-00006-of-00009.safetensors",
|
| 252 |
+
"language_model.layers.20.mlp.down_proj.weight": "model-00006-of-00009.safetensors",
|
| 253 |
+
"language_model.layers.20.mlp.gate_proj.weight": "model-00006-of-00009.safetensors",
|
| 254 |
+
"language_model.layers.20.mlp.up_proj.weight": "model-00006-of-00009.safetensors",
|
| 255 |
+
"language_model.layers.20.post_attention_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 256 |
+
"language_model.layers.21.input_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 257 |
+
"language_model.layers.21.linear_attn.A_log": "model-00006-of-00009.safetensors",
|
| 258 |
+
"language_model.layers.21.linear_attn.conv1d.weight": "model-00006-of-00009.safetensors",
|
| 259 |
+
"language_model.layers.21.linear_attn.dt_bias": "model-00006-of-00009.safetensors",
|
| 260 |
+
"language_model.layers.21.linear_attn.in_proj_a.weight": "model-00006-of-00009.safetensors",
|
| 261 |
+
"language_model.layers.21.linear_attn.in_proj_b.weight": "model-00006-of-00009.safetensors",
|
| 262 |
+
"language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00006-of-00009.safetensors",
|
| 263 |
+
"language_model.layers.21.linear_attn.in_proj_z.weight": "model-00006-of-00009.safetensors",
|
| 264 |
+
"language_model.layers.21.linear_attn.norm.weight": "model-00006-of-00009.safetensors",
|
| 265 |
+
"language_model.layers.21.linear_attn.out_proj.weight": "model-00006-of-00009.safetensors",
|
| 266 |
+
"language_model.layers.21.mlp.down_proj.weight": "model-00006-of-00009.safetensors",
|
| 267 |
+
"language_model.layers.21.mlp.gate_proj.weight": "model-00006-of-00009.safetensors",
|
| 268 |
+
"language_model.layers.21.mlp.up_proj.weight": "model-00006-of-00009.safetensors",
|
| 269 |
+
"language_model.layers.21.post_attention_layernorm.weight": "model-00006-of-00009.safetensors",
|
| 270 |
+
"language_model.layers.22.input_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 271 |
+
"language_model.layers.22.linear_attn.A_log": "model-00006-of-00009.safetensors",
|
| 272 |
+
"language_model.layers.22.linear_attn.conv1d.weight": "model-00006-of-00009.safetensors",
|
| 273 |
+
"language_model.layers.22.linear_attn.dt_bias": "model-00006-of-00009.safetensors",
|
| 274 |
+
"language_model.layers.22.linear_attn.in_proj_a.weight": "model-00006-of-00009.safetensors",
|
| 275 |
+
"language_model.layers.22.linear_attn.in_proj_b.weight": "model-00006-of-00009.safetensors",
|
| 276 |
+
"language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00006-of-00009.safetensors",
|
| 277 |
+
"language_model.layers.22.linear_attn.in_proj_z.weight": "model-00006-of-00009.safetensors",
|
| 278 |
+
"language_model.layers.22.linear_attn.norm.weight": "model-00006-of-00009.safetensors",
|
| 279 |
+
"language_model.layers.22.linear_attn.out_proj.weight": "model-00006-of-00009.safetensors",
|
| 280 |
+
"language_model.layers.22.mlp.down_proj.weight": "model-00007-of-00009.safetensors",
|
| 281 |
+
"language_model.layers.22.mlp.gate_proj.weight": "model-00006-of-00009.safetensors",
|
| 282 |
+
"language_model.layers.22.mlp.up_proj.weight": "model-00007-of-00009.safetensors",
|
| 283 |
+
"language_model.layers.22.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 284 |
+
"language_model.layers.23.input_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 285 |
+
"language_model.layers.23.mlp.down_proj.weight": "model-00007-of-00009.safetensors",
|
| 286 |
+
"language_model.layers.23.mlp.gate_proj.weight": "model-00007-of-00009.safetensors",
|
| 287 |
+
"language_model.layers.23.mlp.up_proj.weight": "model-00007-of-00009.safetensors",
|
| 288 |
+
"language_model.layers.23.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 289 |
+
"language_model.layers.23.self_attn.k_norm.weight": "model-00007-of-00009.safetensors",
|
| 290 |
+
"language_model.layers.23.self_attn.k_proj.weight": "model-00007-of-00009.safetensors",
|
| 291 |
+
"language_model.layers.23.self_attn.o_proj.weight": "model-00007-of-00009.safetensors",
|
| 292 |
+
"language_model.layers.23.self_attn.q_norm.weight": "model-00007-of-00009.safetensors",
|
| 293 |
+
"language_model.layers.23.self_attn.q_proj.weight": "model-00007-of-00009.safetensors",
|
| 294 |
+
"language_model.layers.23.self_attn.v_proj.weight": "model-00007-of-00009.safetensors",
|
| 295 |
+
"language_model.layers.24.input_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 296 |
+
"language_model.layers.24.linear_attn.A_log": "model-00007-of-00009.safetensors",
|
| 297 |
+
"language_model.layers.24.linear_attn.conv1d.weight": "model-00007-of-00009.safetensors",
|
| 298 |
+
"language_model.layers.24.linear_attn.dt_bias": "model-00007-of-00009.safetensors",
|
| 299 |
+
"language_model.layers.24.linear_attn.in_proj_a.weight": "model-00007-of-00009.safetensors",
|
| 300 |
+
"language_model.layers.24.linear_attn.in_proj_b.weight": "model-00007-of-00009.safetensors",
|
| 301 |
+
"language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00007-of-00009.safetensors",
|
| 302 |
+
"language_model.layers.24.linear_attn.in_proj_z.weight": "model-00007-of-00009.safetensors",
|
| 303 |
+
"language_model.layers.24.linear_attn.norm.weight": "model-00007-of-00009.safetensors",
|
| 304 |
+
"language_model.layers.24.linear_attn.out_proj.weight": "model-00007-of-00009.safetensors",
|
| 305 |
+
"language_model.layers.24.mlp.down_proj.weight": "model-00007-of-00009.safetensors",
|
| 306 |
+
"language_model.layers.24.mlp.gate_proj.weight": "model-00007-of-00009.safetensors",
|
| 307 |
+
"language_model.layers.24.mlp.up_proj.weight": "model-00007-of-00009.safetensors",
|
| 308 |
+
"language_model.layers.24.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 309 |
+
"language_model.layers.25.input_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 310 |
+
"language_model.layers.25.linear_attn.A_log": "model-00007-of-00009.safetensors",
|
| 311 |
+
"language_model.layers.25.linear_attn.conv1d.weight": "model-00007-of-00009.safetensors",
|
| 312 |
+
"language_model.layers.25.linear_attn.dt_bias": "model-00007-of-00009.safetensors",
|
| 313 |
+
"language_model.layers.25.linear_attn.in_proj_a.weight": "model-00007-of-00009.safetensors",
|
| 314 |
+
"language_model.layers.25.linear_attn.in_proj_b.weight": "model-00007-of-00009.safetensors",
|
| 315 |
+
"language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00007-of-00009.safetensors",
|
| 316 |
+
"language_model.layers.25.linear_attn.in_proj_z.weight": "model-00007-of-00009.safetensors",
|
| 317 |
+
"language_model.layers.25.linear_attn.norm.weight": "model-00007-of-00009.safetensors",
|
| 318 |
+
"language_model.layers.25.linear_attn.out_proj.weight": "model-00007-of-00009.safetensors",
|
| 319 |
+
"language_model.layers.25.mlp.down_proj.weight": "model-00007-of-00009.safetensors",
|
| 320 |
+
"language_model.layers.25.mlp.gate_proj.weight": "model-00007-of-00009.safetensors",
|
| 321 |
+
"language_model.layers.25.mlp.up_proj.weight": "model-00007-of-00009.safetensors",
|
| 322 |
+
"language_model.layers.25.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 323 |
+
"language_model.layers.26.input_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 324 |
+
"language_model.layers.26.linear_attn.A_log": "model-00007-of-00009.safetensors",
|
| 325 |
+
"language_model.layers.26.linear_attn.conv1d.weight": "model-00007-of-00009.safetensors",
|
| 326 |
+
"language_model.layers.26.linear_attn.dt_bias": "model-00007-of-00009.safetensors",
|
| 327 |
+
"language_model.layers.26.linear_attn.in_proj_a.weight": "model-00007-of-00009.safetensors",
|
| 328 |
+
"language_model.layers.26.linear_attn.in_proj_b.weight": "model-00007-of-00009.safetensors",
|
| 329 |
+
"language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00007-of-00009.safetensors",
|
| 330 |
+
"language_model.layers.26.linear_attn.in_proj_z.weight": "model-00007-of-00009.safetensors",
|
| 331 |
+
"language_model.layers.26.linear_attn.norm.weight": "model-00007-of-00009.safetensors",
|
| 332 |
+
"language_model.layers.26.linear_attn.out_proj.weight": "model-00007-of-00009.safetensors",
|
| 333 |
+
"language_model.layers.26.mlp.down_proj.weight": "model-00007-of-00009.safetensors",
|
| 334 |
+
"language_model.layers.26.mlp.gate_proj.weight": "model-00007-of-00009.safetensors",
|
| 335 |
+
"language_model.layers.26.mlp.up_proj.weight": "model-00007-of-00009.safetensors",
|
| 336 |
+
"language_model.layers.26.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
|
| 337 |
+
"language_model.layers.27.input_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 338 |
+
"language_model.layers.27.mlp.down_proj.weight": "model-00008-of-00009.safetensors",
|
| 339 |
+
"language_model.layers.27.mlp.gate_proj.weight": "model-00008-of-00009.safetensors",
|
| 340 |
+
"language_model.layers.27.mlp.up_proj.weight": "model-00008-of-00009.safetensors",
|
| 341 |
+
"language_model.layers.27.post_attention_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 342 |
+
"language_model.layers.27.self_attn.k_norm.weight": "model-00008-of-00009.safetensors",
|
| 343 |
+
"language_model.layers.27.self_attn.k_proj.weight": "model-00008-of-00009.safetensors",
|
| 344 |
+
"language_model.layers.27.self_attn.o_proj.weight": "model-00008-of-00009.safetensors",
|
| 345 |
+
"language_model.layers.27.self_attn.q_norm.weight": "model-00008-of-00009.safetensors",
|
| 346 |
+
"language_model.layers.27.self_attn.q_proj.weight": "model-00007-of-00009.safetensors",
|
| 347 |
+
"language_model.layers.27.self_attn.v_proj.weight": "model-00008-of-00009.safetensors",
|
| 348 |
+
"language_model.layers.28.input_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 349 |
+
"language_model.layers.28.linear_attn.A_log": "model-00008-of-00009.safetensors",
|
| 350 |
+
"language_model.layers.28.linear_attn.conv1d.weight": "model-00008-of-00009.safetensors",
|
| 351 |
+
"language_model.layers.28.linear_attn.dt_bias": "model-00008-of-00009.safetensors",
|
| 352 |
+
"language_model.layers.28.linear_attn.in_proj_a.weight": "model-00008-of-00009.safetensors",
|
| 353 |
+
"language_model.layers.28.linear_attn.in_proj_b.weight": "model-00008-of-00009.safetensors",
|
| 354 |
+
"language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00008-of-00009.safetensors",
|
| 355 |
+
"language_model.layers.28.linear_attn.in_proj_z.weight": "model-00008-of-00009.safetensors",
|
| 356 |
+
"language_model.layers.28.linear_attn.norm.weight": "model-00008-of-00009.safetensors",
|
| 357 |
+
"language_model.layers.28.linear_attn.out_proj.weight": "model-00008-of-00009.safetensors",
|
| 358 |
+
"language_model.layers.28.mlp.down_proj.weight": "model-00008-of-00009.safetensors",
|
| 359 |
+
"language_model.layers.28.mlp.gate_proj.weight": "model-00008-of-00009.safetensors",
|
| 360 |
+
"language_model.layers.28.mlp.up_proj.weight": "model-00008-of-00009.safetensors",
|
| 361 |
+
"language_model.layers.28.post_attention_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 362 |
+
"language_model.layers.29.input_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 363 |
+
"language_model.layers.29.linear_attn.A_log": "model-00008-of-00009.safetensors",
|
| 364 |
+
"language_model.layers.29.linear_attn.conv1d.weight": "model-00008-of-00009.safetensors",
|
| 365 |
+
"language_model.layers.29.linear_attn.dt_bias": "model-00008-of-00009.safetensors",
|
| 366 |
+
"language_model.layers.29.linear_attn.in_proj_a.weight": "model-00008-of-00009.safetensors",
|
| 367 |
+
"language_model.layers.29.linear_attn.in_proj_b.weight": "model-00008-of-00009.safetensors",
|
| 368 |
+
"language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00008-of-00009.safetensors",
|
| 369 |
+
"language_model.layers.29.linear_attn.in_proj_z.weight": "model-00008-of-00009.safetensors",
|
| 370 |
+
"language_model.layers.29.linear_attn.norm.weight": "model-00008-of-00009.safetensors",
|
| 371 |
+
"language_model.layers.29.linear_attn.out_proj.weight": "model-00008-of-00009.safetensors",
|
| 372 |
+
"language_model.layers.29.mlp.down_proj.weight": "model-00008-of-00009.safetensors",
|
| 373 |
+
"language_model.layers.29.mlp.gate_proj.weight": "model-00008-of-00009.safetensors",
|
| 374 |
+
"language_model.layers.29.mlp.up_proj.weight": "model-00008-of-00009.safetensors",
|
| 375 |
+
"language_model.layers.29.post_attention_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 376 |
+
"language_model.layers.3.input_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 377 |
+
"language_model.layers.3.mlp.down_proj.weight": "model-00002-of-00009.safetensors",
|
| 378 |
+
"language_model.layers.3.mlp.gate_proj.weight": "model-00002-of-00009.safetensors",
|
| 379 |
+
"language_model.layers.3.mlp.up_proj.weight": "model-00002-of-00009.safetensors",
|
| 380 |
+
"language_model.layers.3.post_attention_layernorm.weight": "model-00002-of-00009.safetensors",
|
| 381 |
+
"language_model.layers.3.self_attn.k_norm.weight": "model-00002-of-00009.safetensors",
|
| 382 |
+
"language_model.layers.3.self_attn.k_proj.weight": "model-00002-of-00009.safetensors",
|
| 383 |
+
"language_model.layers.3.self_attn.o_proj.weight": "model-00002-of-00009.safetensors",
|
| 384 |
+
"language_model.layers.3.self_attn.q_norm.weight": "model-00002-of-00009.safetensors",
|
| 385 |
+
"language_model.layers.3.self_attn.q_proj.weight": "model-00002-of-00009.safetensors",
|
| 386 |
+
"language_model.layers.3.self_attn.v_proj.weight": "model-00002-of-00009.safetensors",
|
| 387 |
+
"language_model.layers.30.input_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 388 |
+
"language_model.layers.30.linear_attn.A_log": "model-00008-of-00009.safetensors",
|
| 389 |
+
"language_model.layers.30.linear_attn.conv1d.weight": "model-00008-of-00009.safetensors",
|
| 390 |
+
"language_model.layers.30.linear_attn.dt_bias": "model-00008-of-00009.safetensors",
|
| 391 |
+
"language_model.layers.30.linear_attn.in_proj_a.weight": "model-00008-of-00009.safetensors",
|
| 392 |
+
"language_model.layers.30.linear_attn.in_proj_b.weight": "model-00008-of-00009.safetensors",
|
| 393 |
+
"language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00008-of-00009.safetensors",
|
| 394 |
+
"language_model.layers.30.linear_attn.in_proj_z.weight": "model-00008-of-00009.safetensors",
|
| 395 |
+
"language_model.layers.30.linear_attn.norm.weight": "model-00008-of-00009.safetensors",
|
| 396 |
+
"language_model.layers.30.linear_attn.out_proj.weight": "model-00008-of-00009.safetensors",
|
| 397 |
+
"language_model.layers.30.mlp.down_proj.weight": "model-00008-of-00009.safetensors",
|
| 398 |
+
"language_model.layers.30.mlp.gate_proj.weight": "model-00008-of-00009.safetensors",
|
| 399 |
+
"language_model.layers.30.mlp.up_proj.weight": "model-00008-of-00009.safetensors",
|
| 400 |
+
"language_model.layers.30.post_attention_layernorm.weight": "model-00008-of-00009.safetensors",
|
| 401 |
+
"language_model.layers.31.input_layernorm.weight": "model-00009-of-00009.safetensors",
|
| 402 |
+
"language_model.layers.31.mlp.down_proj.weight": "model-00009-of-00009.safetensors",
|
| 403 |
+
"language_model.layers.31.mlp.gate_proj.weight": "model-00008-of-00009.safetensors",
|
| 404 |
+
"language_model.layers.31.mlp.up_proj.weight": "model-00008-of-00009.safetensors",
|
| 405 |
+
"language_model.layers.31.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
|
| 406 |
+
"language_model.layers.31.self_attn.k_norm.weight": "model-00008-of-00009.safetensors",
|
| 407 |
+
"language_model.layers.31.self_attn.k_proj.weight": "model-00008-of-00009.safetensors",
|
| 408 |
+
"language_model.layers.31.self_attn.o_proj.weight": "model-00008-of-00009.safetensors",
|
| 409 |
+
"language_model.layers.31.self_attn.q_norm.weight": "model-00008-of-00009.safetensors",
|
| 410 |
+
"language_model.layers.31.self_attn.q_proj.weight": "model-00008-of-00009.safetensors",
|
| 411 |
+
"language_model.layers.31.self_attn.v_proj.weight": "model-00008-of-00009.safetensors",
|
| 412 |
+
"language_model.layers.4.input_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 413 |
+
"language_model.layers.4.linear_attn.A_log": "model-00002-of-00009.safetensors",
|
| 414 |
+
"language_model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00009.safetensors",
|
| 415 |
+
"language_model.layers.4.linear_attn.dt_bias": "model-00002-of-00009.safetensors",
|
| 416 |
+
"language_model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00009.safetensors",
|
| 417 |
+
"language_model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00009.safetensors",
|
| 418 |
+
"language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00009.safetensors",
|
| 419 |
+
"language_model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00009.safetensors",
|
| 420 |
+
"language_model.layers.4.linear_attn.norm.weight": "model-00002-of-00009.safetensors",
|
| 421 |
+
"language_model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00009.safetensors",
|
| 422 |
+
"language_model.layers.4.mlp.down_proj.weight": "model-00003-of-00009.safetensors",
|
| 423 |
+
"language_model.layers.4.mlp.gate_proj.weight": "model-00002-of-00009.safetensors",
|
| 424 |
+
"language_model.layers.4.mlp.up_proj.weight": "model-00003-of-00009.safetensors",
|
| 425 |
+
"language_model.layers.4.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 426 |
+
"language_model.layers.5.input_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 427 |
+
"language_model.layers.5.linear_attn.A_log": "model-00003-of-00009.safetensors",
|
| 428 |
+
"language_model.layers.5.linear_attn.conv1d.weight": "model-00003-of-00009.safetensors",
|
| 429 |
+
"language_model.layers.5.linear_attn.dt_bias": "model-00003-of-00009.safetensors",
|
| 430 |
+
"language_model.layers.5.linear_attn.in_proj_a.weight": "model-00003-of-00009.safetensors",
|
| 431 |
+
"language_model.layers.5.linear_attn.in_proj_b.weight": "model-00003-of-00009.safetensors",
|
| 432 |
+
"language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00003-of-00009.safetensors",
|
| 433 |
+
"language_model.layers.5.linear_attn.in_proj_z.weight": "model-00003-of-00009.safetensors",
|
| 434 |
+
"language_model.layers.5.linear_attn.norm.weight": "model-00003-of-00009.safetensors",
|
| 435 |
+
"language_model.layers.5.linear_attn.out_proj.weight": "model-00003-of-00009.safetensors",
|
| 436 |
+
"language_model.layers.5.mlp.down_proj.weight": "model-00003-of-00009.safetensors",
|
| 437 |
+
"language_model.layers.5.mlp.gate_proj.weight": "model-00003-of-00009.safetensors",
|
| 438 |
+
"language_model.layers.5.mlp.up_proj.weight": "model-00003-of-00009.safetensors",
|
| 439 |
+
"language_model.layers.5.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 440 |
+
"language_model.layers.6.input_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 441 |
+
"language_model.layers.6.linear_attn.A_log": "model-00003-of-00009.safetensors",
|
| 442 |
+
"language_model.layers.6.linear_attn.conv1d.weight": "model-00003-of-00009.safetensors",
|
| 443 |
+
"language_model.layers.6.linear_attn.dt_bias": "model-00003-of-00009.safetensors",
|
| 444 |
+
"language_model.layers.6.linear_attn.in_proj_a.weight": "model-00003-of-00009.safetensors",
|
| 445 |
+
"language_model.layers.6.linear_attn.in_proj_b.weight": "model-00003-of-00009.safetensors",
|
| 446 |
+
"language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00003-of-00009.safetensors",
|
| 447 |
+
"language_model.layers.6.linear_attn.in_proj_z.weight": "model-00003-of-00009.safetensors",
|
| 448 |
+
"language_model.layers.6.linear_attn.norm.weight": "model-00003-of-00009.safetensors",
|
| 449 |
+
"language_model.layers.6.linear_attn.out_proj.weight": "model-00003-of-00009.safetensors",
|
| 450 |
+
"language_model.layers.6.mlp.down_proj.weight": "model-00003-of-00009.safetensors",
|
| 451 |
+
"language_model.layers.6.mlp.gate_proj.weight": "model-00003-of-00009.safetensors",
|
| 452 |
+
"language_model.layers.6.mlp.up_proj.weight": "model-00003-of-00009.safetensors",
|
| 453 |
+
"language_model.layers.6.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 454 |
+
"language_model.layers.7.input_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 455 |
+
"language_model.layers.7.mlp.down_proj.weight": "model-00003-of-00009.safetensors",
|
| 456 |
+
"language_model.layers.7.mlp.gate_proj.weight": "model-00003-of-00009.safetensors",
|
| 457 |
+
"language_model.layers.7.mlp.up_proj.weight": "model-00003-of-00009.safetensors",
|
| 458 |
+
"language_model.layers.7.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 459 |
+
"language_model.layers.7.self_attn.k_norm.weight": "model-00003-of-00009.safetensors",
|
| 460 |
+
"language_model.layers.7.self_attn.k_proj.weight": "model-00003-of-00009.safetensors",
|
| 461 |
+
"language_model.layers.7.self_attn.o_proj.weight": "model-00003-of-00009.safetensors",
|
| 462 |
+
"language_model.layers.7.self_attn.q_norm.weight": "model-00003-of-00009.safetensors",
|
| 463 |
+
"language_model.layers.7.self_attn.q_proj.weight": "model-00003-of-00009.safetensors",
|
| 464 |
+
"language_model.layers.7.self_attn.v_proj.weight": "model-00003-of-00009.safetensors",
|
| 465 |
+
"language_model.layers.8.input_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 466 |
+
"language_model.layers.8.linear_attn.A_log": "model-00003-of-00009.safetensors",
|
| 467 |
+
"language_model.layers.8.linear_attn.conv1d.weight": "model-00003-of-00009.safetensors",
|
| 468 |
+
"language_model.layers.8.linear_attn.dt_bias": "model-00003-of-00009.safetensors",
|
| 469 |
+
"language_model.layers.8.linear_attn.in_proj_a.weight": "model-00003-of-00009.safetensors",
|
| 470 |
+
"language_model.layers.8.linear_attn.in_proj_b.weight": "model-00003-of-00009.safetensors",
|
| 471 |
+
"language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00003-of-00009.safetensors",
|
| 472 |
+
"language_model.layers.8.linear_attn.in_proj_z.weight": "model-00003-of-00009.safetensors",
|
| 473 |
+
"language_model.layers.8.linear_attn.norm.weight": "model-00003-of-00009.safetensors",
|
| 474 |
+
"language_model.layers.8.linear_attn.out_proj.weight": "model-00003-of-00009.safetensors",
|
| 475 |
+
"language_model.layers.8.mlp.down_proj.weight": "model-00003-of-00009.safetensors",
|
| 476 |
+
"language_model.layers.8.mlp.gate_proj.weight": "model-00003-of-00009.safetensors",
|
| 477 |
+
"language_model.layers.8.mlp.up_proj.weight": "model-00003-of-00009.safetensors",
|
| 478 |
+
"language_model.layers.8.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
|
| 479 |
+
"language_model.layers.9.input_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 480 |
+
"language_model.layers.9.linear_attn.A_log": "model-00003-of-00009.safetensors",
|
| 481 |
+
"language_model.layers.9.linear_attn.conv1d.weight": "model-00003-of-00009.safetensors",
|
| 482 |
+
"language_model.layers.9.linear_attn.dt_bias": "model-00003-of-00009.safetensors",
|
| 483 |
+
"language_model.layers.9.linear_attn.in_proj_a.weight": "model-00004-of-00009.safetensors",
|
| 484 |
+
"language_model.layers.9.linear_attn.in_proj_b.weight": "model-00004-of-00009.safetensors",
|
| 485 |
+
"language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00004-of-00009.safetensors",
|
| 486 |
+
"language_model.layers.9.linear_attn.in_proj_z.weight": "model-00004-of-00009.safetensors",
|
| 487 |
+
"language_model.layers.9.linear_attn.norm.weight": "model-00003-of-00009.safetensors",
|
| 488 |
+
"language_model.layers.9.linear_attn.out_proj.weight": "model-00003-of-00009.safetensors",
|
| 489 |
+
"language_model.layers.9.mlp.down_proj.weight": "model-00004-of-00009.safetensors",
|
| 490 |
+
"language_model.layers.9.mlp.gate_proj.weight": "model-00004-of-00009.safetensors",
|
| 491 |
+
"language_model.layers.9.mlp.up_proj.weight": "model-00004-of-00009.safetensors",
|
| 492 |
+
"language_model.layers.9.post_attention_layernorm.weight": "model-00004-of-00009.safetensors",
|
| 493 |
+
"language_model.norm.weight": "model-00009-of-00009.safetensors",
|
| 494 |
+
"visual.blocks.0.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 495 |
+
"visual.blocks.0.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 496 |
+
"visual.blocks.0.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 497 |
+
"visual.blocks.0.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 498 |
+
"visual.blocks.0.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 499 |
+
"visual.blocks.0.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 500 |
+
"visual.blocks.0.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 501 |
+
"visual.blocks.0.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 502 |
+
"visual.blocks.0.norm1.bias": "model-00009-of-00009.safetensors",
|
| 503 |
+
"visual.blocks.0.norm1.weight": "model-00009-of-00009.safetensors",
|
| 504 |
+
"visual.blocks.0.norm2.bias": "model-00009-of-00009.safetensors",
|
| 505 |
+
"visual.blocks.0.norm2.weight": "model-00009-of-00009.safetensors",
|
| 506 |
+
"visual.blocks.1.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 507 |
+
"visual.blocks.1.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 508 |
+
"visual.blocks.1.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 509 |
+
"visual.blocks.1.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 510 |
+
"visual.blocks.1.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 511 |
+
"visual.blocks.1.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 512 |
+
"visual.blocks.1.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 513 |
+
"visual.blocks.1.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 514 |
+
"visual.blocks.1.norm1.bias": "model-00009-of-00009.safetensors",
|
| 515 |
+
"visual.blocks.1.norm1.weight": "model-00009-of-00009.safetensors",
|
| 516 |
+
"visual.blocks.1.norm2.bias": "model-00009-of-00009.safetensors",
|
| 517 |
+
"visual.blocks.1.norm2.weight": "model-00009-of-00009.safetensors",
|
| 518 |
+
"visual.blocks.10.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 519 |
+
"visual.blocks.10.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 520 |
+
"visual.blocks.10.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 521 |
+
"visual.blocks.10.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 522 |
+
"visual.blocks.10.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 523 |
+
"visual.blocks.10.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 524 |
+
"visual.blocks.10.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 525 |
+
"visual.blocks.10.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 526 |
+
"visual.blocks.10.norm1.bias": "model-00009-of-00009.safetensors",
|
| 527 |
+
"visual.blocks.10.norm1.weight": "model-00009-of-00009.safetensors",
|
| 528 |
+
"visual.blocks.10.norm2.bias": "model-00009-of-00009.safetensors",
|
| 529 |
+
"visual.blocks.10.norm2.weight": "model-00009-of-00009.safetensors",
|
| 530 |
+
"visual.blocks.11.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 531 |
+
"visual.blocks.11.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 532 |
+
"visual.blocks.11.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 533 |
+
"visual.blocks.11.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 534 |
+
"visual.blocks.11.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 535 |
+
"visual.blocks.11.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 536 |
+
"visual.blocks.11.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 537 |
+
"visual.blocks.11.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 538 |
+
"visual.blocks.11.norm1.bias": "model-00009-of-00009.safetensors",
|
| 539 |
+
"visual.blocks.11.norm1.weight": "model-00009-of-00009.safetensors",
|
| 540 |
+
"visual.blocks.11.norm2.bias": "model-00009-of-00009.safetensors",
|
| 541 |
+
"visual.blocks.11.norm2.weight": "model-00009-of-00009.safetensors",
|
| 542 |
+
"visual.blocks.12.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 543 |
+
"visual.blocks.12.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 544 |
+
"visual.blocks.12.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 545 |
+
"visual.blocks.12.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 546 |
+
"visual.blocks.12.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 547 |
+
"visual.blocks.12.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 548 |
+
"visual.blocks.12.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 549 |
+
"visual.blocks.12.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 550 |
+
"visual.blocks.12.norm1.bias": "model-00009-of-00009.safetensors",
|
| 551 |
+
"visual.blocks.12.norm1.weight": "model-00009-of-00009.safetensors",
|
| 552 |
+
"visual.blocks.12.norm2.bias": "model-00009-of-00009.safetensors",
|
| 553 |
+
"visual.blocks.12.norm2.weight": "model-00009-of-00009.safetensors",
|
| 554 |
+
"visual.blocks.13.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 555 |
+
"visual.blocks.13.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 556 |
+
"visual.blocks.13.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 557 |
+
"visual.blocks.13.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 558 |
+
"visual.blocks.13.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 559 |
+
"visual.blocks.13.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 560 |
+
"visual.blocks.13.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 561 |
+
"visual.blocks.13.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 562 |
+
"visual.blocks.13.norm1.bias": "model-00009-of-00009.safetensors",
|
| 563 |
+
"visual.blocks.13.norm1.weight": "model-00009-of-00009.safetensors",
|
| 564 |
+
"visual.blocks.13.norm2.bias": "model-00009-of-00009.safetensors",
|
| 565 |
+
"visual.blocks.13.norm2.weight": "model-00009-of-00009.safetensors",
|
| 566 |
+
"visual.blocks.14.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 567 |
+
"visual.blocks.14.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 568 |
+
"visual.blocks.14.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 569 |
+
"visual.blocks.14.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 570 |
+
"visual.blocks.14.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 571 |
+
"visual.blocks.14.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 572 |
+
"visual.blocks.14.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 573 |
+
"visual.blocks.14.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 574 |
+
"visual.blocks.14.norm1.bias": "model-00009-of-00009.safetensors",
|
| 575 |
+
"visual.blocks.14.norm1.weight": "model-00009-of-00009.safetensors",
|
| 576 |
+
"visual.blocks.14.norm2.bias": "model-00009-of-00009.safetensors",
|
| 577 |
+
"visual.blocks.14.norm2.weight": "model-00009-of-00009.safetensors",
|
| 578 |
+
"visual.blocks.15.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 579 |
+
"visual.blocks.15.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 580 |
+
"visual.blocks.15.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 581 |
+
"visual.blocks.15.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 582 |
+
"visual.blocks.15.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 583 |
+
"visual.blocks.15.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 584 |
+
"visual.blocks.15.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 585 |
+
"visual.blocks.15.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 586 |
+
"visual.blocks.15.norm1.bias": "model-00009-of-00009.safetensors",
|
| 587 |
+
"visual.blocks.15.norm1.weight": "model-00009-of-00009.safetensors",
|
| 588 |
+
"visual.blocks.15.norm2.bias": "model-00009-of-00009.safetensors",
|
| 589 |
+
"visual.blocks.15.norm2.weight": "model-00009-of-00009.safetensors",
|
| 590 |
+
"visual.blocks.16.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 591 |
+
"visual.blocks.16.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 592 |
+
"visual.blocks.16.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 593 |
+
"visual.blocks.16.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 594 |
+
"visual.blocks.16.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 595 |
+
"visual.blocks.16.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 596 |
+
"visual.blocks.16.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 597 |
+
"visual.blocks.16.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 598 |
+
"visual.blocks.16.norm1.bias": "model-00009-of-00009.safetensors",
|
| 599 |
+
"visual.blocks.16.norm1.weight": "model-00009-of-00009.safetensors",
|
| 600 |
+
"visual.blocks.16.norm2.bias": "model-00009-of-00009.safetensors",
|
| 601 |
+
"visual.blocks.16.norm2.weight": "model-00009-of-00009.safetensors",
|
| 602 |
+
"visual.blocks.17.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 603 |
+
"visual.blocks.17.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 604 |
+
"visual.blocks.17.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 605 |
+
"visual.blocks.17.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 606 |
+
"visual.blocks.17.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 607 |
+
"visual.blocks.17.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 608 |
+
"visual.blocks.17.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 609 |
+
"visual.blocks.17.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 610 |
+
"visual.blocks.17.norm1.bias": "model-00009-of-00009.safetensors",
|
| 611 |
+
"visual.blocks.17.norm1.weight": "model-00009-of-00009.safetensors",
|
| 612 |
+
"visual.blocks.17.norm2.bias": "model-00009-of-00009.safetensors",
|
| 613 |
+
"visual.blocks.17.norm2.weight": "model-00009-of-00009.safetensors",
|
| 614 |
+
"visual.blocks.18.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 615 |
+
"visual.blocks.18.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 616 |
+
"visual.blocks.18.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 617 |
+
"visual.blocks.18.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 618 |
+
"visual.blocks.18.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 619 |
+
"visual.blocks.18.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 620 |
+
"visual.blocks.18.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 621 |
+
"visual.blocks.18.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 622 |
+
"visual.blocks.18.norm1.bias": "model-00009-of-00009.safetensors",
|
| 623 |
+
"visual.blocks.18.norm1.weight": "model-00009-of-00009.safetensors",
|
| 624 |
+
"visual.blocks.18.norm2.bias": "model-00009-of-00009.safetensors",
|
| 625 |
+
"visual.blocks.18.norm2.weight": "model-00009-of-00009.safetensors",
|
| 626 |
+
"visual.blocks.19.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 627 |
+
"visual.blocks.19.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 628 |
+
"visual.blocks.19.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 629 |
+
"visual.blocks.19.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 630 |
+
"visual.blocks.19.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 631 |
+
"visual.blocks.19.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 632 |
+
"visual.blocks.19.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 633 |
+
"visual.blocks.19.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 634 |
+
"visual.blocks.19.norm1.bias": "model-00009-of-00009.safetensors",
|
| 635 |
+
"visual.blocks.19.norm1.weight": "model-00009-of-00009.safetensors",
|
| 636 |
+
"visual.blocks.19.norm2.bias": "model-00009-of-00009.safetensors",
|
| 637 |
+
"visual.blocks.19.norm2.weight": "model-00009-of-00009.safetensors",
|
| 638 |
+
"visual.blocks.2.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 639 |
+
"visual.blocks.2.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 640 |
+
"visual.blocks.2.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 641 |
+
"visual.blocks.2.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 642 |
+
"visual.blocks.2.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 643 |
+
"visual.blocks.2.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 644 |
+
"visual.blocks.2.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 645 |
+
"visual.blocks.2.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 646 |
+
"visual.blocks.2.norm1.bias": "model-00009-of-00009.safetensors",
|
| 647 |
+
"visual.blocks.2.norm1.weight": "model-00009-of-00009.safetensors",
|
| 648 |
+
"visual.blocks.2.norm2.bias": "model-00009-of-00009.safetensors",
|
| 649 |
+
"visual.blocks.2.norm2.weight": "model-00009-of-00009.safetensors",
|
| 650 |
+
"visual.blocks.20.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 651 |
+
"visual.blocks.20.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 652 |
+
"visual.blocks.20.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 653 |
+
"visual.blocks.20.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 654 |
+
"visual.blocks.20.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 655 |
+
"visual.blocks.20.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 656 |
+
"visual.blocks.20.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 657 |
+
"visual.blocks.20.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 658 |
+
"visual.blocks.20.norm1.bias": "model-00009-of-00009.safetensors",
|
| 659 |
+
"visual.blocks.20.norm1.weight": "model-00009-of-00009.safetensors",
|
| 660 |
+
"visual.blocks.20.norm2.bias": "model-00009-of-00009.safetensors",
|
| 661 |
+
"visual.blocks.20.norm2.weight": "model-00009-of-00009.safetensors",
|
| 662 |
+
"visual.blocks.21.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 663 |
+
"visual.blocks.21.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 664 |
+
"visual.blocks.21.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 665 |
+
"visual.blocks.21.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 666 |
+
"visual.blocks.21.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 667 |
+
"visual.blocks.21.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 668 |
+
"visual.blocks.21.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 669 |
+
"visual.blocks.21.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 670 |
+
"visual.blocks.21.norm1.bias": "model-00009-of-00009.safetensors",
|
| 671 |
+
"visual.blocks.21.norm1.weight": "model-00009-of-00009.safetensors",
|
| 672 |
+
"visual.blocks.21.norm2.bias": "model-00009-of-00009.safetensors",
|
| 673 |
+
"visual.blocks.21.norm2.weight": "model-00009-of-00009.safetensors",
|
| 674 |
+
"visual.blocks.22.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 675 |
+
"visual.blocks.22.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 676 |
+
"visual.blocks.22.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 677 |
+
"visual.blocks.22.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 678 |
+
"visual.blocks.22.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 679 |
+
"visual.blocks.22.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 680 |
+
"visual.blocks.22.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 681 |
+
"visual.blocks.22.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 682 |
+
"visual.blocks.22.norm1.bias": "model-00009-of-00009.safetensors",
|
| 683 |
+
"visual.blocks.22.norm1.weight": "model-00009-of-00009.safetensors",
|
| 684 |
+
"visual.blocks.22.norm2.bias": "model-00009-of-00009.safetensors",
|
| 685 |
+
"visual.blocks.22.norm2.weight": "model-00009-of-00009.safetensors",
|
| 686 |
+
"visual.blocks.23.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 687 |
+
"visual.blocks.23.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 688 |
+
"visual.blocks.23.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 689 |
+
"visual.blocks.23.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 690 |
+
"visual.blocks.23.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 691 |
+
"visual.blocks.23.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 692 |
+
"visual.blocks.23.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 693 |
+
"visual.blocks.23.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 694 |
+
"visual.blocks.23.norm1.bias": "model-00009-of-00009.safetensors",
|
| 695 |
+
"visual.blocks.23.norm1.weight": "model-00009-of-00009.safetensors",
|
| 696 |
+
"visual.blocks.23.norm2.bias": "model-00009-of-00009.safetensors",
|
| 697 |
+
"visual.blocks.23.norm2.weight": "model-00009-of-00009.safetensors",
|
| 698 |
+
"visual.blocks.24.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 699 |
+
"visual.blocks.24.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 700 |
+
"visual.blocks.24.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 701 |
+
"visual.blocks.24.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 702 |
+
"visual.blocks.24.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 703 |
+
"visual.blocks.24.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 704 |
+
"visual.blocks.24.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 705 |
+
"visual.blocks.24.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 706 |
+
"visual.blocks.24.norm1.bias": "model-00009-of-00009.safetensors",
|
| 707 |
+
"visual.blocks.24.norm1.weight": "model-00009-of-00009.safetensors",
|
| 708 |
+
"visual.blocks.24.norm2.bias": "model-00009-of-00009.safetensors",
|
| 709 |
+
"visual.blocks.24.norm2.weight": "model-00009-of-00009.safetensors",
|
| 710 |
+
"visual.blocks.25.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 711 |
+
"visual.blocks.25.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 712 |
+
"visual.blocks.25.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 713 |
+
"visual.blocks.25.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 714 |
+
"visual.blocks.25.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 715 |
+
"visual.blocks.25.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 716 |
+
"visual.blocks.25.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 717 |
+
"visual.blocks.25.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 718 |
+
"visual.blocks.25.norm1.bias": "model-00009-of-00009.safetensors",
|
| 719 |
+
"visual.blocks.25.norm1.weight": "model-00009-of-00009.safetensors",
|
| 720 |
+
"visual.blocks.25.norm2.bias": "model-00009-of-00009.safetensors",
|
| 721 |
+
"visual.blocks.25.norm2.weight": "model-00009-of-00009.safetensors",
|
| 722 |
+
"visual.blocks.26.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 723 |
+
"visual.blocks.26.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 724 |
+
"visual.blocks.26.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 725 |
+
"visual.blocks.26.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 726 |
+
"visual.blocks.26.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 727 |
+
"visual.blocks.26.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 728 |
+
"visual.blocks.26.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 729 |
+
"visual.blocks.26.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 730 |
+
"visual.blocks.26.norm1.bias": "model-00009-of-00009.safetensors",
|
| 731 |
+
"visual.blocks.26.norm1.weight": "model-00009-of-00009.safetensors",
|
| 732 |
+
"visual.blocks.26.norm2.bias": "model-00009-of-00009.safetensors",
|
| 733 |
+
"visual.blocks.26.norm2.weight": "model-00009-of-00009.safetensors",
|
| 734 |
+
"visual.blocks.3.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 735 |
+
"visual.blocks.3.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 736 |
+
"visual.blocks.3.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 737 |
+
"visual.blocks.3.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 738 |
+
"visual.blocks.3.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 739 |
+
"visual.blocks.3.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 740 |
+
"visual.blocks.3.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 741 |
+
"visual.blocks.3.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 742 |
+
"visual.blocks.3.norm1.bias": "model-00009-of-00009.safetensors",
|
| 743 |
+
"visual.blocks.3.norm1.weight": "model-00009-of-00009.safetensors",
|
| 744 |
+
"visual.blocks.3.norm2.bias": "model-00009-of-00009.safetensors",
|
| 745 |
+
"visual.blocks.3.norm2.weight": "model-00009-of-00009.safetensors",
|
| 746 |
+
"visual.blocks.4.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 747 |
+
"visual.blocks.4.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 748 |
+
"visual.blocks.4.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 749 |
+
"visual.blocks.4.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 750 |
+
"visual.blocks.4.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 751 |
+
"visual.blocks.4.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 752 |
+
"visual.blocks.4.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 753 |
+
"visual.blocks.4.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 754 |
+
"visual.blocks.4.norm1.bias": "model-00009-of-00009.safetensors",
|
| 755 |
+
"visual.blocks.4.norm1.weight": "model-00009-of-00009.safetensors",
|
| 756 |
+
"visual.blocks.4.norm2.bias": "model-00009-of-00009.safetensors",
|
| 757 |
+
"visual.blocks.4.norm2.weight": "model-00009-of-00009.safetensors",
|
| 758 |
+
"visual.blocks.5.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 759 |
+
"visual.blocks.5.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 760 |
+
"visual.blocks.5.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 761 |
+
"visual.blocks.5.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 762 |
+
"visual.blocks.5.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 763 |
+
"visual.blocks.5.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 764 |
+
"visual.blocks.5.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 765 |
+
"visual.blocks.5.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 766 |
+
"visual.blocks.5.norm1.bias": "model-00009-of-00009.safetensors",
|
| 767 |
+
"visual.blocks.5.norm1.weight": "model-00009-of-00009.safetensors",
|
| 768 |
+
"visual.blocks.5.norm2.bias": "model-00009-of-00009.safetensors",
|
| 769 |
+
"visual.blocks.5.norm2.weight": "model-00009-of-00009.safetensors",
|
| 770 |
+
"visual.blocks.6.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 771 |
+
"visual.blocks.6.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 772 |
+
"visual.blocks.6.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 773 |
+
"visual.blocks.6.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 774 |
+
"visual.blocks.6.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 775 |
+
"visual.blocks.6.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 776 |
+
"visual.blocks.6.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 777 |
+
"visual.blocks.6.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 778 |
+
"visual.blocks.6.norm1.bias": "model-00009-of-00009.safetensors",
|
| 779 |
+
"visual.blocks.6.norm1.weight": "model-00009-of-00009.safetensors",
|
| 780 |
+
"visual.blocks.6.norm2.bias": "model-00009-of-00009.safetensors",
|
| 781 |
+
"visual.blocks.6.norm2.weight": "model-00009-of-00009.safetensors",
|
| 782 |
+
"visual.blocks.7.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 783 |
+
"visual.blocks.7.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 784 |
+
"visual.blocks.7.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 785 |
+
"visual.blocks.7.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 786 |
+
"visual.blocks.7.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 787 |
+
"visual.blocks.7.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 788 |
+
"visual.blocks.7.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 789 |
+
"visual.blocks.7.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 790 |
+
"visual.blocks.7.norm1.bias": "model-00009-of-00009.safetensors",
|
| 791 |
+
"visual.blocks.7.norm1.weight": "model-00009-of-00009.safetensors",
|
| 792 |
+
"visual.blocks.7.norm2.bias": "model-00009-of-00009.safetensors",
|
| 793 |
+
"visual.blocks.7.norm2.weight": "model-00009-of-00009.safetensors",
|
| 794 |
+
"visual.blocks.8.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 795 |
+
"visual.blocks.8.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 796 |
+
"visual.blocks.8.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 797 |
+
"visual.blocks.8.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 798 |
+
"visual.blocks.8.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 799 |
+
"visual.blocks.8.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 800 |
+
"visual.blocks.8.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 801 |
+
"visual.blocks.8.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 802 |
+
"visual.blocks.8.norm1.bias": "model-00009-of-00009.safetensors",
|
| 803 |
+
"visual.blocks.8.norm1.weight": "model-00009-of-00009.safetensors",
|
| 804 |
+
"visual.blocks.8.norm2.bias": "model-00009-of-00009.safetensors",
|
| 805 |
+
"visual.blocks.8.norm2.weight": "model-00009-of-00009.safetensors",
|
| 806 |
+
"visual.blocks.9.attn.proj.bias": "model-00009-of-00009.safetensors",
|
| 807 |
+
"visual.blocks.9.attn.proj.weight": "model-00009-of-00009.safetensors",
|
| 808 |
+
"visual.blocks.9.attn.qkv.bias": "model-00009-of-00009.safetensors",
|
| 809 |
+
"visual.blocks.9.attn.qkv.weight": "model-00009-of-00009.safetensors",
|
| 810 |
+
"visual.blocks.9.mlp.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 811 |
+
"visual.blocks.9.mlp.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 812 |
+
"visual.blocks.9.mlp.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 813 |
+
"visual.blocks.9.mlp.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 814 |
+
"visual.blocks.9.norm1.bias": "model-00009-of-00009.safetensors",
|
| 815 |
+
"visual.blocks.9.norm1.weight": "model-00009-of-00009.safetensors",
|
| 816 |
+
"visual.blocks.9.norm2.bias": "model-00009-of-00009.safetensors",
|
| 817 |
+
"visual.blocks.9.norm2.weight": "model-00009-of-00009.safetensors",
|
| 818 |
+
"visual.merger.linear_fc1.bias": "model-00009-of-00009.safetensors",
|
| 819 |
+
"visual.merger.linear_fc1.weight": "model-00009-of-00009.safetensors",
|
| 820 |
+
"visual.merger.linear_fc2.bias": "model-00009-of-00009.safetensors",
|
| 821 |
+
"visual.merger.linear_fc2.weight": "model-00009-of-00009.safetensors",
|
| 822 |
+
"visual.merger.norm.bias": "model-00009-of-00009.safetensors",
|
| 823 |
+
"visual.merger.norm.weight": "model-00009-of-00009.safetensors",
|
| 824 |
+
"visual.patch_embed.proj.bias": "model-00009-of-00009.safetensors",
|
| 825 |
+
"visual.patch_embed.proj.weight": "model-00009-of-00009.safetensors",
|
| 826 |
+
"visual.pos_embed.weight": "model-00009-of-00009.safetensors"
|
| 827 |
+
}
|
| 828 |
+
}
|
modular_gs1.py
ADDED
|
@@ -0,0 +1,971 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GS1: native hybrid text/vision decision model with attention residuals."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
from torch.nn import functional as F
|
| 9 |
+
from torch.nn.attention import SDPBackend, sdpa_kernel
|
| 10 |
+
from torch.utils.checkpoint import checkpoint
|
| 11 |
+
from transformers import AutoConfig, AutoModel, Qwen3_5Config
|
| 12 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import (
|
| 13 |
+
Qwen3_5Attention,
|
| 14 |
+
Qwen3_5DynamicCache,
|
| 15 |
+
Qwen3_5GatedDeltaNet,
|
| 16 |
+
Qwen3_5MLP,
|
| 17 |
+
Qwen3_5PreTrainedModel,
|
| 18 |
+
Qwen3_5RMSNorm,
|
| 19 |
+
Qwen3_5TextRotaryEmbedding,
|
| 20 |
+
Qwen3_5VisionModel,
|
| 21 |
+
apply_rotary_pos_emb,
|
| 22 |
+
)
|
| 23 |
+
from transformers.utils import ModelOutput
|
| 24 |
+
|
| 25 |
+
from .config import INFERENCE_LIMITS, CalibrationConfig, DecisionConfig
|
| 26 |
+
from .util import named_seed
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class GS1Config(Qwen3_5Config):
|
| 30 |
+
model_type = "gs1"
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
decision=None,
|
| 35 |
+
calibration=None,
|
| 36 |
+
initialization_seed=12,
|
| 37 |
+
head_compute_dtype="float32",
|
| 38 |
+
inference_limits=None,
|
| 39 |
+
**kwargs,
|
| 40 |
+
):
|
| 41 |
+
kwargs.pop("model_type", None)
|
| 42 |
+
kwargs.pop("architectures", None)
|
| 43 |
+
super().__init__(**kwargs)
|
| 44 |
+
self.decision = DecisionConfig.model_validate(decision or {}).model_dump()
|
| 45 |
+
self.calibration = CalibrationConfig.model_validate(calibration or {}).model_dump()
|
| 46 |
+
self.initialization_seed = initialization_seed
|
| 47 |
+
self.inference_limits = {**INFERENCE_LIMITS, **(inference_limits or {})}
|
| 48 |
+
if self.inference_limits.keys() != INFERENCE_LIMITS.keys() or any(
|
| 49 |
+
type(value) is not int or value < 1 for value in self.inference_limits.values()
|
| 50 |
+
):
|
| 51 |
+
raise ValueError("Invalid GS1 inference limits")
|
| 52 |
+
if head_compute_dtype not in {"float32", "bfloat16"}:
|
| 53 |
+
raise ValueError("head_compute_dtype must be float32 or bfloat16")
|
| 54 |
+
self.head_compute_dtype = head_compute_dtype
|
| 55 |
+
self.architectures = ["GS1Model"]
|
| 56 |
+
self.tie_word_embeddings = False
|
| 57 |
+
self.text_config.use_cache = False
|
| 58 |
+
self.text_config._attn_implementation = "sdpa"
|
| 59 |
+
self.vision_config._attn_implementation = "sdpa"
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass
|
| 63 |
+
class GS1Output(ModelOutput):
|
| 64 |
+
logits: torch.Tensor | None = None
|
| 65 |
+
last_hidden_state: torch.Tensor | None = None
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@dataclass
|
| 69 |
+
class GS1TextOutput(ModelOutput):
|
| 70 |
+
last_hidden_state: torch.Tensor | None = None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@dataclass(frozen=True)
|
| 74 |
+
class GDNState:
|
| 75 |
+
"""Native GDN prefix, including the last complete scan-chunk boundary."""
|
| 76 |
+
|
| 77 |
+
conv: torch.Tensor
|
| 78 |
+
recurrent: torch.Tensor
|
| 79 |
+
boundary_conv: torch.Tensor | None
|
| 80 |
+
boundary_recurrent: torch.Tensor | None
|
| 81 |
+
tail: torch.Tensor
|
| 82 |
+
position: int
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class GS1DecoderLayer(nn.Module):
|
| 86 |
+
"""A hybrid decoder block; only its full-attention input can read depth history."""
|
| 87 |
+
|
| 88 |
+
def __init__(self, config, index):
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.layer_type = config.layer_types[index]
|
| 91 |
+
self.config = config
|
| 92 |
+
self.index = index
|
| 93 |
+
if self.layer_type == "full_attention":
|
| 94 |
+
self.self_attn = Qwen3_5Attention(config, index)
|
| 95 |
+
elif self.layer_type == "linear_attention":
|
| 96 |
+
self.linear_attn = Qwen3_5GatedDeltaNet(config, index)
|
| 97 |
+
else:
|
| 98 |
+
raise ValueError(f"Unsupported decoder layer type: {self.layer_type}")
|
| 99 |
+
self.mlp = Qwen3_5MLP(config, config.intermediate_size)
|
| 100 |
+
self.input_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 101 |
+
self.post_attention_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 102 |
+
|
| 103 |
+
def forward(self, carry, positional, mask, read, terminal_query, valid_lengths=None, cohorts=None):
|
| 104 |
+
if self.layer_type == "full_attention":
|
| 105 |
+
mixed, _ = self.self_attn(
|
| 106 |
+
self.input_layernorm(read), position_embeddings=positional, attention_mask=mask
|
| 107 |
+
)
|
| 108 |
+
elif valid_lengths is not None and min(valid_lengths) != carry.shape[1]:
|
| 109 |
+
# GDN scans only real tokens. Equal-length cohorts retain native
|
| 110 |
+
# batched kernels and their original chunk boundaries; FA and FFN
|
| 111 |
+
# still execute over the complete decoder batch.
|
| 112 |
+
mixed = torch.zeros_like(carry)
|
| 113 |
+
if cohorts is None:
|
| 114 |
+
cohorts = tuple(
|
| 115 |
+
(
|
| 116 |
+
length,
|
| 117 |
+
torch.tensor(
|
| 118 |
+
[i for i, value in enumerate(valid_lengths) if value == length],
|
| 119 |
+
device=carry.device,
|
| 120 |
+
),
|
| 121 |
+
)
|
| 122 |
+
for length in sorted(set(valid_lengths))
|
| 123 |
+
)
|
| 124 |
+
for length, rows in cohorts:
|
| 125 |
+
value = carry.index_select(0, rows)[:, -length:]
|
| 126 |
+
normalized = self.input_layernorm(value)
|
| 127 |
+
if terminal_query:
|
| 128 |
+
cache = Qwen3_5DynamicCache(self.config)
|
| 129 |
+
cache.last_linear_layer = self.index
|
| 130 |
+
context = self.linear_attn(
|
| 131 |
+
normalized[:, :-1],
|
| 132 |
+
cache_params=cache,
|
| 133 |
+
cache_position=torch.arange(length - 1, device=carry.device),
|
| 134 |
+
)
|
| 135 |
+
query, _ = _linear_context(
|
| 136 |
+
self.linear_attn,
|
| 137 |
+
normalized[:, -1:],
|
| 138 |
+
(cache.conv_states[self.index], cache.recurrent_states[self.index]),
|
| 139 |
+
self.config,
|
| 140 |
+
)
|
| 141 |
+
value = torch.cat((context, query), 1)
|
| 142 |
+
else:
|
| 143 |
+
value = self.linear_attn(normalized)
|
| 144 |
+
mixed = mixed.index_copy(
|
| 145 |
+
0, rows, F.pad(value, (0, 0, carry.shape[1] - length, 0)).to(mixed.dtype)
|
| 146 |
+
)
|
| 147 |
+
elif terminal_query:
|
| 148 |
+
# Keep the trained context scan and differentiable one-token continuation.
|
| 149 |
+
cache = Qwen3_5DynamicCache(self.config)
|
| 150 |
+
cache.last_linear_layer = self.index
|
| 151 |
+
context = carry[:, :-1]
|
| 152 |
+
context_mixed = self.linear_attn(
|
| 153 |
+
self.input_layernorm(context),
|
| 154 |
+
cache_params=cache,
|
| 155 |
+
cache_position=torch.arange(context.shape[1], device=carry.device),
|
| 156 |
+
)
|
| 157 |
+
query_mixed, _ = _linear_context(
|
| 158 |
+
self.linear_attn,
|
| 159 |
+
self.input_layernorm(carry[:, -1:]),
|
| 160 |
+
(cache.conv_states[self.index], cache.recurrent_states[self.index]),
|
| 161 |
+
self.config,
|
| 162 |
+
)
|
| 163 |
+
context = context + context_mixed
|
| 164 |
+
query = carry[:, -1:] + query_mixed
|
| 165 |
+
context_ffn = self.mlp(self.post_attention_layernorm(context))
|
| 166 |
+
query_ffn = self.mlp(self.post_attention_layernorm(query))
|
| 167 |
+
return (
|
| 168 |
+
torch.cat((context + context_ffn, query + query_ffn), 1),
|
| 169 |
+
torch.cat((context_mixed, query_mixed), 1),
|
| 170 |
+
torch.cat((context_ffn, query_ffn), 1),
|
| 171 |
+
)
|
| 172 |
+
else:
|
| 173 |
+
mixed = self.linear_attn(self.input_layernorm(carry))
|
| 174 |
+
hidden = carry + mixed
|
| 175 |
+
ffn = self.mlp(self.post_attention_layernorm(hidden))
|
| 176 |
+
return hidden + ffn, mixed, ffn
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class CrossAttentionDecisionHead(nn.Module):
|
| 180 |
+
"""Read state at backbone width; project only the final dot-product vectors."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, hidden_size, decision=None, seed=12):
|
| 183 |
+
super().__init__()
|
| 184 |
+
self.config = DecisionConfig.model_validate(decision or {})
|
| 185 |
+
if hidden_size % self.config.cross_attention_heads:
|
| 186 |
+
raise ValueError("cross_attention_heads must divide the backbone hidden size")
|
| 187 |
+
width = self.config.projection_size
|
| 188 |
+
eps = self.config.norm_eps
|
| 189 |
+
self.bf16_compute = False
|
| 190 |
+
with torch.random.fork_rng(devices=[]):
|
| 191 |
+
torch.manual_seed(named_seed(seed, "initialization/cross_attention_head"))
|
| 192 |
+
self.memory_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 193 |
+
self.memory_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 194 |
+
self.question_seed = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 195 |
+
self.question_attention = nn.MultiheadAttention(
|
| 196 |
+
hidden_size, self.config.cross_attention_heads, batch_first=True
|
| 197 |
+
)
|
| 198 |
+
self.decision_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 199 |
+
self.question_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 200 |
+
self.question_evidence_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 201 |
+
self.context_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 202 |
+
self.option_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 203 |
+
self.condition_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 204 |
+
self.option_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 205 |
+
self.routing_attention = nn.ModuleList(
|
| 206 |
+
nn.MultiheadAttention(hidden_size, self.config.cross_attention_heads, batch_first=True)
|
| 207 |
+
for _ in range(self.config.cross_attention_layers)
|
| 208 |
+
)
|
| 209 |
+
self.routing_attention_norm = nn.ModuleList(
|
| 210 |
+
nn.LayerNorm(hidden_size, eps=eps) for _ in range(self.config.cross_attention_layers)
|
| 211 |
+
)
|
| 212 |
+
self.routing_ffn = nn.ModuleList(
|
| 213 |
+
nn.Sequential(
|
| 214 |
+
nn.Linear(hidden_size, hidden_size),
|
| 215 |
+
nn.GELU(),
|
| 216 |
+
nn.Linear(hidden_size, hidden_size),
|
| 217 |
+
)
|
| 218 |
+
for _ in range(self.config.cross_attention_layers)
|
| 219 |
+
)
|
| 220 |
+
self.routing_output_norm = nn.ModuleList(
|
| 221 |
+
nn.LayerNorm(hidden_size, eps=eps) for _ in range(self.config.cross_attention_layers)
|
| 222 |
+
)
|
| 223 |
+
self.query_mlp = nn.Sequential(
|
| 224 |
+
nn.Linear(hidden_size, hidden_size), nn.GELU(), nn.Linear(hidden_size, width)
|
| 225 |
+
)
|
| 226 |
+
self.key_mlp = nn.Sequential(
|
| 227 |
+
nn.Linear(hidden_size, hidden_size), nn.GELU(), nn.Linear(hidden_size, width)
|
| 228 |
+
)
|
| 229 |
+
self.query_norm = nn.LayerNorm(width, eps=eps)
|
| 230 |
+
self.key_norm = nn.LayerNorm(width, eps=eps)
|
| 231 |
+
|
| 232 |
+
def prepare_memory(self, state):
|
| 233 |
+
"""Prepare immutable state memory and each attention module's own K/V."""
|
| 234 |
+
dtype = torch.bfloat16 if self.bf16_compute else self.memory_projection.weight.dtype
|
| 235 |
+
with torch.autocast(device_type=state.device.type, dtype=torch.bfloat16, enabled=self.bf16_compute):
|
| 236 |
+
memory = self.memory_projection(self.memory_norm(state.to(dtype))).unsqueeze(0)
|
| 237 |
+
projections = []
|
| 238 |
+
for attention in (self.question_attention, *self.routing_attention):
|
| 239 |
+
width = attention.embed_dim
|
| 240 |
+
bias = attention.in_proj_bias[width:] if attention.in_proj_bias is not None else None
|
| 241 |
+
key, value = F.linear(memory, attention.in_proj_weight[width:], bias).chunk(2, -1)
|
| 242 |
+
shape = (1, memory.shape[1], attention.num_heads, width // attention.num_heads)
|
| 243 |
+
projections.append((key.reshape(shape).transpose(1, 2), value.reshape(shape).transpose(1, 2)))
|
| 244 |
+
return memory, tuple(projections)
|
| 245 |
+
|
| 246 |
+
@staticmethod
|
| 247 |
+
def _attend(attention, query, memory, projected=None):
|
| 248 |
+
if projected is None:
|
| 249 |
+
return attention(query, memory, memory, need_weights=False)[0]
|
| 250 |
+
width = attention.embed_dim
|
| 251 |
+
bias = attention.in_proj_bias[:width] if attention.in_proj_bias is not None else None
|
| 252 |
+
query = F.linear(query, attention.in_proj_weight[:width], bias)
|
| 253 |
+
batch, length = query.shape[:2]
|
| 254 |
+
query = query.reshape(batch, length, attention.num_heads, width // attention.num_heads).transpose(
|
| 255 |
+
1, 2
|
| 256 |
+
)
|
| 257 |
+
value = F.scaled_dot_product_attention(query, *projected, dropout_p=0.0)
|
| 258 |
+
value = value.transpose(1, 2).reshape(batch, length, width)
|
| 259 |
+
return F.linear(value, attention.out_proj.weight, attention.out_proj.bias)
|
| 260 |
+
|
| 261 |
+
def forward(self, state, question, decision, options, *, prepared_memory=None):
|
| 262 |
+
if state.ndim != 2 or question.ndim != 1 or decision.ndim != 1 or options.ndim != 2:
|
| 263 |
+
raise ValueError(
|
| 264 |
+
"Cross-attention readout expects state [T,D], question/decision [D], options [K,D]"
|
| 265 |
+
)
|
| 266 |
+
if not len(state) or not len(options):
|
| 267 |
+
raise ValueError("Cross-attention readout requires state and candidates")
|
| 268 |
+
dtype = torch.bfloat16 if self.bf16_compute else self.memory_projection.weight.dtype
|
| 269 |
+
with torch.autocast(device_type=state.device.type, dtype=torch.bfloat16, enabled=self.bf16_compute):
|
| 270 |
+
state = state.to(dtype)
|
| 271 |
+
question = question.to(dtype)
|
| 272 |
+
decision = decision.to(dtype)
|
| 273 |
+
options = options.to(dtype)
|
| 274 |
+
memory, projected = (
|
| 275 |
+
(self.memory_projection(self.memory_norm(state)).unsqueeze(0), None)
|
| 276 |
+
if prepared_memory is None
|
| 277 |
+
else prepared_memory
|
| 278 |
+
)
|
| 279 |
+
question_query = self.question_seed(question)[None, None]
|
| 280 |
+
question_evidence = self._attend(
|
| 281 |
+
self.question_attention, question_query, memory, None if projected is None else projected[0]
|
| 282 |
+
)[0, 0]
|
| 283 |
+
context = self.context_norm(
|
| 284 |
+
self.decision_projection(decision)
|
| 285 |
+
+ self.question_projection(question)
|
| 286 |
+
+ self.question_evidence_projection(question_evidence)
|
| 287 |
+
)
|
| 288 |
+
routed = self.option_norm(self.option_projection(options) + self.condition_projection(context))
|
| 289 |
+
for layer_index, (attention, attn_norm, ffn, output_norm) in enumerate(
|
| 290 |
+
zip(
|
| 291 |
+
self.routing_attention,
|
| 292 |
+
self.routing_attention_norm,
|
| 293 |
+
self.routing_ffn,
|
| 294 |
+
self.routing_output_norm,
|
| 295 |
+
strict=True,
|
| 296 |
+
)
|
| 297 |
+
):
|
| 298 |
+
evidence = self._attend(
|
| 299 |
+
attention, routed[None], memory, None if projected is None else projected[layer_index + 1]
|
| 300 |
+
)[0]
|
| 301 |
+
updated = attn_norm(routed + evidence)
|
| 302 |
+
routed = output_norm(updated + ffn(updated))
|
| 303 |
+
query = self.query_norm(self.query_mlp(context))
|
| 304 |
+
keys = self.key_norm(self.key_mlp(routed))
|
| 305 |
+
with torch.autocast(device_type=state.device.type, enabled=False):
|
| 306 |
+
return (keys.float() @ query.float()) / math.sqrt(self.config.projection_size)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
class FADepthRouter(nn.Module):
|
| 310 |
+
def __init__(self, hidden_size, rms_norm_eps=1e-6):
|
| 311 |
+
super().__init__()
|
| 312 |
+
self.query = nn.Parameter(torch.zeros(hidden_size))
|
| 313 |
+
self.rms_norm_eps = rms_norm_eps
|
| 314 |
+
|
| 315 |
+
def forward(self, carry, sources):
|
| 316 |
+
"""Return m * sum(alpha_j * v_j), with m equal to the source count.
|
| 317 |
+
|
| 318 |
+
Centering the weights around 1/m uses the existing carry as the sum of
|
| 319 |
+
sources. This equivalent form avoids regrouping low-precision residual
|
| 320 |
+
additions at initialization: query=0 gives exactly r=carry. Neither
|
| 321 |
+
keys nor values are detached; all depth paths retain their gradients.
|
| 322 |
+
"""
|
| 323 |
+
with torch.autocast(device_type=carry.device.type, enabled=False):
|
| 324 |
+
values = torch.stack(sources).float()
|
| 325 |
+
keys = values * torch.rsqrt(values.square().mean(-1, keepdim=True) + self.rms_norm_eps)
|
| 326 |
+
logits = torch.einsum("d,nbtd->nbt", self.query.float(), keys)
|
| 327 |
+
alpha = logits.softmax(dim=0)
|
| 328 |
+
count = len(sources)
|
| 329 |
+
centered = (alpha - 1.0 / count) * count
|
| 330 |
+
correction = torch.einsum("nbt,nbtd->btd", centered, values)
|
| 331 |
+
return (carry.float() + correction).to(carry.dtype)
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _attention(module, hidden, positional, previous, *, bidirectional=False):
|
| 335 |
+
batch, length = hidden.shape[:2]
|
| 336 |
+
shape = (batch, length, -1, module.head_dim)
|
| 337 |
+
query, gate = module.q_proj(hidden).view(batch, length, -1, 2 * module.head_dim).chunk(2, dim=-1)
|
| 338 |
+
query = module.q_norm(query).transpose(1, 2)
|
| 339 |
+
key = module.k_norm(module.k_proj(hidden).view(shape)).transpose(1, 2)
|
| 340 |
+
value = module.v_proj(hidden).view(shape).transpose(1, 2)
|
| 341 |
+
query, key = apply_rotary_pos_emb(query, key, *positional)
|
| 342 |
+
prefix_length = previous[0].shape[2] if previous else 0
|
| 343 |
+
if previous:
|
| 344 |
+
key = torch.cat((previous[0], key), dim=2)
|
| 345 |
+
value = torch.cat((previous[1], value), dim=2)
|
| 346 |
+
# Keep compact GQA cache tensors while avoiding a quadratic math fallback
|
| 347 |
+
# in SDPA implementations that do not fuse grouped-query attention.
|
| 348 |
+
groups = query.shape[1] // key.shape[1]
|
| 349 |
+
attention_query = query.to(value.dtype)
|
| 350 |
+
attention_key = key.to(value.dtype)
|
| 351 |
+
attention_value = value
|
| 352 |
+
if groups > 1:
|
| 353 |
+
attention_key = attention_key.repeat_interleave(groups, dim=1)
|
| 354 |
+
attention_value = attention_value.repeat_interleave(groups, dim=1)
|
| 355 |
+
padding = (-module.head_dim) % 8
|
| 356 |
+
if padding:
|
| 357 |
+
attention_query, attention_key, attention_value = (
|
| 358 |
+
F.pad(tensor, (0, padding)) for tensor in (attention_query, attention_key, attention_value)
|
| 359 |
+
)
|
| 360 |
+
mask = None
|
| 361 |
+
if prefix_length and not bidirectional:
|
| 362 |
+
q_positions = torch.arange(length, device=hidden.device)[:, None]
|
| 363 |
+
k_positions = torch.arange(prefix_length + length, device=hidden.device)[None, :]
|
| 364 |
+
allowed = k_positions <= prefix_length + q_positions
|
| 365 |
+
width = math.ceil((prefix_length + length) / 8) * 8
|
| 366 |
+
mask = attention_query.new_zeros((length, width))[:, : prefix_length + length]
|
| 367 |
+
mask.masked_fill_(~allowed, float("-inf"))
|
| 368 |
+
with (
|
| 369 |
+
torch.autocast(device_type=hidden.device.type, enabled=False),
|
| 370 |
+
sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]),
|
| 371 |
+
):
|
| 372 |
+
attended = F.scaled_dot_product_attention(
|
| 373 |
+
attention_query,
|
| 374 |
+
attention_key,
|
| 375 |
+
attention_value,
|
| 376 |
+
attn_mask=mask,
|
| 377 |
+
is_causal=not prefix_length and not bidirectional,
|
| 378 |
+
dropout_p=0.0,
|
| 379 |
+
scale=module.scaling,
|
| 380 |
+
)[..., : module.head_dim]
|
| 381 |
+
attended = attended.transpose(1, 2).reshape(batch, length, -1)
|
| 382 |
+
output = module.o_proj(attended * gate.reshape(batch, length, -1).sigmoid())
|
| 383 |
+
return output, (key, value)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _linear_context(module, hidden, previous, config):
|
| 387 |
+
if previous is None:
|
| 388 |
+
cache = Qwen3_5DynamicCache(config)
|
| 389 |
+
cache.last_linear_layer = module.layer_idx
|
| 390 |
+
output = module(hidden, cache_params=cache)
|
| 391 |
+
return output, (cache.conv_states[module.layer_idx], cache.recurrent_states[module.layer_idx])
|
| 392 |
+
|
| 393 |
+
batch, length = hidden.shape[:2]
|
| 394 |
+
raw = module.in_proj_qkv(hidden).transpose(1, 2)
|
| 395 |
+
window = torch.cat((previous[0], raw), dim=-1)
|
| 396 |
+
# The cached window holds kernel_size raw values. Its oldest value precedes
|
| 397 |
+
# the first suffix output's receptive field; exclude that value from conv.
|
| 398 |
+
if module.causal_conv1d_fn is not None:
|
| 399 |
+
# Match the upstream fused convolution+SiLU rounding in BF16. Applying
|
| 400 |
+
# the same causal kernel to the cached window preserves continuation
|
| 401 |
+
# gradients and avoids an extra low-precision convolution round trip.
|
| 402 |
+
mixed = module.causal_conv1d_fn(
|
| 403 |
+
x=window,
|
| 404 |
+
weight=module.conv1d.weight.squeeze(1),
|
| 405 |
+
bias=module.conv1d.bias,
|
| 406 |
+
activation=module.activation,
|
| 407 |
+
)[:, :, -length:]
|
| 408 |
+
else:
|
| 409 |
+
mixed = F.silu(
|
| 410 |
+
F.conv1d(window[:, :, 1:], module.conv1d.weight, module.conv1d.bias, groups=raw.shape[1])
|
| 411 |
+
)
|
| 412 |
+
mixed = mixed.transpose(1, 2)
|
| 413 |
+
query, key, value = mixed.split((module.key_dim, module.key_dim, module.value_dim), dim=-1)
|
| 414 |
+
query = query.reshape(batch, length, -1, module.head_k_dim)
|
| 415 |
+
key = key.reshape(batch, length, -1, module.head_k_dim)
|
| 416 |
+
value = value.reshape(batch, length, -1, module.head_v_dim)
|
| 417 |
+
repeat = module.num_v_heads // module.num_k_heads
|
| 418 |
+
if repeat > 1:
|
| 419 |
+
query = query.repeat_interleave(repeat, dim=2)
|
| 420 |
+
key = key.repeat_interleave(repeat, dim=2)
|
| 421 |
+
beta = module.in_proj_b(hidden).sigmoid()
|
| 422 |
+
decay = -module.A_log.float().exp() * F.softplus(module.in_proj_a(hidden).float() + module.dt_bias)
|
| 423 |
+
mixed, recurrent = module.chunk_gated_delta_rule(
|
| 424 |
+
query,
|
| 425 |
+
key,
|
| 426 |
+
value,
|
| 427 |
+
g=decay,
|
| 428 |
+
beta=beta,
|
| 429 |
+
initial_state=previous[1],
|
| 430 |
+
output_final_state=True,
|
| 431 |
+
use_qk_l2norm_in_kernel=True,
|
| 432 |
+
)
|
| 433 |
+
gate = module.in_proj_z(hidden).reshape(-1, module.head_v_dim)
|
| 434 |
+
mixed = module.norm(mixed.reshape(-1, module.head_v_dim), gate).reshape(batch, length, -1)
|
| 435 |
+
return module.out_proj(mixed), (window[:, :, -module.conv_kernel_size :].clone(), recurrent)
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def _continue_gdn(module, hidden, previous, config, *, terminal=False, scan=None):
|
| 439 |
+
"""Continue on the native 64-token chunk grid, preserving terminal scans."""
|
| 440 |
+
scan = scan or _linear_context
|
| 441 |
+
chunk_size = 64
|
| 442 |
+
|
| 443 |
+
if torch.is_grad_enabled() or module.training:
|
| 444 |
+
raise ValueError("Chunk-aware prefix state is inference-only")
|
| 445 |
+
if previous is not None and not isinstance(previous, GDNState):
|
| 446 |
+
raise ValueError("GS1 continuation requires chunk-aware native prefix state")
|
| 447 |
+
if terminal:
|
| 448 |
+
# The trained GS1 reference scans context first, then the terminal token
|
| 449 |
+
# from that final state. Do not merge the terminal back into the chunk.
|
| 450 |
+
if previous is None or hidden.shape[1] != 1:
|
| 451 |
+
raise ValueError("A terminal segment requires a prefix and exactly one token")
|
| 452 |
+
return scan(module, hidden, (previous.conv, previous.recurrent), config)
|
| 453 |
+
boundary = (
|
| 454 |
+
(previous.boundary_conv, previous.boundary_recurrent)
|
| 455 |
+
if previous is not None and previous.boundary_conv is not None
|
| 456 |
+
else None
|
| 457 |
+
)
|
| 458 |
+
tail = previous.tail if previous is not None else hidden[:, :0]
|
| 459 |
+
combined = torch.cat((tail, hidden), dim=1) if tail.shape[1] else hidden
|
| 460 |
+
result = scan(module, combined, boundary, config)
|
| 461 |
+
output, final = result[:2]
|
| 462 |
+
exported = result[2] if len(result) == 3 else None
|
| 463 |
+
completed = combined.shape[1] // chunk_size * chunk_size
|
| 464 |
+
if completed:
|
| 465 |
+
if completed == combined.shape[1]:
|
| 466 |
+
boundary = final
|
| 467 |
+
elif exported is not None:
|
| 468 |
+
boundary = exported
|
| 469 |
+
else:
|
| 470 |
+
boundary = scan(module, combined[:, :completed], boundary, config)[1]
|
| 471 |
+
state = GDNState(
|
| 472 |
+
*final,
|
| 473 |
+
*(boundary if boundary is not None else (None, None)),
|
| 474 |
+
combined[:, completed:].clone(),
|
| 475 |
+
(previous.position if previous is not None else 0) + hidden.shape[1],
|
| 476 |
+
)
|
| 477 |
+
return output[:, tail.shape[1] :], state
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
class GS1PretrainedModel(Qwen3_5PreTrainedModel):
|
| 481 |
+
"""Shared initialization and strict native serialization for GS1 modules."""
|
| 482 |
+
|
| 483 |
+
config_class = GS1Config
|
| 484 |
+
base_model_prefix = ""
|
| 485 |
+
_no_split_modules = ["GS1DecoderLayer", "Qwen3_5VisionBlock"]
|
| 486 |
+
_tied_weights_keys = {}
|
| 487 |
+
|
| 488 |
+
def get_output_embeddings(self):
|
| 489 |
+
return None
|
| 490 |
+
|
| 491 |
+
def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
|
| 492 |
+
for module in self.modules():
|
| 493 |
+
if isinstance(module, GS1TextModel):
|
| 494 |
+
module.gradient_checkpointing = True
|
| 495 |
+
|
| 496 |
+
def gradient_checkpointing_disable(self):
|
| 497 |
+
for module in self.modules():
|
| 498 |
+
if isinstance(module, GS1TextModel):
|
| 499 |
+
module.gradient_checkpointing = False
|
| 500 |
+
|
| 501 |
+
def save_pretrained(self, save_directory, *, state_dict=None, **kwargs):
|
| 502 |
+
from .util import export_state
|
| 503 |
+
|
| 504 |
+
self.config.decision = self.decision_config.model_dump()
|
| 505 |
+
if hasattr(self, "calibrator"):
|
| 506 |
+
self.config.calibration = self.calibrator.config.model_dump()
|
| 507 |
+
if hasattr(self, "decision_head"):
|
| 508 |
+
self.config.head_compute_dtype = "bfloat16" if self.decision_head.bf16_compute else "float32"
|
| 509 |
+
kwargs.pop("safe_serialization", None)
|
| 510 |
+
return super().save_pretrained(
|
| 511 |
+
save_directory,
|
| 512 |
+
state_dict=export_state(self, state_dict),
|
| 513 |
+
safe_serialization=True,
|
| 514 |
+
**kwargs,
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
@classmethod
|
| 518 |
+
def from_pretrained(cls, source, *, device="cpu", dtype=None, **kwargs):
|
| 519 |
+
from .util import load_native
|
| 520 |
+
|
| 521 |
+
return load_native(source, device=device, dtype=dtype, model_class=cls, **kwargs)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
class GS1TextModel(GS1PretrainedModel):
|
| 525 |
+
"""Native hybrid text decoder with GS1 depth routing and segmented attention."""
|
| 526 |
+
|
| 527 |
+
def __init__(self, config):
|
| 528 |
+
super().__init__(config)
|
| 529 |
+
text = config.text_config
|
| 530 |
+
self.decision_config = DecisionConfig.model_validate(config.decision)
|
| 531 |
+
self.initialization_seed = config.initialization_seed
|
| 532 |
+
self.full_layers = tuple(i for i, kind in enumerate(text.layer_types) if kind == "full_attention")
|
| 533 |
+
if text.linear_num_value_heads % text.linear_num_key_heads:
|
| 534 |
+
raise ValueError("GDN value heads must be a multiple of key heads")
|
| 535 |
+
if text.num_attention_heads % text.num_key_value_heads:
|
| 536 |
+
raise ValueError("Attention heads must be a multiple of key/value heads")
|
| 537 |
+
for selected in (self.decision_config.attnres_layers, self.decision_config.bidirectional_layers):
|
| 538 |
+
if selected is not None and not set(selected) <= set(self.full_layers):
|
| 539 |
+
raise ValueError("Selected layers must be full-attention decoder indices")
|
| 540 |
+
self.embed_tokens = nn.Embedding(text.vocab_size, text.hidden_size, text.pad_token_id)
|
| 541 |
+
self.layers = nn.ModuleList(GS1DecoderLayer(text, i) for i in range(text.num_hidden_layers))
|
| 542 |
+
self.norm = Qwen3_5RMSNorm(text.hidden_size, eps=text.rms_norm_eps)
|
| 543 |
+
self.rotary_emb = Qwen3_5TextRotaryEmbedding(text)
|
| 544 |
+
self.gradient_checkpointing = False
|
| 545 |
+
self.post_init()
|
| 546 |
+
receivers = self.decision_config.attnres_layers or self.full_layers
|
| 547 |
+
self.attnres_routers = nn.ModuleDict(
|
| 548 |
+
{
|
| 549 |
+
str(i): FADepthRouter(text.hidden_size, self.decision_config.attnres_eps)
|
| 550 |
+
for i in receivers
|
| 551 |
+
if self.decision_config.attnres
|
| 552 |
+
}
|
| 553 |
+
)
|
| 554 |
+
|
| 555 |
+
def get_input_embeddings(self):
|
| 556 |
+
return self.embed_tokens
|
| 557 |
+
|
| 558 |
+
def set_input_embeddings(self, value):
|
| 559 |
+
self.embed_tokens = value
|
| 560 |
+
|
| 561 |
+
def is_bidirectional(self, index):
|
| 562 |
+
selected = self.decision_config.bidirectional_layers
|
| 563 |
+
return self.decision_config.bidirectional and (selected is None or index in selected)
|
| 564 |
+
|
| 565 |
+
def _depth_input(self, index, carry, bank, partial):
|
| 566 |
+
if str(index) not in self.attnres_routers:
|
| 567 |
+
return carry
|
| 568 |
+
sources = bank if partial is None else (*bank, partial)
|
| 569 |
+
return self.attnres_routers[str(index)](carry, sources)
|
| 570 |
+
|
| 571 |
+
@staticmethod
|
| 572 |
+
def _advance_depth(bank, partial, mixed, ffn, full):
|
| 573 |
+
increment = mixed + ffn
|
| 574 |
+
partial = increment if partial is None else partial + increment
|
| 575 |
+
return ((*bank, partial), None) if full else (bank, partial)
|
| 576 |
+
|
| 577 |
+
def prepare_encode(self, hidden, position_ids, roles=None, *, terminal_query=False, attention_mask=None):
|
| 578 |
+
"""Validate immutable batch geometry before tensor execution or CUDA capture."""
|
| 579 |
+
batch, length = hidden.shape[:2]
|
| 580 |
+
if not batch or not length:
|
| 581 |
+
raise ValueError("GS1 expects nonempty branch rows")
|
| 582 |
+
valid = (
|
| 583 |
+
torch.ones((batch, length), dtype=torch.bool, device=hidden.device)
|
| 584 |
+
if attention_mask is None
|
| 585 |
+
else attention_mask.bool()
|
| 586 |
+
)
|
| 587 |
+
if valid.shape != (batch, length) or not valid[:, -1].all() or (valid[:, :-1] & ~valid[:, 1:]).any():
|
| 588 |
+
raise ValueError("GS1 batches require nonempty left-padded rows")
|
| 589 |
+
if terminal_query and (valid.sum(-1) < 2).any():
|
| 590 |
+
raise ValueError("A decision requires context before its terminal token")
|
| 591 |
+
if roles is None:
|
| 592 |
+
roles = torch.zeros((batch, length), dtype=torch.long, device=hidden.device)
|
| 593 |
+
elif roles.shape == (length,):
|
| 594 |
+
roles = roles[None].expand(batch, -1)
|
| 595 |
+
if roles.shape != (batch, length):
|
| 596 |
+
raise ValueError("Token roles must match the branch rows")
|
| 597 |
+
keep = valid.unsqueeze(-1)
|
| 598 |
+
valid_lengths = tuple(valid.sum(-1).tolist())
|
| 599 |
+
hidden = hidden * keep
|
| 600 |
+
masks = {}
|
| 601 |
+
eye = torch.eye(length, dtype=torch.bool, device=hidden.device)[None]
|
| 602 |
+
for bidirectional in {self.is_bidirectional(i) for i in self.full_layers}:
|
| 603 |
+
if bidirectional:
|
| 604 |
+
allowed = (roles[:, :, None] >= roles[:, None, :]) & (roles[:, None, :] != 3)
|
| 605 |
+
allowed |= eye
|
| 606 |
+
else:
|
| 607 |
+
allowed = torch.ones((batch, length, length), dtype=torch.bool, device=hidden.device).tril_()
|
| 608 |
+
allowed &= valid[:, None, :]
|
| 609 |
+
allowed |= ~valid[:, :, None] & eye
|
| 610 |
+
masks[bidirectional] = hidden.new_zeros((batch, 1, length, length)).masked_fill_(
|
| 611 |
+
~allowed[:, None], torch.finfo(hidden.dtype).min
|
| 612 |
+
)
|
| 613 |
+
positional = self.rotary_emb(hidden, position_ids)
|
| 614 |
+
cohorts = tuple(
|
| 615 |
+
(
|
| 616 |
+
length,
|
| 617 |
+
torch.tensor(
|
| 618 |
+
[i for i, value in enumerate(valid_lengths) if value == length], device=hidden.device
|
| 619 |
+
),
|
| 620 |
+
)
|
| 621 |
+
for length in sorted(set(valid_lengths))
|
| 622 |
+
)
|
| 623 |
+
return keep, valid_lengths, masks, positional, cohorts
|
| 624 |
+
|
| 625 |
+
def encode(self, hidden, position_ids, roles=None, *, terminal_query=False, attention_mask=None):
|
| 626 |
+
layout = self.prepare_encode(
|
| 627 |
+
hidden, position_ids, roles, terminal_query=terminal_query, attention_mask=attention_mask
|
| 628 |
+
)
|
| 629 |
+
return self.encode_prepared(hidden, layout, terminal_query=terminal_query)
|
| 630 |
+
|
| 631 |
+
def encode_prepared(self, hidden, layout, *, terminal_query=False, execution_policy=None):
|
| 632 |
+
"""The unchanged decoder equations over already validated, static geometry."""
|
| 633 |
+
if execution_policy is not None:
|
| 634 |
+
return execution_policy.encode(self, hidden, layout, terminal_query=terminal_query)
|
| 635 |
+
keep, valid_lengths, masks, positional, cohorts = layout
|
| 636 |
+
hidden = hidden * keep
|
| 637 |
+
bank, partial = (hidden,), None
|
| 638 |
+
|
| 639 |
+
def run(carry, layer, index, bank, partial):
|
| 640 |
+
read = self._depth_input(index, carry, bank, partial)
|
| 641 |
+
mask = masks.get(self.is_bidirectional(index))
|
| 642 |
+
output, mixed, ffn = layer(carry, positional, mask, read, terminal_query, valid_lengths, cohorts)
|
| 643 |
+
# Prefix PADs start from zero GDN state and remain zero after every
|
| 644 |
+
# layer. Valid tokens retain the ordinary convolution/time scan.
|
| 645 |
+
output, mixed, ffn = output * keep, mixed * keep, ffn * keep
|
| 646 |
+
if self.attnres_routers:
|
| 647 |
+
bank, partial = self._advance_depth(
|
| 648 |
+
bank, partial, mixed, ffn, layer.layer_type == "full_attention"
|
| 649 |
+
)
|
| 650 |
+
return output, bank, partial
|
| 651 |
+
|
| 652 |
+
for index, layer in enumerate(self.layers):
|
| 653 |
+
if self.training and self.gradient_checkpointing:
|
| 654 |
+
hidden, bank, partial = checkpoint(
|
| 655 |
+
run, hidden, layer, index, bank, partial, use_reentrant=False
|
| 656 |
+
)
|
| 657 |
+
else:
|
| 658 |
+
hidden, bank, partial = run(hidden, layer, index, bank, partial)
|
| 659 |
+
return self.norm(hidden) * keep
|
| 660 |
+
|
| 661 |
+
def segment_forward(
|
| 662 |
+
self, input_ids, start_position=0, prefix=None, *, terminal_query=False, execution_policy=None
|
| 663 |
+
):
|
| 664 |
+
"""Inference-only segment continuation, including GDN convolution/state caches."""
|
| 665 |
+
if torch.is_grad_enabled() or self.training:
|
| 666 |
+
raise ValueError("Shared segment caches require eval mode without gradients")
|
| 667 |
+
if execution_policy is not None:
|
| 668 |
+
return execution_policy.segment(
|
| 669 |
+
self, input_ids, start_position, prefix, terminal_query=terminal_query
|
| 670 |
+
)
|
| 671 |
+
hidden = self.embed_tokens(input_ids)
|
| 672 |
+
positions = torch.arange(start_position, start_position + hidden.shape[1], device=hidden.device)[
|
| 673 |
+
None
|
| 674 |
+
].expand(hidden.shape[0], -1)
|
| 675 |
+
positional = self.rotary_emb(hidden, positions)
|
| 676 |
+
bank, partial, states = (hidden,), None, []
|
| 677 |
+
for index, layer in enumerate(self.layers):
|
| 678 |
+
read = self._depth_input(index, hidden, bank, partial)
|
| 679 |
+
normalized = layer.input_layernorm(read)
|
| 680 |
+
previous = prefix[index] if prefix is not None else None
|
| 681 |
+
if layer.layer_type == "full_attention":
|
| 682 |
+
mixed, state = _attention(
|
| 683 |
+
layer.self_attn,
|
| 684 |
+
normalized,
|
| 685 |
+
positional,
|
| 686 |
+
previous,
|
| 687 |
+
bidirectional=self.is_bidirectional(index),
|
| 688 |
+
)
|
| 689 |
+
else:
|
| 690 |
+
mixed, state = _continue_gdn(
|
| 691 |
+
layer.linear_attn, normalized, previous, self.config.text_config, terminal=terminal_query
|
| 692 |
+
)
|
| 693 |
+
hidden = hidden + mixed
|
| 694 |
+
ffn = layer.mlp(layer.post_attention_layernorm(hidden))
|
| 695 |
+
hidden = hidden + ffn
|
| 696 |
+
if self.attnres_routers:
|
| 697 |
+
bank, partial = self._advance_depth(
|
| 698 |
+
bank, partial, mixed, ffn, layer.layer_type == "full_attention"
|
| 699 |
+
)
|
| 700 |
+
states.append(state)
|
| 701 |
+
return self.norm(hidden), tuple(states)
|
| 702 |
+
|
| 703 |
+
def forward(
|
| 704 |
+
self,
|
| 705 |
+
input_ids=None,
|
| 706 |
+
*,
|
| 707 |
+
inputs_embeds=None,
|
| 708 |
+
position_ids=None,
|
| 709 |
+
roles=None,
|
| 710 |
+
terminal_query=False,
|
| 711 |
+
attention_mask=None,
|
| 712 |
+
prepared_layout=None,
|
| 713 |
+
execution_policy=None,
|
| 714 |
+
):
|
| 715 |
+
if (input_ids is None) == (inputs_embeds is None):
|
| 716 |
+
raise ValueError("Specify exactly one of input_ids or inputs_embeds")
|
| 717 |
+
hidden = self.embed_tokens(input_ids) if inputs_embeds is None else inputs_embeds
|
| 718 |
+
if position_ids is None:
|
| 719 |
+
position_ids = torch.arange(hidden.shape[1], device=hidden.device)[None].expand(
|
| 720 |
+
hidden.shape[0], -1
|
| 721 |
+
)
|
| 722 |
+
output = (
|
| 723 |
+
self.encode(
|
| 724 |
+
hidden, position_ids, roles, terminal_query=terminal_query, attention_mask=attention_mask
|
| 725 |
+
)
|
| 726 |
+
if prepared_layout is None
|
| 727 |
+
else self.encode_prepared(
|
| 728 |
+
hidden, prepared_layout, terminal_query=terminal_query, execution_policy=execution_policy
|
| 729 |
+
)
|
| 730 |
+
)
|
| 731 |
+
return GS1TextOutput(last_hidden_state=output)
|
| 732 |
+
|
| 733 |
+
|
| 734 |
+
class GS1Model(GS1PretrainedModel):
|
| 735 |
+
"""Complete GS1: native text, upstream vision and one decision head."""
|
| 736 |
+
|
| 737 |
+
def __init__(self, config):
|
| 738 |
+
super().__init__(config)
|
| 739 |
+
self.initialization_seed = config.initialization_seed
|
| 740 |
+
self.language_model = GS1TextModel(config)
|
| 741 |
+
self.decision_config = self.language_model.decision_config
|
| 742 |
+
self.visual = Qwen3_5VisionModel._from_config(config.vision_config)
|
| 743 |
+
self.post_init()
|
| 744 |
+
self.decision_head = CrossAttentionDecisionHead(
|
| 745 |
+
config.text_config.hidden_size, self.decision_config, self.initialization_seed
|
| 746 |
+
)
|
| 747 |
+
self.decision_head.bf16_compute = config.head_compute_dtype == "bfloat16"
|
| 748 |
+
from .schema import Calibrator
|
| 749 |
+
|
| 750 |
+
self.calibrator = Calibrator(config.calibration)
|
| 751 |
+
|
| 752 |
+
def get_input_embeddings(self):
|
| 753 |
+
return self.language_model.get_input_embeddings()
|
| 754 |
+
|
| 755 |
+
def set_input_embeddings(self, value):
|
| 756 |
+
self.language_model.set_input_embeddings(value)
|
| 757 |
+
|
| 758 |
+
def _vision_positions(self, input_ids, types, image_grid, video_grid=None):
|
| 759 |
+
positions = torch.zeros(3, *input_ids.shape, device=input_ids.device, dtype=torch.long)
|
| 760 |
+
grids = {
|
| 761 |
+
1: iter(image_grid) if image_grid is not None else None,
|
| 762 |
+
2: iter(video_grid) if video_grid is not None else None,
|
| 763 |
+
}
|
| 764 |
+
merge = self.config.vision_config.spatial_merge_size
|
| 765 |
+
for row in range(input_ids.shape[0]):
|
| 766 |
+
begin, offset = 0, 0
|
| 767 |
+
while begin < input_ids.shape[1]:
|
| 768 |
+
kind = int(types[row, begin])
|
| 769 |
+
end = begin + 1
|
| 770 |
+
while end < input_ids.shape[1] and int(types[row, end]) == kind:
|
| 771 |
+
end += 1
|
| 772 |
+
if kind == 0:
|
| 773 |
+
piece = torch.arange(end - begin, device=input_ids.device)[None].expand(3, -1) + offset
|
| 774 |
+
offset += end - begin
|
| 775 |
+
else:
|
| 776 |
+
t, h, w = map(int, next(grids[kind]))
|
| 777 |
+
h, w = h // merge, w // merge
|
| 778 |
+
piece = torch.stack(
|
| 779 |
+
(
|
| 780 |
+
torch.full((t * h * w,), offset, device=input_ids.device),
|
| 781 |
+
torch.arange(offset, offset + h, device=input_ids.device).repeat_interleave(
|
| 782 |
+
w * t
|
| 783 |
+
),
|
| 784 |
+
torch.arange(offset, offset + w, device=input_ids.device).repeat(h * t),
|
| 785 |
+
)
|
| 786 |
+
)
|
| 787 |
+
offset += max(h, w)
|
| 788 |
+
if piece.shape[1] != end - begin:
|
| 789 |
+
raise ValueError("Vision grid and placeholder token count differ")
|
| 790 |
+
positions[:, row, begin:end] = piece
|
| 791 |
+
begin = end
|
| 792 |
+
return positions
|
| 793 |
+
|
| 794 |
+
def forward(
|
| 795 |
+
self,
|
| 796 |
+
input_ids,
|
| 797 |
+
*,
|
| 798 |
+
branch=None,
|
| 799 |
+
state_length=None,
|
| 800 |
+
branches=None,
|
| 801 |
+
state_lengths=None,
|
| 802 |
+
position_ids=None,
|
| 803 |
+
pixel_values=None,
|
| 804 |
+
image_grid_thw=None,
|
| 805 |
+
mm_token_type_ids=None,
|
| 806 |
+
attention_mask=None,
|
| 807 |
+
readout_callback=None,
|
| 808 |
+
**kwargs,
|
| 809 |
+
):
|
| 810 |
+
hidden = self.get_input_embeddings()(input_ids)
|
| 811 |
+
if pixel_values is not None:
|
| 812 |
+
features = self.visual(
|
| 813 |
+
pixel_values.to(self.visual.dtype), grid_thw=image_grid_thw, return_dict=True
|
| 814 |
+
).pooler_output.to(hidden.device, hidden.dtype)
|
| 815 |
+
mask = (input_ids == self.config.image_token_id).unsqueeze(-1).expand_as(hidden)
|
| 816 |
+
if hidden[mask].numel() != features.numel():
|
| 817 |
+
raise ValueError("Image features and placeholder tokens differ")
|
| 818 |
+
hidden = hidden.masked_scatter(mask, features)
|
| 819 |
+
if position_ids is None:
|
| 820 |
+
if mm_token_type_ids is None:
|
| 821 |
+
raise ValueError("Visual branches require modality token types")
|
| 822 |
+
position_ids = self._vision_positions(input_ids, mm_token_type_ids, image_grid_thw)
|
| 823 |
+
if position_ids is None:
|
| 824 |
+
position_ids = torch.arange(hidden.shape[1], device=hidden.device)[None].expand(
|
| 825 |
+
hidden.shape[0], -1
|
| 826 |
+
)
|
| 827 |
+
if branch is not None and branches is not None:
|
| 828 |
+
raise ValueError("Specify one branch or batched branch metadata")
|
| 829 |
+
rows = branches if branches is not None else [branch] if branch is not None else None
|
| 830 |
+
lengths = state_lengths if branches is not None else [state_length] if branch is not None else None
|
| 831 |
+
roles = None
|
| 832 |
+
starts = []
|
| 833 |
+
if rows is not None:
|
| 834 |
+
if len(rows) != input_ids.shape[0] or lengths is None or len(lengths) != len(rows):
|
| 835 |
+
raise ValueError("Compiled branch metadata differs from batch rows")
|
| 836 |
+
roles = torch.full_like(input_ids, -1)
|
| 837 |
+
for row, (item, offset) in enumerate(zip(rows, lengths, strict=True)):
|
| 838 |
+
total = offset + len(item.suffix)
|
| 839 |
+
begin = input_ids.shape[1] - total
|
| 840 |
+
if begin < 0 or (attention_mask is not None and int(attention_mask[row].sum()) != total):
|
| 841 |
+
raise ValueError("Compiled branch length differs from model input")
|
| 842 |
+
if branch is not None and begin:
|
| 843 |
+
raise ValueError("Single compiled branches must be unpadded")
|
| 844 |
+
starts.append(begin)
|
| 845 |
+
roles[row, begin : begin + offset] = 0
|
| 846 |
+
roles[row, begin + offset :] = torch.tensor(
|
| 847 |
+
[
|
| 848 |
+
1 if r == "question_context" else 2 if r == "candidate_context" else 3
|
| 849 |
+
for r in item.token_roles
|
| 850 |
+
],
|
| 851 |
+
device=hidden.device,
|
| 852 |
+
)
|
| 853 |
+
hidden = self.language_model(
|
| 854 |
+
inputs_embeds=hidden,
|
| 855 |
+
position_ids=position_ids,
|
| 856 |
+
roles=roles,
|
| 857 |
+
terminal_query=rows is not None,
|
| 858 |
+
attention_mask=attention_mask,
|
| 859 |
+
).last_hidden_state
|
| 860 |
+
logits = None
|
| 861 |
+
if rows is not None:
|
| 862 |
+
values = [
|
| 863 |
+
(readout_callback or self.readout)(hidden[row, begin:], item, offset)
|
| 864 |
+
for row, (begin, item, offset) in enumerate(zip(starts, rows, lengths, strict=True))
|
| 865 |
+
]
|
| 866 |
+
width = max(value.numel() for value in values)
|
| 867 |
+
logits = (
|
| 868 |
+
values[0]
|
| 869 |
+
if branch is not None
|
| 870 |
+
else torch.stack(
|
| 871 |
+
[F.pad(value, (0, width - value.numel()), value=-torch.inf) for value in values]
|
| 872 |
+
)
|
| 873 |
+
)
|
| 874 |
+
return GS1Output(logits=logits, last_hidden_state=hidden)
|
| 875 |
+
|
| 876 |
+
def readout(self, hidden, branch, offset, *, prepared_memory=None, state_hidden=None):
|
| 877 |
+
def pool(start, span, end):
|
| 878 |
+
content = hidden[offset + span[0] : offset + span[1]].mean(0)
|
| 879 |
+
closing = hidden[offset + end]
|
| 880 |
+
return (
|
| 881 |
+
(hidden[offset + start] + content + closing) / 3
|
| 882 |
+
if self.decision_config.bidirectional
|
| 883 |
+
else (content + closing) / 2
|
| 884 |
+
)
|
| 885 |
+
|
| 886 |
+
question = pool(branch.question_start_position, branch.question_span, branch.question_end_position)
|
| 887 |
+
options = torch.stack(
|
| 888 |
+
[
|
| 889 |
+
pool(start, span, end)
|
| 890 |
+
for start, span, end in zip(
|
| 891 |
+
branch.option_starts, branch.option_spans, branch.option_ends, strict=True
|
| 892 |
+
)
|
| 893 |
+
]
|
| 894 |
+
)
|
| 895 |
+
return self.decision_head(
|
| 896 |
+
hidden[branch.state_evidence_start : offset] if state_hidden is None else state_hidden,
|
| 897 |
+
question,
|
| 898 |
+
hidden[offset + branch.decision_position],
|
| 899 |
+
options,
|
| 900 |
+
prepared_memory=prepared_memory,
|
| 901 |
+
)
|
| 902 |
+
|
| 903 |
+
def probabilities(self, logits, candidate_mask=None, *, calibration_active=True):
|
| 904 |
+
from .schema import probabilities
|
| 905 |
+
|
| 906 |
+
return probabilities(logits, candidate_mask, self.calibrator if calibration_active else None)
|
| 907 |
+
|
| 908 |
+
def decision_compiler(self, tokenizer=None, **overrides):
|
| 909 |
+
"""Build the native prompt compiler from standard pretrained assets."""
|
| 910 |
+
from .compiler import Compiler
|
| 911 |
+
|
| 912 |
+
if tokenizer is None:
|
| 913 |
+
from transformers import AutoTokenizer
|
| 914 |
+
|
| 915 |
+
if not hasattr(self, "_decision_tokenizer"):
|
| 916 |
+
self._decision_tokenizer = AutoTokenizer.from_pretrained(
|
| 917 |
+
self._processor_source, local_files_only=True
|
| 918 |
+
)
|
| 919 |
+
tokenizer = self._decision_tokenizer
|
| 920 |
+
source = getattr(self, "_processor_source", None)
|
| 921 |
+
limits = {
|
| 922 |
+
key: self.config.inference_limits[key]
|
| 923 |
+
for key in ("max_context", "max_questions", "max_request_tokens", "max_branches")
|
| 924 |
+
}
|
| 925 |
+
for key, value in overrides.items():
|
| 926 |
+
limits[key] = min(limits[key], value)
|
| 927 |
+
compiler = Compiler(tokenizer, decision=self.decision_config, processor=source, **limits)
|
| 928 |
+
if compiler.controls != getattr(self.config, "structural_token_ids", {}):
|
| 929 |
+
raise ValueError("Structural token identity mismatch")
|
| 930 |
+
return compiler
|
| 931 |
+
|
| 932 |
+
@torch.inference_mode()
|
| 933 |
+
def decide(self, state, questions, *, tokenizer=None, precision="bf16"):
|
| 934 |
+
"""Return typed decisions using the standalone eager reference path.
|
| 935 |
+
|
| 936 |
+
This method needs only the files in this model's Hub repository. The
|
| 937 |
+
separately installed gs1 runtime provides qualified CUDA graph execution.
|
| 938 |
+
"""
|
| 939 |
+
from .compiler import branch_logits
|
| 940 |
+
from .schema import DecisionRequest, DecisionResponse, Usage, make_answer
|
| 941 |
+
|
| 942 |
+
if precision not in {"fp32", "bf16"}:
|
| 943 |
+
raise ValueError("precision must be fp32 or bf16")
|
| 944 |
+
if self.training:
|
| 945 |
+
raise ValueError("Call model.eval() before making decisions")
|
| 946 |
+
compiler = self.decision_compiler(tokenizer)
|
| 947 |
+
tokenizer = compiler.tokenizer
|
| 948 |
+
name = getattr(self.config, "_name_or_path", "") or "GS1"
|
| 949 |
+
request = DecisionRequest(state=state, questions=questions, model=name)
|
| 950 |
+
compiled = compiler.compile(request)
|
| 951 |
+
device = self.get_input_embeddings().weight.device
|
| 952 |
+
answers = {}
|
| 953 |
+
with torch.autocast(device.type, dtype=torch.bfloat16, enabled=precision == "bf16"):
|
| 954 |
+
for branch in compiled.branches:
|
| 955 |
+
logits = branch_logits(
|
| 956 |
+
self, [(compiled.state, branch)], pad_token_id=tokenizer.pad_token_id or 0
|
| 957 |
+
)[0]
|
| 958 |
+
values = self.probabilities(logits).p_final.tolist()
|
| 959 |
+
answers[branch.question_id] = make_answer(request.questions[branch.question_id], values)
|
| 960 |
+
response = DecisionResponse(
|
| 961 |
+
model=name, answers=answers, usage=Usage(input_tokens=compiled.input_tokens, output_tokens=0)
|
| 962 |
+
)
|
| 963 |
+
return response.model_dump(mode="json")
|
| 964 |
+
|
| 965 |
+
|
| 966 |
+
# Native discovery resolves the complete text/vision/decision model.
|
| 967 |
+
AutoConfig.register("gs1", GS1Config, exist_ok=True)
|
| 968 |
+
AutoModel.register(GS1Config, GS1Model, exist_ok=True)
|
| 969 |
+
|
| 970 |
+
GS1Config.register_for_auto_class("AutoConfig")
|
| 971 |
+
GS1Model.register_for_auto_class("AutoModel")
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 16777216,
|
| 4 |
+
"shortest_edge": 65536
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"image_processor_type": "Qwen2VLImageProcessorFast"
|
| 21 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Standalone default inference; no gs1 repository installation is required.
|
| 2 |
+
torch==2.9.1
|
| 3 |
+
torchvision==0.24.1
|
| 4 |
+
transformers==5.3.0
|
| 5 |
+
safetensors>=0.5,<1
|
| 6 |
+
accelerate>=1.6,<2
|
| 7 |
+
huggingface-hub>=0.30,<2
|
| 8 |
+
pydantic>=2.8,<3
|
| 9 |
+
typesafe-sdk==0.7.0
|
| 10 |
+
pillow>=11,<13
|
schema.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Typed decisions, FP32 probabilities, calibration and confidence."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from typing import Annotated, Any, Literal
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
| 10 |
+
from torch import nn
|
| 11 |
+
from typesafe_sdk import Choice as Choice
|
| 12 |
+
from typesafe_sdk import Noul as Noul
|
| 13 |
+
from typesafe_sdk import Question as Question
|
| 14 |
+
from typesafe_sdk import Score as Score
|
| 15 |
+
|
| 16 |
+
from .config import CalibrationConfig
|
| 17 |
+
|
| 18 |
+
Description = str | dict[str, JsonValue] | list[JsonValue]
|
| 19 |
+
CONFIDENCE_VERSION = "typesafe_v1"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class StrictModel(BaseModel):
|
| 23 |
+
model_config = ConfigDict(extra="forbid", strict=True)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class DecisionRequest(StrictModel):
|
| 27 |
+
state: Description
|
| 28 |
+
model: str = Field(min_length=1)
|
| 29 |
+
questions: dict[str, Question] = Field(min_length=1)
|
| 30 |
+
|
| 31 |
+
@field_validator("questions", mode="before")
|
| 32 |
+
@classmethod
|
| 33 |
+
def sdk_primitives(cls, value):
|
| 34 |
+
if not isinstance(value, dict):
|
| 35 |
+
raise ValueError("questions must be an object")
|
| 36 |
+
kinds = {"choice": Choice, "noul": Noul, "score": Score}
|
| 37 |
+
parsed = {}
|
| 38 |
+
for qid, question in value.items():
|
| 39 |
+
if isinstance(question, dict):
|
| 40 |
+
kind = question.get("type")
|
| 41 |
+
if kind not in kinds:
|
| 42 |
+
raise ValueError("Question type must be choice, noul, or score")
|
| 43 |
+
question = kinds[kind].model_validate(question, strict=True)
|
| 44 |
+
parsed[qid] = question
|
| 45 |
+
return parsed
|
| 46 |
+
|
| 47 |
+
@model_validator(mode="after")
|
| 48 |
+
def server_contract(self):
|
| 49 |
+
# Keep SDK-native primitives, but enforce the narrower public serving contract.
|
| 50 |
+
# SDK 0.7.0 permits omitted instructions and singleton Score questions.
|
| 51 |
+
for question in self.questions.values():
|
| 52 |
+
if not isinstance(question.instructions, (str, dict, list)):
|
| 53 |
+
raise ValueError("Every question requires text, object, or array instructions")
|
| 54 |
+
if question.type == "choice":
|
| 55 |
+
if not 1 <= len(question.criteria) <= 255:
|
| 56 |
+
raise ValueError("Choice requires between 1 and 255 candidates")
|
| 57 |
+
descriptions = [v for v in question.criteria.values() if v is not None]
|
| 58 |
+
elif question.type == "score":
|
| 59 |
+
if not 2 <= len(question.criteria) <= 10:
|
| 60 |
+
raise ValueError("Score requires between 2 and 10 levels")
|
| 61 |
+
descriptions = question.criteria
|
| 62 |
+
else:
|
| 63 |
+
descriptions = list(noul_descriptions(question))
|
| 64 |
+
if any(not isinstance(v, (str, dict, list)) for v in descriptions):
|
| 65 |
+
raise ValueError("Criteria descriptions must be text, objects, or arrays")
|
| 66 |
+
# Reject NaN/Infinity at every depth, including structured descriptions.
|
| 67 |
+
json.dumps(self.model_dump(mode="json"), allow_nan=False)
|
| 68 |
+
return self
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def noul_descriptions(question: Question) -> tuple[Description, Description]:
|
| 72 |
+
criteria = question.criteria or {}
|
| 73 |
+
return (
|
| 74 |
+
criteria["false"]
|
| 75 |
+
if criteria.get("false") is not None
|
| 76 |
+
else "The proposition is false under the question's stated semantics.",
|
| 77 |
+
criteria["true"]
|
| 78 |
+
if criteria.get("true") is not None
|
| 79 |
+
else "The proposition is true under the question's stated semantics.",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class Usage(StrictModel):
|
| 84 |
+
input_tokens: int = Field(ge=0)
|
| 85 |
+
output_tokens: int = Field(ge=0)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class ChoiceAnswer(StrictModel):
|
| 89 |
+
type: Literal["choice"] = "choice"
|
| 90 |
+
choice: str
|
| 91 |
+
probabilities: dict[str, float]
|
| 92 |
+
confidence: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class NoulAnswer(StrictModel):
|
| 96 |
+
type: Literal["noul"] = "noul"
|
| 97 |
+
noul: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class ScoreAnswer(StrictModel):
|
| 101 |
+
type: Literal["score"] = "score"
|
| 102 |
+
score: float = Field(ge=0, allow_inf_nan=False)
|
| 103 |
+
legend: dict[str, Description]
|
| 104 |
+
probabilities: dict[str, float]
|
| 105 |
+
confidence: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
Answer = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")]
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
class DecisionResponse(StrictModel):
|
| 112 |
+
model: str
|
| 113 |
+
answers: dict[str, Answer]
|
| 114 |
+
usage: Usage
|
| 115 |
+
|
| 116 |
+
@model_validator(mode="after")
|
| 117 |
+
def numerical_validity(self):
|
| 118 |
+
for answer in self.answers.values():
|
| 119 |
+
if isinstance(answer, NoulAnswer):
|
| 120 |
+
continue
|
| 121 |
+
p = answer.probabilities
|
| 122 |
+
if not p or any(not math.isfinite(v) or not 0 <= v <= 1 for v in p.values()):
|
| 123 |
+
raise ValueError("Probabilities must be finite and within [0, 1]")
|
| 124 |
+
if abs(sum(p.values()) - 1) > 1e-6:
|
| 125 |
+
raise ValueError("Probabilities must sum to one")
|
| 126 |
+
if isinstance(answer, ChoiceAnswer):
|
| 127 |
+
if answer.choice not in p or p[answer.choice] != max(p.values()):
|
| 128 |
+
raise ValueError("Choice must be an argmax")
|
| 129 |
+
else:
|
| 130 |
+
if set(p) != set(answer.legend) or set(p) != {str(k) for k in range(len(p))}:
|
| 131 |
+
raise ValueError("Score indices and legend must align")
|
| 132 |
+
if abs(answer.score - sum(int(k) * v for k, v in p.items())) > 1e-6:
|
| 133 |
+
raise ValueError("Score must equal its probability-weighted level")
|
| 134 |
+
return self
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def candidate_ids(question: Question) -> list[str]:
|
| 138 |
+
if question.type == "choice":
|
| 139 |
+
return list(question.criteria)
|
| 140 |
+
if question.type == "noul":
|
| 141 |
+
return ["false", "true"]
|
| 142 |
+
return [str(k) for k in range(len(question.criteria))]
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def validate_response(request: DecisionRequest, response: DecisionResponse):
|
| 146 |
+
if response.answers.keys() != request.questions.keys():
|
| 147 |
+
raise ValueError("Response question identities do not match the request")
|
| 148 |
+
for qid, question in request.questions.items():
|
| 149 |
+
answer = response.answers[qid]
|
| 150 |
+
if answer.type != question.type:
|
| 151 |
+
raise ValueError(f"Primitive mismatch for {qid}")
|
| 152 |
+
if answer.type != "noul" and set(answer.probabilities) != set(candidate_ids(question)):
|
| 153 |
+
raise ValueError(f"Candidate mismatch for {qid}")
|
| 154 |
+
if answer.type == "score" and answer.legend != dict(zip(candidate_ids(question), question.criteria)):
|
| 155 |
+
raise ValueError(f"Score legend mismatch for {qid}")
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _bounded_confidence(value: float) -> float:
|
| 159 |
+
if not math.isfinite(value) or value < -1e-6 or value > 1 + 1e-6:
|
| 160 |
+
raise FloatingPointError("Invalid confidence")
|
| 161 |
+
return max(0.0, min(1.0, value))
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# TypeSafe formulas: https://docs.typesafe.ai/confidence
|
| 165 |
+
def noul_confidence(probability: float) -> float:
|
| 166 |
+
"""Optional TypeSafe confidence measure; Noul answers expose only p(true)."""
|
| 167 |
+
return _bounded_confidence(abs(2 * probability - 1))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def choice_confidence(probabilities: list[float]) -> float:
|
| 171 |
+
"""TypeSafe's top probability above an even split, with singleton support."""
|
| 172 |
+
n = len(probabilities)
|
| 173 |
+
if n == 1:
|
| 174 |
+
return 1.0
|
| 175 |
+
return _bounded_confidence((max(probabilities) - 1 / n) / (1 - 1 / n))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def score_confidence(probabilities: list[float]) -> float:
|
| 179 |
+
"""TypeSafe's spread around the first most likely level, in rubric order."""
|
| 180 |
+
n = len(probabilities)
|
| 181 |
+
m = probabilities.index(max(probabilities))
|
| 182 |
+
spread = sum(p * abs(i - m) for i, p in enumerate(probabilities))
|
| 183 |
+
even_spread = sum(abs(i - (n - 1) / 2) for i in range(n)) / n
|
| 184 |
+
return _bounded_confidence(max(0.0, 1 - spread / even_spread))
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def make_answer(question: Question, probabilities: list[float]) -> Any:
|
| 188 |
+
mapping = dict(zip(candidate_ids(question), probabilities, strict=True))
|
| 189 |
+
if question.type == "noul":
|
| 190 |
+
return NoulAnswer(noul=probabilities[1])
|
| 191 |
+
if question.type == "choice":
|
| 192 |
+
return ChoiceAnswer(
|
| 193 |
+
choice=max(mapping, key=mapping.get),
|
| 194 |
+
probabilities=mapping,
|
| 195 |
+
confidence=choice_confidence(probabilities),
|
| 196 |
+
)
|
| 197 |
+
return ScoreAnswer(
|
| 198 |
+
score=sum(k * p for k, p in enumerate(probabilities)),
|
| 199 |
+
legend=dict(zip(candidate_ids(question), question.criteria)),
|
| 200 |
+
probabilities=mapping,
|
| 201 |
+
confidence=score_confidence(probabilities),
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def aligned_probabilities(question: Question, answer: Answer) -> list[float]:
|
| 206 |
+
if answer.type == "noul":
|
| 207 |
+
return [1 - answer.noul, answer.noul]
|
| 208 |
+
return [answer.probabilities[k] for k in candidate_ids(question)]
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class Calibrator(nn.Module):
|
| 212 |
+
def __init__(self, config=None):
|
| 213 |
+
super().__init__()
|
| 214 |
+
self.config = CalibrationConfig.model_validate(config or {})
|
| 215 |
+
if self.config.mode == "scalar_temperature":
|
| 216 |
+
self.beta = nn.Parameter(torch.zeros(()))
|
| 217 |
+
|
| 218 |
+
def forward(self):
|
| 219 |
+
if self.config.mode == "identity":
|
| 220 |
+
return 1.0
|
| 221 |
+
if not torch.isfinite(self.beta):
|
| 222 |
+
raise FloatingPointError("Nonfinite scalar temperature")
|
| 223 |
+
return (self.config.log_temperature_bound * self.beta.float().tanh()).exp()
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
@dataclass
|
| 227 |
+
class Probabilities:
|
| 228 |
+
candidate_mask: torch.Tensor
|
| 229 |
+
log_p_raw: torch.Tensor
|
| 230 |
+
p_raw: torch.Tensor
|
| 231 |
+
entropy: torch.Tensor
|
| 232 |
+
margin: torch.Tensor
|
| 233 |
+
temperature: torch.Tensor
|
| 234 |
+
log_p_final: torch.Tensor
|
| 235 |
+
p_final: torch.Tensor
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def probabilities(logits, candidate_mask=None, calibrator=None):
|
| 239 |
+
if logits.ndim < 1 or logits.shape[-1] == 0:
|
| 240 |
+
raise ValueError("Expected a nonempty candidate axis")
|
| 241 |
+
with torch.autocast(device_type=logits.device.type, enabled=False):
|
| 242 |
+
logits = logits.float()
|
| 243 |
+
mask = torch.ones_like(logits, dtype=torch.bool) if candidate_mask is None else candidate_mask
|
| 244 |
+
if mask.dtype != torch.bool or mask.shape != logits.shape or not mask.any(-1).all():
|
| 245 |
+
raise ValueError("Candidate mask must match logits and contain nonempty support")
|
| 246 |
+
if not torch.isfinite(logits[mask]).all():
|
| 247 |
+
raise FloatingPointError("Nonfinite valid decision logits")
|
| 248 |
+
log_raw = logits.masked_fill(~mask, -torch.inf).log_softmax(-1)
|
| 249 |
+
if not torch.isfinite(log_raw[mask]).all():
|
| 250 |
+
raise FloatingPointError("Valid log probabilities exceed FP32 range")
|
| 251 |
+
raw = log_raw.exp()
|
| 252 |
+
entropy = -(raw * log_raw.masked_fill(~mask, 0)).sum(-1)
|
| 253 |
+
top = raw.topk(min(2, logits.shape[-1]), dim=-1).values
|
| 254 |
+
margin = top[..., 0] - (top[..., 1] if top.shape[-1] > 1 else 0)
|
| 255 |
+
value = calibrator() if calibrator is not None else 1.0
|
| 256 |
+
temperature = torch.ones_like(entropy) * value
|
| 257 |
+
temperature = torch.where(mask.sum(-1) == 1, torch.ones_like(temperature), temperature)
|
| 258 |
+
if calibrator is None or calibrator.config.mode == "identity":
|
| 259 |
+
final_log = log_raw
|
| 260 |
+
else:
|
| 261 |
+
final_log = (
|
| 262 |
+
(logits.masked_fill(~mask, 0) / temperature.unsqueeze(-1))
|
| 263 |
+
.masked_fill(~mask, -torch.inf)
|
| 264 |
+
.log_softmax(-1)
|
| 265 |
+
)
|
| 266 |
+
return Probabilities(mask, log_raw, raw, entropy, margin, temperature, final_log, final_log.exp())
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0579ce9ee8ce5d0c244b069842aac4f8e1e65babd8ebfe4d98fe4fbf406831f1
|
| 3 |
+
size 19990645
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"audio_bos_token": "<|audio_start|>",
|
| 4 |
+
"audio_eos_token": "<|audio_end|>",
|
| 5 |
+
"audio_token": "<|audio_pad|>",
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": null,
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"extra_special_tokens": [
|
| 12 |
+
"<state>",
|
| 13 |
+
"</state>",
|
| 14 |
+
"<question>",
|
| 15 |
+
"</question>",
|
| 16 |
+
"<option>",
|
| 17 |
+
"</option>",
|
| 18 |
+
"<|decide|>"
|
| 19 |
+
],
|
| 20 |
+
"image_token": "<|image_pad|>",
|
| 21 |
+
"is_local": true,
|
| 22 |
+
"model_max_length": 262144,
|
| 23 |
+
"model_specific_special_tokens": {
|
| 24 |
+
"audio_bos_token": "<|audio_start|>",
|
| 25 |
+
"audio_eos_token": "<|audio_end|>",
|
| 26 |
+
"audio_token": "<|audio_pad|>",
|
| 27 |
+
"image_token": "<|image_pad|>",
|
| 28 |
+
"video_token": "<|video_pad|>",
|
| 29 |
+
"vision_bos_token": "<|vision_start|>",
|
| 30 |
+
"vision_eos_token": "<|vision_end|>"
|
| 31 |
+
},
|
| 32 |
+
"pad_token": "<|endoftext|>",
|
| 33 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 34 |
+
"split_special_tokens": false,
|
| 35 |
+
"tokenizer_class": "TokenizersBackend",
|
| 36 |
+
"unk_token": null,
|
| 37 |
+
"video_token": "<|video_pad|>",
|
| 38 |
+
"vision_bos_token": "<|vision_start|>",
|
| 39 |
+
"vision_eos_token": "<|vision_end|>"
|
| 40 |
+
}
|
util.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Native checkpoint loading, serialization and deterministic identities."""
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import init_empty_weights
|
| 9 |
+
from safetensors.torch import load_file
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def canonical(value) -> str:
|
| 13 |
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def digest(value) -> str:
|
| 17 |
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def file_hash(path: Path) -> str:
|
| 21 |
+
h = hashlib.sha256()
|
| 22 |
+
with path.open("rb") as stream:
|
| 23 |
+
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
|
| 24 |
+
h.update(chunk)
|
| 25 |
+
return h.hexdigest()
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def named_seed(seed, name):
|
| 29 |
+
return int.from_bytes(hashlib.sha256(canonical(["gs1.rng", seed, name]).encode()).digest()[:8], "big") % (
|
| 30 |
+
2**63
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _weight_files(directory):
|
| 35 |
+
directory = Path(directory)
|
| 36 |
+
index = directory / "model.safetensors.index.json"
|
| 37 |
+
if index.is_file():
|
| 38 |
+
weight_map = json.loads(index.read_text())["weight_map"]
|
| 39 |
+
files = sorted(set(weight_map.values()))
|
| 40 |
+
else:
|
| 41 |
+
files = sorted(p.name for p in directory.glob("*.safetensors"))
|
| 42 |
+
weight_map = None
|
| 43 |
+
if not files:
|
| 44 |
+
raise ValueError("Checkpoint contains no safetensors weights")
|
| 45 |
+
if any(Path(name).is_absolute() or ".." in Path(name).parts for name in files):
|
| 46 |
+
raise ValueError("Checkpoint shard is outside its directory")
|
| 47 |
+
# Hub snapshots link to shared cache blobs; validate shard names rather
|
| 48 |
+
# than rejecting those normal cache symlinks after resolving their targets.
|
| 49 |
+
return [directory / name for name in files], weight_map
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def checkpoint_files(directory):
|
| 53 |
+
"""Files that define the weights, native configuration and input encoding."""
|
| 54 |
+
directory = Path(directory)
|
| 55 |
+
names = (
|
| 56 |
+
"config.json",
|
| 57 |
+
"model.safetensors.index.json",
|
| 58 |
+
"tokenizer.json",
|
| 59 |
+
"tokenizer_config.json",
|
| 60 |
+
"special_tokens_map.json",
|
| 61 |
+
"added_tokens.json",
|
| 62 |
+
"vocab.json",
|
| 63 |
+
"vocab.txt",
|
| 64 |
+
"merges.txt",
|
| 65 |
+
"tokenizer.model",
|
| 66 |
+
"chat_template.jinja",
|
| 67 |
+
"preprocessor_config.json",
|
| 68 |
+
"video_preprocessor_config.json",
|
| 69 |
+
)
|
| 70 |
+
weights, _ = _weight_files(directory)
|
| 71 |
+
return sorted({*weights, *(directory / name for name in names if (directory / name).is_file())})
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def read_weights(directory):
|
| 75 |
+
files, weight_map = _weight_files(directory)
|
| 76 |
+
state = {}
|
| 77 |
+
for path in files:
|
| 78 |
+
shard = load_file(path, device="cpu")
|
| 79 |
+
if state.keys() & shard.keys():
|
| 80 |
+
raise ValueError("Duplicate checkpoint parameter")
|
| 81 |
+
state.update(shard)
|
| 82 |
+
if weight_map is not None and state.keys() != weight_map.keys():
|
| 83 |
+
raise ValueError("Checkpoint index differs from its parameter contents")
|
| 84 |
+
return state
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _construct(config, model_class=None):
|
| 88 |
+
from .modular_gs1 import GS1Model
|
| 89 |
+
|
| 90 |
+
# Nonpersistent rotary buffers must remain real tensors, not meta placeholders.
|
| 91 |
+
with init_empty_weights(include_buffers=False):
|
| 92 |
+
return (model_class or GS1Model)(config)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _assign(model, state):
|
| 96 |
+
expected = model.state_dict()
|
| 97 |
+
if state.keys() != expected.keys():
|
| 98 |
+
raise ValueError(
|
| 99 |
+
f"Incomplete GS1 weights: missing={sorted(expected.keys() - state.keys())}, "
|
| 100 |
+
f"unexpected={sorted(state.keys() - expected.keys())}"
|
| 101 |
+
)
|
| 102 |
+
for name, value in state.items():
|
| 103 |
+
if value.shape != expected[name].shape:
|
| 104 |
+
raise ValueError(f"GS1 parameter shape mismatch: {name}")
|
| 105 |
+
model.load_state_dict(state, strict=True, assign=True)
|
| 106 |
+
return model
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def resolve_source(
|
| 110 |
+
source, *, revision=None, local_files_only=False, cache_dir=None, token=None, force_download=False
|
| 111 |
+
):
|
| 112 |
+
"""Resolve a native directory or an immutable Hub snapshot."""
|
| 113 |
+
path = Path(source).expanduser()
|
| 114 |
+
if path.is_dir():
|
| 115 |
+
return path.resolve()
|
| 116 |
+
from huggingface_hub import snapshot_download
|
| 117 |
+
|
| 118 |
+
return Path(
|
| 119 |
+
snapshot_download(
|
| 120 |
+
str(source),
|
| 121 |
+
revision=revision,
|
| 122 |
+
local_files_only=local_files_only,
|
| 123 |
+
cache_dir=cache_dir,
|
| 124 |
+
token=token,
|
| 125 |
+
force_download=force_download,
|
| 126 |
+
allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.py", "*.model", "*.txt"],
|
| 127 |
+
)
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def load_native(
|
| 132 |
+
source,
|
| 133 |
+
*,
|
| 134 |
+
device="cpu",
|
| 135 |
+
dtype=None,
|
| 136 |
+
model_class=None,
|
| 137 |
+
config=None,
|
| 138 |
+
revision=None,
|
| 139 |
+
local_files_only=False,
|
| 140 |
+
cache_dir=None,
|
| 141 |
+
token=None,
|
| 142 |
+
force_download=False,
|
| 143 |
+
**kwargs,
|
| 144 |
+
):
|
| 145 |
+
from .modular_gs1 import GS1Config
|
| 146 |
+
|
| 147 |
+
for name in (
|
| 148 |
+
"_from_auto",
|
| 149 |
+
"_from_pipeline",
|
| 150 |
+
"_commit_hash",
|
| 151 |
+
"trust_remote_code",
|
| 152 |
+
"code_revision",
|
| 153 |
+
"weights_only",
|
| 154 |
+
"use_safetensors",
|
| 155 |
+
"adapter_kwargs",
|
| 156 |
+
):
|
| 157 |
+
kwargs.pop(name, None)
|
| 158 |
+
if "torch_dtype" in kwargs:
|
| 159 |
+
if dtype is not None:
|
| 160 |
+
raise ValueError("Use only dtype or torch_dtype")
|
| 161 |
+
dtype = kwargs.pop("torch_dtype")
|
| 162 |
+
device_map = kwargs.pop("device_map", None)
|
| 163 |
+
if device_map is not None:
|
| 164 |
+
if isinstance(device_map, dict) and set(device_map) == {""}:
|
| 165 |
+
device = device_map[""]
|
| 166 |
+
elif isinstance(device_map, str) and (
|
| 167 |
+
device_map in {"cpu", "cuda"} or device_map.startswith("cuda:")
|
| 168 |
+
):
|
| 169 |
+
device = device_map
|
| 170 |
+
else:
|
| 171 |
+
raise ValueError(
|
| 172 |
+
"GS1 supports a single device; use .to(device) instead of automatic device mapping"
|
| 173 |
+
)
|
| 174 |
+
if kwargs:
|
| 175 |
+
raise TypeError(f"Unsupported GS1 loading options: {sorted(kwargs)}")
|
| 176 |
+
directory = resolve_source(
|
| 177 |
+
source,
|
| 178 |
+
revision=revision,
|
| 179 |
+
local_files_only=local_files_only,
|
| 180 |
+
cache_dir=cache_dir,
|
| 181 |
+
token=token,
|
| 182 |
+
force_download=force_download,
|
| 183 |
+
)
|
| 184 |
+
raw = json.loads((directory / "config.json").read_text())
|
| 185 |
+
if raw.get("model_type") != "gs1":
|
| 186 |
+
raise ValueError("Expected native GS1 weights")
|
| 187 |
+
config = config or GS1Config(**raw)
|
| 188 |
+
config._name_or_path = str(source)
|
| 189 |
+
model = _construct(config, model_class)
|
| 190 |
+
model = _assign(model, read_weights(directory))
|
| 191 |
+
model._processor_source = directory
|
| 192 |
+
if dtype is not None and dtype != "auto":
|
| 193 |
+
if isinstance(dtype, str):
|
| 194 |
+
dtype = getattr(torch, dtype)
|
| 195 |
+
model.to(dtype=dtype)
|
| 196 |
+
return model.to(device).eval()
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def export_state(model, state=None):
|
| 200 |
+
"""Save the native inference parameter tree without dtype conversion."""
|
| 201 |
+
state = model.state_dict() if state is None else state
|
| 202 |
+
return {name: value.detach().contiguous() for name, value in state.items()}
|
video_preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 25165824,
|
| 4 |
+
"shortest_edge": 4096
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"video_processor_type": "Qwen3VLVideoProcessor"
|
| 21 |
+
}
|