Upload GS1-4B checkpoint and inference files
Browse files- .gitattributes +1 -0
- LICENSE +202 -0
- NOTICE +6 -0
- README.md +57 -192
- README.zh-CN.md +111 -0
- chat_template.jinja +154 -0
- compiler.py +395 -0
- config.json +166 -0
- config.py +46 -0
- model-00001-of-00005.safetensors +3 -0
- model-00002-of-00005.safetensors +3 -0
- model-00003-of-00005.safetensors +3 -0
- model-00004-of-00005.safetensors +3 -0
- model-00005-of-00005.safetensors +3 -0
- model.safetensors.index.json +792 -0
- modular_gs1.py +971 -0
- preprocessor_config.json +21 -0
- requirements.txt +10 -0
- schema.py +266 -0
- tokenizer.json +3 -0
- tokenizer_config.json +40 -0
- util.py +202 -0
- video_preprocessor_config.json +21 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
GS1
|
| 2 |
+
Copyright 2026 BIGAI.
|
| 3 |
+
|
| 4 |
+
GS1 uses Qwen3.5 components from Hugging Face Transformers under Apache-2.0.
|
| 5 |
+
Third-party libraries retain their respective copyright and licenses.
|
| 6 |
+
Model weights are distributed under the license stated in each model card.
|
README.md
CHANGED
|
@@ -1,123 +1,79 @@
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
-
language:
|
| 4 |
-
- en
|
| 5 |
-
base_model:
|
| 6 |
-
- Qwen/Qwen3.5-4B
|
| 7 |
-
base_model_relation: finetune
|
| 8 |
-
model_name: GS1-4B
|
| 9 |
-
thumbnail: https://huggingface.co/bigai/GS1-4B/resolve/main/assets/banner.svg
|
| 10 |
-
tags:
|
| 11 |
-
- decision-making
|
| 12 |
-
- typed-decisions
|
| 13 |
-
- agent-routing
|
| 14 |
-
- verification
|
| 15 |
-
- uncertainty-estimation
|
| 16 |
-
- classification
|
| 17 |
-
- qwen3.5
|
| 18 |
library_name: transformers
|
| 19 |
-
|
| 20 |
-
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
dataset:
|
| 26 |
-
type: decision-index
|
| 27 |
-
name: Decision Index 0.3 (Public)
|
| 28 |
-
metrics:
|
| 29 |
-
- type: decision-index
|
| 30 |
-
name: Decision Index Score
|
| 31 |
-
value: 68.97
|
| 32 |
-
source:
|
| 33 |
-
name: GS1 complete public evaluation
|
| 34 |
-
url: https://github.com/bigai-nlco/GS1/blob/main/benchmark/results.md
|
| 35 |
---
|
| 36 |
-
# GS1-4B
|
| 37 |
-
|
| 38 |
-
> **Fast, Faithful, Frontier System One Intelligence.**
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
[Code](https://github.com/bigai-nlco/GS1) · [Models](https://huggingface.co/collections/bigai/gs1) · [Quick Start](#quick-start) · [Results](#evaluation) · [Playground](#interactive-playground)
|
| 43 |
|
| 44 |
-
|
| 45 |
|
| 46 |
-
|
| 47 |
-
> Load this model directly with Transformers `AutoModel` and call `model.decide()`. The GS1 repository is optional for default inference; install it for optimized execution, serving, and evaluation.
|
| 48 |
|
| 49 |
-
|
| 50 |
|
| 51 |
-
|
| 52 |
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
-
|
| 59 |
|
| 60 |
-
|
| 61 |
|
| 62 |
-
##
|
| 63 |
|
| 64 |
-
**Python 3.11+
|
| 65 |
|
| 66 |
```bash
|
| 67 |
-
python -m pip install
|
| 68 |
```
|
| 69 |
|
| 70 |
```python
|
| 71 |
-
import json
|
| 72 |
from transformers import AutoModel
|
| 73 |
|
| 74 |
model = AutoModel.from_pretrained(
|
| 75 |
-
"bigai/GS1-4B", #
|
| 76 |
trust_remote_code=True,
|
| 77 |
).eval().to("cuda")
|
| 78 |
|
| 79 |
result = model.decide(
|
| 80 |
-
state=
|
| 81 |
questions={
|
| 82 |
-
"
|
| 83 |
"type": "choice",
|
| 84 |
-
"instructions": "Choose
|
| 85 |
-
"criteria": {"
|
| 86 |
},
|
| 87 |
-
"
|
| 88 |
-
|
| 89 |
-
"instructions": "Can delivery finish with at least the required battery reserve?",
|
| 90 |
-
},
|
| 91 |
-
"risk": {
|
| 92 |
"type": "score",
|
| 93 |
-
"instructions": "Rate the
|
| 94 |
-
"criteria": ["
|
| 95 |
},
|
| 96 |
},
|
| 97 |
-
precision="bf16",
|
| 98 |
)
|
| 99 |
-
print(
|
| 100 |
```
|
| 101 |
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
| Type | Answer fields | Input limits |
|
| 105 |
-
| :--- | :--- | :--- |
|
| 106 |
-
| Choice | `choice`, `probabilities`, `confidence` | 1–255 candidates |
|
| 107 |
-
| Noul | `noul` = `p(true)` | True/false verification |
|
| 108 |
-
| Score | `score`, `probabilities`, `legend`, `confidence` | 2–10 ordered levels, starting at 0 |
|
| 109 |
-
|
| 110 |
-
> [!Tip]
|
| 111 |
-
> Put shared facts in `state` and related questions in one request. Include an explicit `unknown` candidate when evidence may be insufficient. Confidence is a distribution summary, not a guarantee of correctness.
|
| 112 |
|
| 113 |
-
|
| 114 |
-
> Preserve the checkpoint's stored dtypes; use `precision` to select computation precision rather than converting all weights to BF16. For CPU reference inference, use `.to("cpu")` and `precision="fp32"`.
|
| 115 |
|
| 116 |
-
|
| 117 |
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
From a Linux/CUDA environment:
|
| 121 |
|
| 122 |
```bash
|
| 123 |
git clone https://github.com/bigai-nlco/GS1.git
|
|
@@ -126,125 +82,34 @@ uv sync --locked --extra serve --extra cuda
|
|
| 126 |
uv run gs1 serve --model bigai/GS1-4B --model-name GS1
|
| 127 |
```
|
| 128 |
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
```bash
|
| 132 |
-
curl http://127.0.0.1:8000/v1/systemone \
|
| 133 |
-
-H 'Content-Type: application/json' \
|
| 134 |
-
-d '{
|
| 135 |
-
"model": "GS1",
|
| 136 |
-
"state": {"battery": 18, "delivery_cost": 12, "required_reserve": 10},
|
| 137 |
-
"questions": {
|
| 138 |
-
"action": {
|
| 139 |
-
"type": "choice",
|
| 140 |
-
"instructions": "Choose an action that preserves the required battery reserve.",
|
| 141 |
-
"criteria": {"deliver": "Deliver now.", "recharge": "Recharge first."}
|
| 142 |
-
}
|
| 143 |
-
}
|
| 144 |
-
}'
|
| 145 |
-
```
|
| 146 |
-
|
| 147 |
-
The service also provides `GET /v1/models` and interactive API docs at `/docs`. Local requests need no cloud credentials. For an in-process runtime, use `Engine.from_pretrained(...)` from `gs1`. [Inference guide](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.md).
|
| 148 |
-
|
| 149 |
-
### TypeSafe SDK
|
| 150 |
-
|
| 151 |
-
With the service running and `typesafe-sdk==0.7.0` installed:
|
| 152 |
-
|
| 153 |
-
```python
|
| 154 |
-
from typesafe_sdk import Choice, Noul, Score, TypeSafeClient
|
| 155 |
-
|
| 156 |
-
with TypeSafeClient(
|
| 157 |
-
base_url="http://127.0.0.1:8000", api_key="local", model="GS1"
|
| 158 |
-
) as client:
|
| 159 |
-
result = client.system_one(
|
| 160 |
-
state="A customer was charged twice and requests a refund.",
|
| 161 |
-
questions={
|
| 162 |
-
"team": Choice(
|
| 163 |
-
instructions="Which team should handle this ticket?",
|
| 164 |
-
criteria={"billing": "Payments and refunds", "technical": "Software problems"},
|
| 165 |
-
),
|
| 166 |
-
"refund": Noul(instructions="Does the customer request a refund?"),
|
| 167 |
-
"urgency": Score(
|
| 168 |
-
instructions="Rate urgency.", criteria=["Routine", "Soon", "Immediate"]
|
| 169 |
-
),
|
| 170 |
-
},
|
| 171 |
-
)
|
| 172 |
-
|
| 173 |
-
print(result.choices["team"].probabilities)
|
| 174 |
-
print(result.nouls["refund"].noul)
|
| 175 |
-
print(result.scores["urgency"].score)
|
| 176 |
-
```
|
| 177 |
-
|
| 178 |
-
`api_key="local"` satisfies the SDK argument. Use the API root as `base_url` and match `model` to the server alias. [API and confidence definitions](https://github.com/bigai-nlco/GS1/blob/main/docs/api.md).
|
| 179 |
-
|
| 180 |
-
### Interactive Playground
|
| 181 |
-
|
| 182 |
-
With GS1 serving on port 8000, run from the source checkout (**Node.js 22+**):
|
| 183 |
-
|
| 184 |
-
```bash
|
| 185 |
-
cd playground
|
| 186 |
-
npm ci
|
| 187 |
-
npm run build
|
| 188 |
-
npm start
|
| 189 |
-
```
|
| 190 |
-
|
| 191 |
-
Open [localhost:3000](http://127.0.0.1:3000). GS1 is the default backend. Explore **Query, Go, Maze, and Snake**, with probabilities and raw JSON. [Playground guide](https://github.com/bigai-nlco/GS1/blob/main/playground/README.md).
|
| 192 |
-
|
| 193 |
-
## Evaluation
|
| 194 |
-
|
| 195 |
-
**Decision Index 0.3 (Public): 68.97**, from the complete 7 November 2026 public evaluation. This is a chance-corrected composite score, not accuracy. The 4B run covered 37 public benchmarks plus four AA component suites, with 115,479 unique requests, 279,333 decisions, and zero final failed requests. Evaluation used BF16 computation and a 32,768-token branch limit.
|
| 196 |
-
|
| 197 |
-
[](assets/gs1_pareto_frontier.svg)
|
| 198 |
-
|
| 199 |
-
*Technical-report comparison snapshot (6 November 2026); the complete evaluation above uses the subsequent run.*
|
| 200 |
-
|
| 201 |
-
Additional full-split reference-engine results, **9 November 2026**:
|
| 202 |
-
|
| 203 |
-
| Benchmark | Decisions | Accuracy | Brier ↓ | ECE ↓ |
|
| 204 |
-
| :--- | ---: | ---: | ---: | ---: |
|
| 205 |
-
| JevBench public | 231 | 86.15% | 0.179350 | 0.050455 |
|
| 206 |
-
| AG News | 7,600 | 92.01% | 0.122275 | 0.016153 |
|
| 207 |
-
| ToolACE selection | 640 | 94.22% | 0.088759 | 0.027009 |
|
| 208 |
-
|
| 209 |
-
All 8,471 requests completed without failures or retries. ToolACE measures first-tool/no-tool selection, not argument generation or execution. ECE uses ten equal-width top-label probability bins; no test-set calibration is fitted. [Full results and source hashes](https://github.com/bigai-nlco/GS1/blob/main/benchmark/results.md).
|
| 210 |
-
|
| 211 |
-
Reproduce the regular benchmarks from the source checkout:
|
| 212 |
|
| 213 |
```bash
|
| 214 |
-
uv
|
| 215 |
-
|
| 216 |
-
--
|
| 217 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
```
|
| 219 |
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
## Inference Performance
|
| 223 |
-
|
| 224 |
-
GS1-4B, **256-token longest branch**, BF16 computation, one RTX 4090 reporting **48 GiB**, five warmups and 30 timed calls per group:
|
| 225 |
|
| 226 |
-
|
| 227 |
-
| :--- | ---: | ---: | ---: |
|
| 228 |
-
| 1 | 72.87 ms | **35.17 / 35.24 ms** | **2.07×** |
|
| 229 |
-
| 8 | 566.18 ms | **240.44 / 241.26 ms** | **2.35×** |
|
| 230 |
|
| 231 |
-
|
| 232 |
|
| 233 |
-
|
| 234 |
-
> Default loading uses eager inference. CUDA Graph and shared execution require a matching runtime qualification; gains depend on request shape. Follow the [CUDA Graph setup](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.md#cuda-graph-inference) before comparing these timings.
|
| 235 |
|
| 236 |
-
|
| 237 |
|
| 238 |
-
|
| 239 |
|
| 240 |
-
|
| 241 |
-
* Probabilities may be miscalibrated under distribution shift. Validate accuracy, calibration, and abstention thresholds on your workload.
|
| 242 |
-
* Base-model and training-data biases may affect decisions; use human review for consequential applications. Reported benchmarks are text-only and do not establish multimodal or safety performance.
|
| 243 |
-
* Requests exceeding configured context or admission budgets fail explicitly; state and candidates are not silently truncated.
|
| 244 |
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
If you find GS1 useful, please cite:
|
| 248 |
|
| 249 |
```bibtex
|
| 250 |
@techreport{zheng2026gs1,
|
|
@@ -253,4 +118,4 @@ If you find GS1 useful, please cite:
|
|
| 253 |
institution={BIGAI},
|
| 254 |
year={2026}
|
| 255 |
}
|
| 256 |
-
```
|
|
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
library_name: transformers
|
| 4 |
+
tags:
|
| 5 |
+
- gs1
|
| 6 |
+
- decision-model
|
| 7 |
+
- custom_code
|
| 8 |
+
- system-one
|
| 9 |
+
- safetensors
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
---
|
| 11 |
+
# GS1-4B
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
+
[English](README.md) · [简体中文](README.zh-CN.md)
|
| 14 |
|
| 15 |
+
|
|
|
|
| 16 |
|
| 17 |
+
**Fast, Faithful, Frontier.** GS1-4B is a native typed decision model. Supply state, instructions and runtime candidate meanings; get **Choice**, **Noul** and **Score** distributions with zero generated output tokens.
|
| 18 |
|
| 19 |
+
[Code and optimized runtime](https://github.com/bigai-nlco/GS1) · [Model family](https://huggingface.co/collections/bigai/gs1) · [Detailed results](https://github.com/bigai-nlco/GS1/blob/main/benchmark/results.md)
|
| 20 |
|
| 21 |
+
| Measurement | GS1-4B |
|
| 22 |
+
|---|---:|
|
| 23 |
+
| Stored parameters | 4,704,520,192 |
|
| 24 |
+
| Public Decision Index, 37 benchmarks | **68.97** |
|
| 25 |
+
| JevBench public task accuracy, 231 decisions | **86.15%** |
|
| 26 |
+
| AG News test accuracy, 7,600 decisions | **92.01%** |
|
| 27 |
+
| ToolACE selection accuracy, 640 decisions | **94.22%** |
|
| 28 |
+
| Warm complete CUDA Graph decision P50 / P95 | **35.17 / 35.24 ms** |
|
| 29 |
+
| Peak allocated GPU memory in that latency fixture | **23.68 GiB** |
|
| 30 |
|
| 31 |
+
Decision Index: complete public edition 0.3, 115,479 unique requests and 279,333 decisions, 100% final coverage, BF16 native reference computation. This is a public score, separate from private Full scores. The three regular benchmarks were measured on 9 November 2026 with zero failures or retries. JevBench public task accuracy is separate from the sealed composite. ToolACE measures first-tool/no-tool selection on a fixed hash subset, not tool-argument generation.
|
| 32 |
|
| 33 |
+
Latency measured on 9 November 2026: RTX 4090 reporting 48 GiB, BF16 computation, 256-token longest branch, one question, single GPU, five warmups and 30 measured calls. Complete in-process timing includes validation, compilation, model computation and probabilities; loading, graph capture, HTTP and SDK transport are excluded. [Machine-readable decision metrics](assets/results/public-evaluation.json) · [Raw latency samples](assets/results/inference-latency.json) · [Methodology and all models](https://github.com/bigai-nlco/GS1/blob/main/benchmark/latency.md)
|
| 34 |
|
| 35 |
+
## Default Hugging Face inference
|
| 36 |
|
| 37 |
+
**No installation or import of the GS1 repository is required.** Use Python 3.11+ and install this folder's dependencies:
|
| 38 |
|
| 39 |
```bash
|
| 40 |
+
python -m pip install -r requirements.txt
|
| 41 |
```
|
| 42 |
|
| 43 |
```python
|
|
|
|
| 44 |
from transformers import AutoModel
|
| 45 |
|
| 46 |
model = AutoModel.from_pretrained(
|
| 47 |
+
"bigai/GS1-4B", # a local GS1-4B directory also works
|
| 48 |
trust_remote_code=True,
|
| 49 |
).eval().to("cuda")
|
| 50 |
|
| 51 |
result = model.decide(
|
| 52 |
+
state="A customer was charged twice and asks for a refund.",
|
| 53 |
questions={
|
| 54 |
+
"team": {
|
| 55 |
"type": "choice",
|
| 56 |
+
"instructions": "Choose the team that should handle this ticket.",
|
| 57 |
+
"criteria": {"billing": "Payments and refunds", "technical": "Software problems"},
|
| 58 |
},
|
| 59 |
+
"refund": {"type": "noul", "instructions": "Does the customer request a refund?"},
|
| 60 |
+
"urgency": {
|
|
|
|
|
|
|
|
|
|
| 61 |
"type": "score",
|
| 62 |
+
"instructions": "Rate the urgency of handling this ticket.",
|
| 63 |
+
"criteria": ["Routine", "Soon", "Immediate"],
|
| 64 |
},
|
| 65 |
},
|
|
|
|
| 66 |
)
|
| 67 |
+
print(result["answers"])
|
| 68 |
```
|
| 69 |
|
| 70 |
+
This works from a local release directory before the Hub upload as well. `trust_remote_code=True` loads the model code supplied in this folder. Default loading preserves the checkpoint's mixed storage dtypes; avoid a blanket dtype conversion when reproducing the measured precision policy. On a CPU-compatible PyTorch installation, use `.to("cpu")` and `decide(..., precision="fp32")`. Optional upstream CUDA kernels can be installed for faster eager inference; the measured stack uses `flash-linear-attention[cuda]==0.5.2` and `causal-conv1d==1.7.0`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
+
`result` contains `model`, `answers` and `usage`. Choice returns a candidate ID, a probability for every candidate, and confidence. Noul returns `p(true)`. Score returns the expected zero-based rubric level and probabilities over all supplied levels. Output-token usage is zero. The default path evaluates complete question branches independently and has no runtime profile or graph-capture dependency.
|
|
|
|
| 73 |
|
| 74 |
+
## Optimized GS1 inference
|
| 75 |
|
| 76 |
+
The GS1 repository provides source-bound CUDA Graph replay, compatible batching, shared-state reuse, HTTP serving and a playground for these same weights.
|
|
|
|
|
|
|
| 77 |
|
| 78 |
```bash
|
| 79 |
git clone https://github.com/bigai-nlco/GS1.git
|
|
|
|
| 82 |
uv run gs1 serve --model bigai/GS1-4B --model-name GS1
|
| 83 |
```
|
| 84 |
|
| 85 |
+
From the source checkout, install `--extra eval` in addition to the serving/CUDA extras, then qualify the installed model, hardware and request geometry before enabling graphs:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
|
| 87 |
```bash
|
| 88 |
+
uv run python -m benchmark.inference --model bigai/GS1-4B \
|
| 89 |
+
--out runs/runtime.json --runtime-profile-out runs/profile.json \
|
| 90 |
+
--graph-cache-mb 16384 --graph-cache-entries 128
|
| 91 |
+
|
| 92 |
+
uv run gs1 serve --model bigai/GS1-4B --model-name GS1 \
|
| 93 |
+
--runtime-profile runs/profile.json --experimental-cuda-graphs \
|
| 94 |
+
--graph-warmup-requests runs/profile-warmup.json \
|
| 95 |
+
--graph-cache-mb 4096 --graph-cache-entries 128
|
| 96 |
```
|
| 97 |
|
| 98 |
+
The benchmark writes `profile-warmup.json`. Serve known warmed graphs, with native eager fallback for graph misses and single-reference fallback for unqualified geometries. Regenerate the profile when code, model, precision or hardware changes. These profiles belong to the local runtime and are not uploaded as model weights. [Full inference guide](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.md)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
+
## Architecture and numerical policy
|
|
|
|
|
|
|
|
|
|
| 101 |
|
| 102 |
+
|
| 103 |
|
| 104 |
+
GS1-4B has width 2,560, 32 decoder layers and 10 decision-attention heads. It combines a Qwen3.5 hybrid GDN/attention backbone, segmented bidirectional full attention, carry-preserving Attention Residuals and a native cross-attention decision head with two routing layers and a 256-dimensional final comparison space. There is no language-generation output head. Candidate meanings are supplied at inference time rather than fixed to a task-specific classifier vocabulary.
|
|
|
|
| 105 |
|
| 106 |
+
Persistent text/head weights are FP32 and the retained vision tower is BF16; the checkpoint specifies BF16 head computation. Final logits/probability reductions are FP32. Complete branches support up to 65,536 tokens subject to explicit admission and memory budgets; the default HTTP service uses a tighter 4,096-token limit. The published decision results are text-only. Model definitions, token IDs and `inference_limits` are in `config.json`; all safetensors keys and dtypes are preserved.
|
| 107 |
|
| 108 |
+
## Scope and license
|
| 109 |
|
| 110 |
+
Use for routing, classification, tool selection, bounded judgments and other typed decisions. A returned probability is not a guarantee of correctness; calibration depends on the task distribution. The public results include limitations such as low HLE accuracy and zero AA-Omniscience non-hallucination rate at larger sizes when confidence rejection is disabled; inspect [all component results](assets/results/public-evaluation.json).
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
+
Code and released weights: [Apache-2.0](LICENSE), with [upstream attribution](NOTICE). This folder contains model architecture and inference dependencies. Evaluation inputs and local runtime profiles are distributed separately.
|
|
|
|
|
|
|
| 113 |
|
| 114 |
```bibtex
|
| 115 |
@techreport{zheng2026gs1,
|
|
|
|
| 118 |
institution={BIGAI},
|
| 119 |
year={2026}
|
| 120 |
}
|
| 121 |
+
```
|
README.zh-CN.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GS1-4B
|
| 2 |
+
|
| 3 |
+
[English](README.md) · [简体中文](README.zh-CN.md)
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
**Fast, Faithful, Frontier.** GS1-4B 是原生类型化决策模型。输入状态、问题和运行时定义的候选含义，直接获得 **Choice、Noul、Score** 分布，生成输出 token 数为零。
|
| 8 |
+
|
| 9 |
+
[代码与优化推理](https://github.com/bigai-nlco/GS1) · [模型合集](https://huggingface.co/collections/bigai/gs1) · [完整结果](https://github.com/bigai-nlco/GS1/blob/main/benchmark/results.md)
|
| 10 |
+
|
| 11 |
+
| 指标 | GS1-4B |
|
| 12 |
+
|---|---:|
|
| 13 |
+
| 实际存储参数量 | 4,704,520,192 |
|
| 14 |
+
| 37 项公开 Decision Index | **68.97** |
|
| 15 |
+
| JevBench 公开任务准确率，231 个决策 | **86.15%** |
|
| 16 |
+
| AG News 测试准确率，7,600 个决策 | **92.01%** |
|
| 17 |
+
| ToolACE 选择准确率，640 个决策 | **94.22%** |
|
| 18 |
+
| CUDA Graph 完整热决策 P50 / P95 | **35.17 / 35.24 ms** |
|
| 19 |
+
| 上述延迟设置的峰值已分配显存 | **23.68 GiB** |
|
| 20 |
+
|
| 21 |
+
Decision Index 来自公开 edition 0.3 的完整评测：115,479 个唯一请求、279,333 个决策、最终覆盖率 100%，采用 BF16 原生参考计算；该分数与私有 Full 分数分开。三项常规榜单于 2026 年 11 月 9 日完成，均无失败、无重试。JevBench 公开任务准确率不等同于密封榜单综合分；ToolACE 是固定哈希子集的首个工具/不调用工具选择，不评估参数生成。
|
| 22 |
+
|
| 23 |
+
延迟于 2026 年 11 月 9 日测量：报告 48 GiB 的 RTX 4090、BF16 计算、最长分支 256 token、单问题、单 GPU，预热五次、测量 30 次。进程内完整计时包含校验、编译、模型计算和概率，排除加载、图捕获、HTTP 与 SDK 传输。[决策指标](assets/results/public-evaluation.json) · [原始延迟样本](assets/results/inference-latency.json) · [测量方法和全部型号](https://github.com/bigai-nlco/GS1/blob/main/benchmark/latency.md)
|
| 24 |
+
|
| 25 |
+
## Hugging Face 默认推理
|
| 26 |
+
|
| 27 |
+
**不需要安装或导入 GS1 仓库。** 使用 Python 3.11+，安装本目录的依赖：
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
python -m pip install -r requirements.txt
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
```python
|
| 34 |
+
from transformers import AutoModel
|
| 35 |
+
|
| 36 |
+
model = AutoModel.from_pretrained(
|
| 37 |
+
"bigai/GS1-4B", # a local GS1-4B directory also works
|
| 38 |
+
trust_remote_code=True,
|
| 39 |
+
).eval().to("cuda")
|
| 40 |
+
|
| 41 |
+
result = model.decide(
|
| 42 |
+
state="A customer was charged twice and asks for a refund.",
|
| 43 |
+
questions={
|
| 44 |
+
"team": {
|
| 45 |
+
"type": "choice",
|
| 46 |
+
"instructions": "Choose the team that should handle this ticket.",
|
| 47 |
+
"criteria": {"billing": "Payments and refunds", "technical": "Software problems"},
|
| 48 |
+
},
|
| 49 |
+
"refund": {"type": "noul", "instructions": "Does the customer request a refund?"},
|
| 50 |
+
"urgency": {
|
| 51 |
+
"type": "score",
|
| 52 |
+
"instructions": "Rate the urgency of handling this ticket.",
|
| 53 |
+
"criteria": ["Routine", "Soon", "Immediate"],
|
| 54 |
+
},
|
| 55 |
+
},
|
| 56 |
+
)
|
| 57 |
+
print(result["answers"])
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
上传前也可以将模型 ID 替换为本地完整目录。`trust_remote_code=True` 加载本目录提供的模型代码。默认保留原始混合存储精度，复现测量时不要整体转换权重 dtype。兼容的 CPU PyTorch 环境可使用 `.to("cpu")` 和 `decide(..., precision="fp32")`。默认 eager 路径可选安装上游 CUDA 算子；实测环境使用 `flash-linear-attention[cuda]==0.5.2` 和 `causal-conv1d==1.7.0`。
|
| 61 |
+
|
| 62 |
+
返回值包含 `model`、`answers` 和 `usage`。Choice 返回候选 ID、所有候选概率及 confidence；Noul 返回 `p(true)`；Score 返回从零开始的量表等级期望及各等级概率。输出 token 用量为零。默认实现逐问题执行完整分支，不依赖运行时资格文件或图捕获。
|
| 63 |
+
|
| 64 |
+
## GS1 优化推理
|
| 65 |
+
|
| 66 |
+
GS1 仓库为相同权重提供源码绑定的 CUDA Graph、兼容批处理、状态共享、HTTP 服务和 playground。
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
git clone https://github.com/bigai-nlco/GS1.git
|
| 70 |
+
cd GS1
|
| 71 |
+
uv sync --locked --extra serve --extra cuda
|
| 72 |
+
uv run gs1 serve --model bigai/GS1-4B --model-name GS1
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
在源码目录中额外安装 `--extra eval`，再针对本机模型、实现、硬件和请求形状执行图资格验证：
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
uv run python -m benchmark.inference --model bigai/GS1-4B \
|
| 79 |
+
--out runs/runtime.json --runtime-profile-out runs/profile.json \
|
| 80 |
+
--graph-cache-mb 16384 --graph-cache-entries 128
|
| 81 |
+
|
| 82 |
+
uv run gs1 serve --model bigai/GS1-4B --model-name GS1 \
|
| 83 |
+
--runtime-profile runs/profile.json --experimental-cuda-graphs \
|
| 84 |
+
--graph-warmup-requests runs/profile-warmup.json \
|
| 85 |
+
--graph-cache-mb 4096 --graph-cache-entries 128
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
benchmark 同时生成 `profile-warmup.json`。服务启动时预热已知图；图未命中回退原生 eager，未验证形状回退单分支参考计算。代码、模型、精度或硬件改变后需重新验证；资格文件留在本地，不作为权重上传。[完整推理指南](https://github.com/bigai-nlco/GS1/blob/main/docs/inference.zh-CN.md)
|
| 89 |
+
|
| 90 |
+
## 架构与数值策略
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
GS1-4B 的隐藏宽度为 2,560，包含 32 个解码层与 10 个决策注意力头。架构结合 Qwen3.5 混合 GDN/注意力骨干、分段双向全注意力、保持残差累加的 Attention Residuals，以及两层候选路由、256 维最终比较空间的原生交叉注意力决策头。不包含语言生成输出头，候选含义在推理时提供，无固定任务分类词表。
|
| 95 |
+
|
| 96 |
+
文本和决策头持久权重为 FP32，保留的视觉塔为 BF16；配置指定决策头采用 BF16 计算，最终 logits 和概率归约使用 FP32。完整分支上限为 65,536 token，同时受准入和显存预算约束；默认 HTTP 服务使用更紧的 4,096-token 上限。公开决策结果均为纯文本评测。`config.json` 保存原生定义、token ID 和 `inference_limits`；safetensors 参数键和 dtype 均完整保留。
|
| 97 |
+
|
| 98 |
+
## 适用范围与许可
|
| 99 |
+
|
| 100 |
+
适合路由、分类、工具选择、有限选项判断等类型化决策。概率不保证正确，校准效果依赖任务分布。公开结果也完整展示限制：HLE 准确率较低，未启用置信拒答时较大模型的 AA-Omniscience 非幻觉率为零；详见[完整组件结果](assets/results/public-evaluation.json)。
|
| 101 |
+
|
| 102 |
+
代码与发布权重采用 [Apache-2.0](LICENSE)，[上游署名](NOTICE) 保留。本目录包含模型架构和推理依赖；评测输入与本地运行时资格文件另行保存。
|
| 103 |
+
|
| 104 |
+
```bibtex
|
| 105 |
+
@techreport{zheng2026gs1,
|
| 106 |
+
title={GS1: Fast, Faithful, Frontier System One Intelligence},
|
| 107 |
+
author={Zheng, Zilong and Liu, Yang and Qin, Shuhan and Lin, Ziyong and Li, Jiaqi and Jia, Zixia and Zhu, Song-Chun},
|
| 108 |
+
institution={BIGAI},
|
| 109 |
+
year={2026}
|
| 110 |
+
}
|
| 111 |
+
```
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 150 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
compiler.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compile typed decisions into XML prompts and execute native decision branches."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
from html import escape
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from .config import INFERENCE_LIMITS, DecisionConfig
|
| 10 |
+
from .schema import DecisionRequest, candidate_ids, noul_descriptions
|
| 11 |
+
from .util import canonical
|
| 12 |
+
|
| 13 |
+
XML_STATE, XML_STATE_END = "<state>", "</state>"
|
| 14 |
+
XML_QUESTION, XML_QUESTION_END = "<question>", "</question>"
|
| 15 |
+
XML_OPTION, XML_OPTION_END = "<option>", "</option>"
|
| 16 |
+
XML_TAGS = (XML_STATE, XML_STATE_END, XML_QUESTION, XML_QUESTION_END, XML_OPTION, XML_OPTION_END)
|
| 17 |
+
DECISION_TOKEN = "<|decide|>"
|
| 18 |
+
NO_THINKING_PREFIX = "<think>\n\n</think>\n\n"
|
| 19 |
+
COMPILER = "xml_decision"
|
| 20 |
+
POSITIONS = "branch_local"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def structural_tokens():
|
| 24 |
+
return [*XML_TAGS, DECISION_TOKEN]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def no_thinking_token_ids(tokenizer):
|
| 28 |
+
ids = tuple(tokenizer.encode(NO_THINKING_PREFIX, add_special_tokens=False))
|
| 29 |
+
if (
|
| 30 |
+
not ids
|
| 31 |
+
or tokenizer.decode(ids, skip_special_tokens=False, clean_up_tokenization_spaces=False)
|
| 32 |
+
!= NO_THINKING_PREFIX
|
| 33 |
+
):
|
| 34 |
+
raise ValueError("Tokenizer cannot encode the no-thinking prefix exactly")
|
| 35 |
+
return ids
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass(frozen=True)
|
| 39 |
+
class Branch:
|
| 40 |
+
question_id: str
|
| 41 |
+
question_type: str
|
| 42 |
+
branch_id: int
|
| 43 |
+
suffix: tuple[int, ...]
|
| 44 |
+
token_roles: tuple[str, ...]
|
| 45 |
+
logical_positions: tuple[int, ...]
|
| 46 |
+
option_ends: tuple[int, ...]
|
| 47 |
+
candidates: tuple[str, ...]
|
| 48 |
+
decision_position: int
|
| 49 |
+
option_spans: tuple[tuple[int, int], ...]
|
| 50 |
+
option_starts: tuple[int, ...]
|
| 51 |
+
question_end_position: int
|
| 52 |
+
question_start_position: int
|
| 53 |
+
question_span: tuple[int, int]
|
| 54 |
+
state_evidence_start: int = 0
|
| 55 |
+
visual_inputs: dict | None = field(default=None, compare=False, repr=False)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass(frozen=True)
|
| 59 |
+
class CompiledRequest:
|
| 60 |
+
state: tuple[int, ...]
|
| 61 |
+
branches: tuple[Branch, ...]
|
| 62 |
+
question_order: tuple[str, ...]
|
| 63 |
+
candidate_order: dict[str, tuple[str, ...]]
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def input_tokens(self):
|
| 67 |
+
return len(self.state) + sum(len(b.suffix) for b in self.branches)
|
| 68 |
+
|
| 69 |
+
@property
|
| 70 |
+
def longest_path(self):
|
| 71 |
+
return max(len(self.state) + len(b.suffix) for b in self.branches)
|
| 72 |
+
|
| 73 |
+
@property
|
| 74 |
+
def query_tokens(self):
|
| 75 |
+
return len(self.branches)
|
| 76 |
+
|
| 77 |
+
@property
|
| 78 |
+
def candidate_valid_mask(self):
|
| 79 |
+
width = max(map(len, self.candidate_order.values()))
|
| 80 |
+
return tuple(
|
| 81 |
+
tuple(k < len(self.candidate_order[q]) for k in range(width)) for q in self.question_order
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class Compiler:
|
| 86 |
+
def __init__(
|
| 87 |
+
self,
|
| 88 |
+
tokenizer,
|
| 89 |
+
max_context=INFERENCE_LIMITS["max_context"],
|
| 90 |
+
max_questions=512,
|
| 91 |
+
max_request_tokens=131072,
|
| 92 |
+
decision=None,
|
| 93 |
+
max_branches=5120,
|
| 94 |
+
processor=None,
|
| 95 |
+
):
|
| 96 |
+
self.tokenizer = tokenizer
|
| 97 |
+
self.decision = DecisionConfig.model_validate(decision or {})
|
| 98 |
+
self.max_context = max_context
|
| 99 |
+
self.max_questions = max_questions
|
| 100 |
+
self.max_request_tokens = max_request_tokens
|
| 101 |
+
self.max_branches = max_branches
|
| 102 |
+
self.processor = processor
|
| 103 |
+
tokens = structural_tokens()
|
| 104 |
+
self.controls = dict(zip(tokens, tokenizer.convert_tokens_to_ids(tokens), strict=True))
|
| 105 |
+
if len(set(self.controls.values())) != len(tokens) or any(
|
| 106 |
+
type(i) is not int or i < 0 or i == tokenizer.unk_token_id or i not in tokenizer.all_special_ids
|
| 107 |
+
for i in self.controls.values()
|
| 108 |
+
):
|
| 109 |
+
raise ValueError("Tokenizer requires the seven distinct GS1 special tokens")
|
| 110 |
+
self.xml_tags = {tag: (self.controls[tag],) for tag in XML_TAGS}
|
| 111 |
+
self.xml_newline = tuple(tokenizer.encode("\n", add_special_tokens=False))
|
| 112 |
+
self.no_thinking_ids = no_thinking_token_ids(tokenizer)
|
| 113 |
+
|
| 114 |
+
@staticmethod
|
| 115 |
+
def _xml_text(value):
|
| 116 |
+
return escape(value if isinstance(value, str) else canonical(value), quote=False)
|
| 117 |
+
|
| 118 |
+
@staticmethod
|
| 119 |
+
def _visual_state(state):
|
| 120 |
+
return isinstance(state, list) and any(
|
| 121 |
+
isinstance(item, dict) and item.get("type") == "image" for item in state
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
@staticmethod
|
| 125 |
+
def choice_option_text(name, description):
|
| 126 |
+
value = name if description is None else description
|
| 127 |
+
return f"{name}: {value if isinstance(value, str) else canonical(value)}"
|
| 128 |
+
|
| 129 |
+
def _xml_positions(self, token_ids, option_count):
|
| 130 |
+
found = {
|
| 131 |
+
tag: [i for i, value in enumerate(token_ids) if value == self.controls[tag]] for tag in XML_TAGS
|
| 132 |
+
}
|
| 133 |
+
if any(len(found[tag]) != 1 for tag in XML_TAGS[:4]) or any(
|
| 134 |
+
len(found[tag]) != option_count for tag in XML_TAGS[4:]
|
| 135 |
+
):
|
| 136 |
+
raise ValueError("Tokenizer changed the XML prompt tag positions")
|
| 137 |
+
ordered = [found[tag][0] for tag in XML_TAGS[:4]]
|
| 138 |
+
ordered.extend(i for pair in zip(found[XML_OPTION], found[XML_OPTION_END], strict=True) for i in pair)
|
| 139 |
+
if ordered != sorted(set(ordered)):
|
| 140 |
+
raise ValueError("XML prompt tags are out of order")
|
| 141 |
+
prefix = found[XML_QUESTION][0]
|
| 142 |
+
|
| 143 |
+
def span(opening, closing):
|
| 144 |
+
begin, end = opening + 1, closing
|
| 145 |
+
width = len(self.xml_newline)
|
| 146 |
+
if width and tuple(token_ids[begin : begin + width]) == self.xml_newline:
|
| 147 |
+
begin += width
|
| 148 |
+
if width and tuple(token_ids[end - width : end]) == self.xml_newline:
|
| 149 |
+
end -= width
|
| 150 |
+
if begin >= end:
|
| 151 |
+
begin, end = closing, closing + 1
|
| 152 |
+
return begin - prefix, end - prefix
|
| 153 |
+
|
| 154 |
+
return dict(
|
| 155 |
+
prefix_end=prefix,
|
| 156 |
+
state_evidence_start=found[XML_STATE][0],
|
| 157 |
+
question_start_position=0,
|
| 158 |
+
question_end_position=found[XML_QUESTION_END][0] - prefix,
|
| 159 |
+
question_span=span(found[XML_QUESTION][0], found[XML_QUESTION_END][0]),
|
| 160 |
+
option_starts=tuple(i - prefix for i in found[XML_OPTION]),
|
| 161 |
+
option_ends=tuple(i - prefix for i in found[XML_OPTION_END]),
|
| 162 |
+
option_spans=tuple(
|
| 163 |
+
span(a, b) for a, b in zip(found[XML_OPTION], found[XML_OPTION_END], strict=True)
|
| 164 |
+
),
|
| 165 |
+
first_option=found[XML_OPTION][0] - prefix,
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
def _processor(self):
|
| 169 |
+
if self.processor is None:
|
| 170 |
+
raise ValueError("Visual input requires the GS1 processor")
|
| 171 |
+
if isinstance(self.processor, (str, Path)):
|
| 172 |
+
from transformers import AutoProcessor
|
| 173 |
+
|
| 174 |
+
self.processor = AutoProcessor.from_pretrained(
|
| 175 |
+
self.processor, local_files_only=True, trust_remote_code=True
|
| 176 |
+
)
|
| 177 |
+
# Fresh Qwen processor assets precede the seven GS1 structural tokens.
|
| 178 |
+
self.processor.tokenizer = self.tokenizer
|
| 179 |
+
return self.processor
|
| 180 |
+
|
| 181 |
+
def _contents(self, state):
|
| 182 |
+
if not self._visual_state(state):
|
| 183 |
+
return f"{XML_STATE}\n{self._xml_text(state)}\n{XML_STATE_END}"
|
| 184 |
+
contents = [{"type": "text", "text": f"{XML_STATE}\n"}]
|
| 185 |
+
for item in state:
|
| 186 |
+
if item.get("type") == "text" and isinstance(item.get("text"), str):
|
| 187 |
+
contents.append({"type": "text", "text": self._xml_text(item["text"])})
|
| 188 |
+
elif (
|
| 189 |
+
set(item) == {"type", "image"}
|
| 190 |
+
and isinstance(item.get("image"), str)
|
| 191 |
+
and item["image"].startswith("data:image/")
|
| 192 |
+
and ";base64," in item["image"]
|
| 193 |
+
):
|
| 194 |
+
contents.append(item)
|
| 195 |
+
else:
|
| 196 |
+
raise ValueError("Vision state requires text and inline base64 images")
|
| 197 |
+
return [*contents, {"type": "text", "text": f"\n{XML_STATE_END}"}]
|
| 198 |
+
|
| 199 |
+
def compile(self, request: DecisionRequest):
|
| 200 |
+
if len(request.questions) > self.max_questions:
|
| 201 |
+
raise ValueError(f"questions exceeds {self.max_questions}")
|
| 202 |
+
visual = self._visual_state(request.state)
|
| 203 |
+
contents = self._contents(request.state)
|
| 204 |
+
template = self._processor() if visual else self.tokenizer
|
| 205 |
+
branches, candidates, state = [], {}, None
|
| 206 |
+
for qid, question in request.questions.items():
|
| 207 |
+
ids = tuple(candidate_ids(question))
|
| 208 |
+
candidates[qid] = ids
|
| 209 |
+
if question.type == "choice":
|
| 210 |
+
options = [self.choice_option_text(k, d) for k, d in question.criteria.items()]
|
| 211 |
+
elif question.type == "noul":
|
| 212 |
+
options = list(noul_descriptions(question))
|
| 213 |
+
else:
|
| 214 |
+
options = list(question.criteria)
|
| 215 |
+
prompt = (
|
| 216 |
+
f"\n{XML_QUESTION}\n{self._xml_text(question.instructions)}\n{XML_QUESTION_END}"
|
| 217 |
+
+ "".join(f"\n{XML_OPTION}\n{self._xml_text(o)}\n{XML_OPTION_END}" for o in options)
|
| 218 |
+
)
|
| 219 |
+
content = [*contents, {"type": "text", "text": prompt}] if visual else contents + prompt
|
| 220 |
+
rendered = template.apply_chat_template(
|
| 221 |
+
[{"role": "user", "content": content}],
|
| 222 |
+
add_generation_prompt=True,
|
| 223 |
+
enable_thinking=False,
|
| 224 |
+
tokenize=False,
|
| 225 |
+
)
|
| 226 |
+
payload = None
|
| 227 |
+
if visual:
|
| 228 |
+
encoded = template(
|
| 229 |
+
text=rendered,
|
| 230 |
+
images=[[i["image"] for i in request.state if i["type"] == "image"]],
|
| 231 |
+
return_tensors="pt",
|
| 232 |
+
)
|
| 233 |
+
token_ids = encoded["input_ids"][0].tolist()
|
| 234 |
+
else:
|
| 235 |
+
token_ids = self.tokenizer.encode(rendered, add_special_tokens=False)
|
| 236 |
+
if tuple(token_ids[-len(self.no_thinking_ids) :]) != self.no_thinking_ids:
|
| 237 |
+
raise ValueError("Chat template changed the no-thinking assistant prefix")
|
| 238 |
+
token_ids.append(self.controls[DECISION_TOKEN])
|
| 239 |
+
markers = self._xml_positions(token_ids, len(ids))
|
| 240 |
+
prefix = markers.pop("prefix_end")
|
| 241 |
+
first_option = markers.pop("first_option")
|
| 242 |
+
current_state = tuple(token_ids[:prefix])
|
| 243 |
+
if state is None:
|
| 244 |
+
state = current_state
|
| 245 |
+
elif state != current_state:
|
| 246 |
+
raise ValueError("State prefix differs across question branches")
|
| 247 |
+
suffix = tuple(token_ids[prefix:])
|
| 248 |
+
if visual:
|
| 249 |
+
payload = {
|
| 250 |
+
k: v
|
| 251 |
+
for k, v in encoded.items()
|
| 252 |
+
if k
|
| 253 |
+
in {"input_ids", "attention_mask", "pixel_values", "image_grid_thw", "mm_token_type_ids"}
|
| 254 |
+
}
|
| 255 |
+
payload["input_ids"] = torch.tensor([token_ids], dtype=torch.long)
|
| 256 |
+
for key, fill in (("attention_mask", 1), ("mm_token_type_ids", 0)):
|
| 257 |
+
if key in payload:
|
| 258 |
+
value = payload[key]
|
| 259 |
+
payload[key] = torch.cat((value, torch.full((1, 1), fill, dtype=value.dtype)), dim=-1)
|
| 260 |
+
branches.append(
|
| 261 |
+
Branch(
|
| 262 |
+
qid,
|
| 263 |
+
question.type,
|
| 264 |
+
len(branches),
|
| 265 |
+
suffix,
|
| 266 |
+
tuple(
|
| 267 |
+
["question_context"] * first_option
|
| 268 |
+
+ ["candidate_context"] * (len(suffix) - first_option - 1)
|
| 269 |
+
+ ["decision_head_query"]
|
| 270 |
+
),
|
| 271 |
+
tuple(range(prefix, len(token_ids))),
|
| 272 |
+
markers.pop("option_ends"),
|
| 273 |
+
ids,
|
| 274 |
+
len(suffix) - 1,
|
| 275 |
+
**markers,
|
| 276 |
+
visual_inputs=payload,
|
| 277 |
+
)
|
| 278 |
+
)
|
| 279 |
+
result = CompiledRequest(state, tuple(branches), tuple(request.questions), candidates)
|
| 280 |
+
if result.longest_path > self.max_context:
|
| 281 |
+
raise ValueError(f"Branch exceeds {self.max_context} tokens; no truncation")
|
| 282 |
+
if result.input_tokens > self.max_request_tokens:
|
| 283 |
+
raise ValueError(f"Request exceeds {self.max_request_tokens} tokens; no truncation")
|
| 284 |
+
if len(branches) > self.max_branches:
|
| 285 |
+
raise ValueError(f"Physical branches exceed {self.max_branches}")
|
| 286 |
+
return result
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
@dataclass
|
| 290 |
+
class PreparedBatch:
|
| 291 |
+
ids: torch.Tensor
|
| 292 |
+
positions: torch.Tensor
|
| 293 |
+
valid: torch.Tensor
|
| 294 |
+
roles: torch.Tensor
|
| 295 |
+
branches: tuple
|
| 296 |
+
offsets: tuple
|
| 297 |
+
starts: tuple
|
| 298 |
+
layout: tuple
|
| 299 |
+
|
| 300 |
+
def forward(self, model, *, readout=None, execution_policy=None):
|
| 301 |
+
hidden = model.language_model(
|
| 302 |
+
inputs_embeds=model.get_input_embeddings()(self.ids),
|
| 303 |
+
position_ids=self.positions,
|
| 304 |
+
terminal_query=True,
|
| 305 |
+
prepared_layout=self.layout,
|
| 306 |
+
execution_policy=execution_policy,
|
| 307 |
+
).last_hidden_state
|
| 308 |
+
values = [
|
| 309 |
+
(readout or model.readout)(hidden[row, begin:], branch, offset)
|
| 310 |
+
for row, (begin, branch, offset) in enumerate(
|
| 311 |
+
zip(self.starts, self.branches, self.offsets, strict=True)
|
| 312 |
+
)
|
| 313 |
+
]
|
| 314 |
+
return values
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def prepare_batch(model, rows, pad_token_id=0):
|
| 318 |
+
"""Collate text once; no scalar device reads remain in PreparedBatch.forward."""
|
| 319 |
+
if not rows or any(branch.visual_inputs is not None for _, branch in rows):
|
| 320 |
+
raise ValueError("Prepared GS1 batches require nonempty text branches")
|
| 321 |
+
device = model.get_input_embeddings().weight.device
|
| 322 |
+
lengths = [len(state) + len(branch.suffix) for state, branch in rows]
|
| 323 |
+
width = max(lengths)
|
| 324 |
+
ids = torch.full((len(rows), width), pad_token_id, dtype=torch.long, device=device)
|
| 325 |
+
valid = torch.zeros_like(ids, dtype=torch.bool)
|
| 326 |
+
positions = torch.zeros_like(ids)
|
| 327 |
+
roles = torch.full_like(ids, -1)
|
| 328 |
+
starts = []
|
| 329 |
+
for row, ((state, branch), length) in enumerate(zip(rows, lengths, strict=True)):
|
| 330 |
+
begin, offset = width - length, len(state)
|
| 331 |
+
starts.append(begin)
|
| 332 |
+
ids[row, begin:] = torch.tensor(state + branch.suffix, device=device)
|
| 333 |
+
valid[row, begin:] = True
|
| 334 |
+
positions[row, begin:] = torch.arange(length, device=device)
|
| 335 |
+
roles[row, begin : begin + offset] = 0
|
| 336 |
+
roles[row, begin + offset :] = torch.tensor(
|
| 337 |
+
[
|
| 338 |
+
1 if role == "question_context" else 2 if role == "candidate_context" else 3
|
| 339 |
+
for role in branch.token_roles
|
| 340 |
+
],
|
| 341 |
+
device=device,
|
| 342 |
+
)
|
| 343 |
+
hidden = model.get_input_embeddings()(ids)
|
| 344 |
+
layout = model.language_model.prepare_encode(
|
| 345 |
+
hidden, positions, roles, terminal_query=True, attention_mask=valid
|
| 346 |
+
)
|
| 347 |
+
return PreparedBatch(
|
| 348 |
+
ids,
|
| 349 |
+
positions,
|
| 350 |
+
valid,
|
| 351 |
+
roles,
|
| 352 |
+
tuple(b for _, b in rows),
|
| 353 |
+
tuple(len(s) for s, _ in rows),
|
| 354 |
+
tuple(starts),
|
| 355 |
+
layout,
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def branch_logits(model, rows, *, pad_token_id=0, readout=None, execution_policy=None):
|
| 360 |
+
"""Execute text branches in independent, genuinely batched left-padded rows."""
|
| 361 |
+
if not rows:
|
| 362 |
+
return []
|
| 363 |
+
device = model.get_input_embeddings().weight.device
|
| 364 |
+
outputs = [None] * len(rows)
|
| 365 |
+
text = [
|
| 366 |
+
(index, state, branch) for index, (state, branch) in enumerate(rows) if branch.visual_inputs is None
|
| 367 |
+
]
|
| 368 |
+
if text:
|
| 369 |
+
prepared = prepare_batch(model, [(state, branch) for _, state, branch in text], pad_token_id)
|
| 370 |
+
logits = prepared.forward(model, readout=readout, execution_policy=execution_policy)
|
| 371 |
+
for row, (index, _, branch) in enumerate(text):
|
| 372 |
+
outputs[index] = logits[row]
|
| 373 |
+
for index, (state, branch) in enumerate(rows):
|
| 374 |
+
if branch.visual_inputs is not None:
|
| 375 |
+
payload = {
|
| 376 |
+
key: value.to(device) if isinstance(value, torch.Tensor) else value
|
| 377 |
+
for key, value in branch.visual_inputs.items()
|
| 378 |
+
}
|
| 379 |
+
outputs[index] = model(
|
| 380 |
+
**payload, branch=branch, state_length=len(state), readout_callback=readout
|
| 381 |
+
).logits
|
| 382 |
+
return outputs
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def assemble_logits(compiled, outputs):
|
| 386 |
+
if len(outputs) != len(compiled.branches):
|
| 387 |
+
raise ValueError("Missing physical branch results")
|
| 388 |
+
result = {}
|
| 389 |
+
for branch, output in zip(compiled.branches, outputs, strict=True):
|
| 390 |
+
if output.shape != (len(branch.candidates),) or branch.question_id in result:
|
| 391 |
+
raise ValueError("Malformed or duplicate single-decision result")
|
| 392 |
+
result[branch.question_id] = output
|
| 393 |
+
if set(result) != set(compiled.question_order):
|
| 394 |
+
raise ValueError("Missing logical question results")
|
| 395 |
+
return {key: result[key] for key in compiled.question_order}
|
config.json
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GS1Model"
|
| 4 |
+
],
|
| 5 |
+
"calibration": {
|
| 6 |
+
"log_temperature_bound": 2.995732273553991,
|
| 7 |
+
"mode": "identity"
|
| 8 |
+
},
|
| 9 |
+
"decision": {
|
| 10 |
+
"attnres": true,
|
| 11 |
+
"attnres_eps": 1e-06,
|
| 12 |
+
"attnres_layers": [
|
| 13 |
+
3,
|
| 14 |
+
7,
|
| 15 |
+
11,
|
| 16 |
+
15,
|
| 17 |
+
19,
|
| 18 |
+
23,
|
| 19 |
+
27,
|
| 20 |
+
31
|
| 21 |
+
],
|
| 22 |
+
"bidirectional": true,
|
| 23 |
+
"bidirectional_layers": [
|
| 24 |
+
3,
|
| 25 |
+
7,
|
| 26 |
+
11,
|
| 27 |
+
15,
|
| 28 |
+
19,
|
| 29 |
+
23,
|
| 30 |
+
27,
|
| 31 |
+
31
|
| 32 |
+
],
|
| 33 |
+
"cross_attention_heads": 10,
|
| 34 |
+
"cross_attention_layers": 2,
|
| 35 |
+
"norm_eps": 1e-05,
|
| 36 |
+
"projection_size": 256
|
| 37 |
+
},
|
| 38 |
+
"dtype": "float32",
|
| 39 |
+
"head_compute_dtype": "bfloat16",
|
| 40 |
+
"image_token_id": 248056,
|
| 41 |
+
"initialization_seed": 41,
|
| 42 |
+
"model_type": "gs1",
|
| 43 |
+
"structural_token_ids": {
|
| 44 |
+
"</option>": 248082,
|
| 45 |
+
"</question>": 248080,
|
| 46 |
+
"</state>": 248078,
|
| 47 |
+
"<option>": 248081,
|
| 48 |
+
"<question>": 248079,
|
| 49 |
+
"<state>": 248077,
|
| 50 |
+
"<|decide|>": 248083
|
| 51 |
+
},
|
| 52 |
+
"text_config": {
|
| 53 |
+
"attention_bias": false,
|
| 54 |
+
"attention_dropout": 0.0,
|
| 55 |
+
"attn_output_gate": true,
|
| 56 |
+
"bos_token_id": null,
|
| 57 |
+
"dtype": "float32",
|
| 58 |
+
"eos_token_id": 248044,
|
| 59 |
+
"full_attention_interval": 4,
|
| 60 |
+
"head_dim": 256,
|
| 61 |
+
"hidden_act": "silu",
|
| 62 |
+
"hidden_size": 2560,
|
| 63 |
+
"initializer_range": 0.02,
|
| 64 |
+
"intermediate_size": 9216,
|
| 65 |
+
"layer_types": [
|
| 66 |
+
"linear_attention",
|
| 67 |
+
"linear_attention",
|
| 68 |
+
"linear_attention",
|
| 69 |
+
"full_attention",
|
| 70 |
+
"linear_attention",
|
| 71 |
+
"linear_attention",
|
| 72 |
+
"linear_attention",
|
| 73 |
+
"full_attention",
|
| 74 |
+
"linear_attention",
|
| 75 |
+
"linear_attention",
|
| 76 |
+
"linear_attention",
|
| 77 |
+
"full_attention",
|
| 78 |
+
"linear_attention",
|
| 79 |
+
"linear_attention",
|
| 80 |
+
"linear_attention",
|
| 81 |
+
"full_attention",
|
| 82 |
+
"linear_attention",
|
| 83 |
+
"linear_attention",
|
| 84 |
+
"linear_attention",
|
| 85 |
+
"full_attention",
|
| 86 |
+
"linear_attention",
|
| 87 |
+
"linear_attention",
|
| 88 |
+
"linear_attention",
|
| 89 |
+
"full_attention",
|
| 90 |
+
"linear_attention",
|
| 91 |
+
"linear_attention",
|
| 92 |
+
"linear_attention",
|
| 93 |
+
"full_attention",
|
| 94 |
+
"linear_attention",
|
| 95 |
+
"linear_attention",
|
| 96 |
+
"linear_attention",
|
| 97 |
+
"full_attention"
|
| 98 |
+
],
|
| 99 |
+
"linear_conv_kernel_dim": 4,
|
| 100 |
+
"linear_key_head_dim": 128,
|
| 101 |
+
"linear_num_key_heads": 16,
|
| 102 |
+
"linear_num_value_heads": 32,
|
| 103 |
+
"linear_value_head_dim": 128,
|
| 104 |
+
"mamba_ssm_dtype": "float32",
|
| 105 |
+
"max_position_embeddings": 262144,
|
| 106 |
+
"mlp_only_layers": [],
|
| 107 |
+
"model_type": "qwen3_5_text",
|
| 108 |
+
"mtp_num_hidden_layers": 1,
|
| 109 |
+
"mtp_use_dedicated_embeddings": false,
|
| 110 |
+
"num_attention_heads": 16,
|
| 111 |
+
"num_hidden_layers": 32,
|
| 112 |
+
"num_key_value_heads": 4,
|
| 113 |
+
"pad_token_id": null,
|
| 114 |
+
"partial_rotary_factor": 0.25,
|
| 115 |
+
"rms_norm_eps": 1e-06,
|
| 116 |
+
"rope_parameters": {
|
| 117 |
+
"mrope_interleaved": true,
|
| 118 |
+
"mrope_section": [
|
| 119 |
+
11,
|
| 120 |
+
11,
|
| 121 |
+
10
|
| 122 |
+
],
|
| 123 |
+
"partial_rotary_factor": 0.25,
|
| 124 |
+
"rope_theta": 10000000,
|
| 125 |
+
"rope_type": "default"
|
| 126 |
+
},
|
| 127 |
+
"tie_word_embeddings": true,
|
| 128 |
+
"use_cache": false,
|
| 129 |
+
"vocab_size": 248320
|
| 130 |
+
},
|
| 131 |
+
"tie_word_embeddings": false,
|
| 132 |
+
"transformers_version": "5.3.0",
|
| 133 |
+
"video_token_id": 248057,
|
| 134 |
+
"vision_config": {
|
| 135 |
+
"deepstack_visual_indexes": [],
|
| 136 |
+
"depth": 24,
|
| 137 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 138 |
+
"hidden_size": 1024,
|
| 139 |
+
"in_channels": 3,
|
| 140 |
+
"initializer_range": 0.02,
|
| 141 |
+
"intermediate_size": 4096,
|
| 142 |
+
"model_type": "qwen3_5",
|
| 143 |
+
"num_heads": 16,
|
| 144 |
+
"num_position_embeddings": 2304,
|
| 145 |
+
"out_hidden_size": 2560,
|
| 146 |
+
"patch_size": 16,
|
| 147 |
+
"spatial_merge_size": 2,
|
| 148 |
+
"temporal_patch_size": 2
|
| 149 |
+
},
|
| 150 |
+
"vision_end_token_id": 248054,
|
| 151 |
+
"vision_start_token_id": 248053,
|
| 152 |
+
"auto_map": {
|
| 153 |
+
"AutoConfig": "modular_gs1.GS1Config",
|
| 154 |
+
"AutoModel": "modular_gs1.GS1Model"
|
| 155 |
+
},
|
| 156 |
+
"inference_limits": {
|
| 157 |
+
"max_context": 65536,
|
| 158 |
+
"max_questions": 512,
|
| 159 |
+
"max_request_tokens": 131072,
|
| 160 |
+
"max_branches": 5120,
|
| 161 |
+
"batch_tokens": 32768,
|
| 162 |
+
"max_attention_bytes": 549755813888,
|
| 163 |
+
"max_logit_elements": 16777216
|
| 164 |
+
},
|
| 165 |
+
"release_date": "2026-11-07"
|
| 166 |
+
}
|
config.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Semantic architecture and scalar calibration settings for GS1."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from typing import Literal
|
| 5 |
+
|
| 6 |
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
| 7 |
+
|
| 8 |
+
INFERENCE_LIMITS = dict(
|
| 9 |
+
max_context=65536,
|
| 10 |
+
max_questions=512,
|
| 11 |
+
max_request_tokens=131072,
|
| 12 |
+
max_branches=5120,
|
| 13 |
+
batch_tokens=16384,
|
| 14 |
+
max_attention_bytes=1073741824,
|
| 15 |
+
max_logit_elements=16777216,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class ModelConfig(BaseModel):
|
| 20 |
+
model_config = ConfigDict(extra="forbid", strict=True, allow_inf_nan=False)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class DecisionConfig(ModelConfig):
|
| 24 |
+
attnres: bool = True
|
| 25 |
+
attnres_layers: list[int] | None = None
|
| 26 |
+
attnres_eps: float = Field(default=1e-6, gt=0)
|
| 27 |
+
bidirectional: bool = True
|
| 28 |
+
bidirectional_layers: list[int] | None = None
|
| 29 |
+
cross_attention_heads: int = Field(default=8, ge=1)
|
| 30 |
+
cross_attention_layers: int = Field(default=2, ge=1)
|
| 31 |
+
projection_size: int = Field(default=256, ge=2)
|
| 32 |
+
norm_eps: float = Field(default=1e-5, gt=0)
|
| 33 |
+
|
| 34 |
+
@field_validator("attnres_layers", "bidirectional_layers")
|
| 35 |
+
@classmethod
|
| 36 |
+
def selected_layers(cls, value):
|
| 37 |
+
if value is not None:
|
| 38 |
+
if not value or any(i < 0 for i in value) or len(set(value)) != len(value):
|
| 39 |
+
raise ValueError("Layer selection must contain distinct nonnegative indices")
|
| 40 |
+
return sorted(value)
|
| 41 |
+
return value
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class CalibrationConfig(ModelConfig):
|
| 45 |
+
mode: Literal["identity", "scalar_temperature"] = "identity"
|
| 46 |
+
log_temperature_bound: float = Field(default=math.log(20), gt=0)
|
model-00001-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e6b178bb1351074751694f32f537caadfb84898e709f40c6237435cef386f19b
|
| 3 |
+
size 3992260304
|
model-00002-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a45fee1c47f454be7557d8be95fbf15e42ccdbbb5e6db75fb59ea2eee26d4c56
|
| 3 |
+
size 3947705992
|
model-00003-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dfb5138f5492585ca6075ddee1604d6f75f3ee661c5f69c96ebfb486a1cb872c
|
| 3 |
+
size 3979816592
|
model-00004-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6503afc9474759ab1f4087010f35b34744ae91f34e332e2f72ef3e5b3e00a98b
|
| 3 |
+
size 3927234768
|
model-00005-of-00005.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2aa92e905829fdfae4e25dabc68b8341e1e6b3fce15ffdf2095bde9eb18b6be0
|
| 3 |
+
size 2304125896
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,792 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_parameters": 4704520192,
|
| 4 |
+
"total_size": 18151052288
|
| 5 |
+
},
|
| 6 |
+
"weight_map": {
|
| 7 |
+
"decision_head.condition_projection.weight": "model-00005-of-00005.safetensors",
|
| 8 |
+
"decision_head.context_norm.bias": "model-00005-of-00005.safetensors",
|
| 9 |
+
"decision_head.context_norm.weight": "model-00005-of-00005.safetensors",
|
| 10 |
+
"decision_head.decision_projection.weight": "model-00005-of-00005.safetensors",
|
| 11 |
+
"decision_head.key_mlp.0.bias": "model-00005-of-00005.safetensors",
|
| 12 |
+
"decision_head.key_mlp.0.weight": "model-00005-of-00005.safetensors",
|
| 13 |
+
"decision_head.key_mlp.2.bias": "model-00005-of-00005.safetensors",
|
| 14 |
+
"decision_head.key_mlp.2.weight": "model-00005-of-00005.safetensors",
|
| 15 |
+
"decision_head.key_norm.bias": "model-00005-of-00005.safetensors",
|
| 16 |
+
"decision_head.key_norm.weight": "model-00005-of-00005.safetensors",
|
| 17 |
+
"decision_head.memory_norm.bias": "model-00005-of-00005.safetensors",
|
| 18 |
+
"decision_head.memory_norm.weight": "model-00005-of-00005.safetensors",
|
| 19 |
+
"decision_head.memory_projection.weight": "model-00005-of-00005.safetensors",
|
| 20 |
+
"decision_head.option_norm.bias": "model-00005-of-00005.safetensors",
|
| 21 |
+
"decision_head.option_norm.weight": "model-00005-of-00005.safetensors",
|
| 22 |
+
"decision_head.option_projection.weight": "model-00005-of-00005.safetensors",
|
| 23 |
+
"decision_head.query_mlp.0.bias": "model-00005-of-00005.safetensors",
|
| 24 |
+
"decision_head.query_mlp.0.weight": "model-00005-of-00005.safetensors",
|
| 25 |
+
"decision_head.query_mlp.2.bias": "model-00005-of-00005.safetensors",
|
| 26 |
+
"decision_head.query_mlp.2.weight": "model-00005-of-00005.safetensors",
|
| 27 |
+
"decision_head.query_norm.bias": "model-00005-of-00005.safetensors",
|
| 28 |
+
"decision_head.query_norm.weight": "model-00005-of-00005.safetensors",
|
| 29 |
+
"decision_head.question_attention.in_proj_bias": "model-00005-of-00005.safetensors",
|
| 30 |
+
"decision_head.question_attention.in_proj_weight": "model-00005-of-00005.safetensors",
|
| 31 |
+
"decision_head.question_attention.out_proj.bias": "model-00005-of-00005.safetensors",
|
| 32 |
+
"decision_head.question_attention.out_proj.weight": "model-00005-of-00005.safetensors",
|
| 33 |
+
"decision_head.question_evidence_projection.weight": "model-00005-of-00005.safetensors",
|
| 34 |
+
"decision_head.question_projection.weight": "model-00005-of-00005.safetensors",
|
| 35 |
+
"decision_head.question_seed.weight": "model-00005-of-00005.safetensors",
|
| 36 |
+
"decision_head.routing_attention.0.in_proj_bias": "model-00005-of-00005.safetensors",
|
| 37 |
+
"decision_head.routing_attention.0.in_proj_weight": "model-00005-of-00005.safetensors",
|
| 38 |
+
"decision_head.routing_attention.0.out_proj.bias": "model-00005-of-00005.safetensors",
|
| 39 |
+
"decision_head.routing_attention.0.out_proj.weight": "model-00005-of-00005.safetensors",
|
| 40 |
+
"decision_head.routing_attention.1.in_proj_bias": "model-00005-of-00005.safetensors",
|
| 41 |
+
"decision_head.routing_attention.1.in_proj_weight": "model-00005-of-00005.safetensors",
|
| 42 |
+
"decision_head.routing_attention.1.out_proj.bias": "model-00005-of-00005.safetensors",
|
| 43 |
+
"decision_head.routing_attention.1.out_proj.weight": "model-00005-of-00005.safetensors",
|
| 44 |
+
"decision_head.routing_attention_norm.0.bias": "model-00005-of-00005.safetensors",
|
| 45 |
+
"decision_head.routing_attention_norm.0.weight": "model-00005-of-00005.safetensors",
|
| 46 |
+
"decision_head.routing_attention_norm.1.bias": "model-00005-of-00005.safetensors",
|
| 47 |
+
"decision_head.routing_attention_norm.1.weight": "model-00005-of-00005.safetensors",
|
| 48 |
+
"decision_head.routing_ffn.0.0.bias": "model-00005-of-00005.safetensors",
|
| 49 |
+
"decision_head.routing_ffn.0.0.weight": "model-00005-of-00005.safetensors",
|
| 50 |
+
"decision_head.routing_ffn.0.2.bias": "model-00005-of-00005.safetensors",
|
| 51 |
+
"decision_head.routing_ffn.0.2.weight": "model-00005-of-00005.safetensors",
|
| 52 |
+
"decision_head.routing_ffn.1.0.bias": "model-00005-of-00005.safetensors",
|
| 53 |
+
"decision_head.routing_ffn.1.0.weight": "model-00005-of-00005.safetensors",
|
| 54 |
+
"decision_head.routing_ffn.1.2.bias": "model-00005-of-00005.safetensors",
|
| 55 |
+
"decision_head.routing_ffn.1.2.weight": "model-00005-of-00005.safetensors",
|
| 56 |
+
"decision_head.routing_output_norm.0.bias": "model-00005-of-00005.safetensors",
|
| 57 |
+
"decision_head.routing_output_norm.0.weight": "model-00005-of-00005.safetensors",
|
| 58 |
+
"decision_head.routing_output_norm.1.bias": "model-00005-of-00005.safetensors",
|
| 59 |
+
"decision_head.routing_output_norm.1.weight": "model-00005-of-00005.safetensors",
|
| 60 |
+
"language_model.attnres_routers.11.query": "model-00005-of-00005.safetensors",
|
| 61 |
+
"language_model.attnres_routers.15.query": "model-00005-of-00005.safetensors",
|
| 62 |
+
"language_model.attnres_routers.19.query": "model-00005-of-00005.safetensors",
|
| 63 |
+
"language_model.attnres_routers.23.query": "model-00005-of-00005.safetensors",
|
| 64 |
+
"language_model.attnres_routers.27.query": "model-00005-of-00005.safetensors",
|
| 65 |
+
"language_model.attnres_routers.3.query": "model-00005-of-00005.safetensors",
|
| 66 |
+
"language_model.attnres_routers.31.query": "model-00005-of-00005.safetensors",
|
| 67 |
+
"language_model.attnres_routers.7.query": "model-00005-of-00005.safetensors",
|
| 68 |
+
"language_model.embed_tokens.weight": "model-00001-of-00005.safetensors",
|
| 69 |
+
"language_model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 70 |
+
"language_model.layers.0.linear_attn.A_log": "model-00001-of-00005.safetensors",
|
| 71 |
+
"language_model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00005.safetensors",
|
| 72 |
+
"language_model.layers.0.linear_attn.dt_bias": "model-00001-of-00005.safetensors",
|
| 73 |
+
"language_model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00005.safetensors",
|
| 74 |
+
"language_model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00005.safetensors",
|
| 75 |
+
"language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00005.safetensors",
|
| 76 |
+
"language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00005.safetensors",
|
| 77 |
+
"language_model.layers.0.linear_attn.norm.weight": "model-00001-of-00005.safetensors",
|
| 78 |
+
"language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00005.safetensors",
|
| 79 |
+
"language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 80 |
+
"language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 81 |
+
"language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 82 |
+
"language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 83 |
+
"language_model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 84 |
+
"language_model.layers.1.linear_attn.A_log": "model-00001-of-00005.safetensors",
|
| 85 |
+
"language_model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00005.safetensors",
|
| 86 |
+
"language_model.layers.1.linear_attn.dt_bias": "model-00001-of-00005.safetensors",
|
| 87 |
+
"language_model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00005.safetensors",
|
| 88 |
+
"language_model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00005.safetensors",
|
| 89 |
+
"language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00005.safetensors",
|
| 90 |
+
"language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00005.safetensors",
|
| 91 |
+
"language_model.layers.1.linear_attn.norm.weight": "model-00001-of-00005.safetensors",
|
| 92 |
+
"language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00005.safetensors",
|
| 93 |
+
"language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 94 |
+
"language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 95 |
+
"language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 96 |
+
"language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 97 |
+
"language_model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 98 |
+
"language_model.layers.10.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 99 |
+
"language_model.layers.10.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 100 |
+
"language_model.layers.10.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 101 |
+
"language_model.layers.10.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 102 |
+
"language_model.layers.10.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 103 |
+
"language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 104 |
+
"language_model.layers.10.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 105 |
+
"language_model.layers.10.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 106 |
+
"language_model.layers.10.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 107 |
+
"language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 108 |
+
"language_model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 109 |
+
"language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 110 |
+
"language_model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 111 |
+
"language_model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 112 |
+
"language_model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 113 |
+
"language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 114 |
+
"language_model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 115 |
+
"language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 116 |
+
"language_model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 117 |
+
"language_model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 118 |
+
"language_model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 119 |
+
"language_model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 120 |
+
"language_model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 121 |
+
"language_model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 122 |
+
"language_model.layers.12.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 123 |
+
"language_model.layers.12.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 124 |
+
"language_model.layers.12.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 125 |
+
"language_model.layers.12.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 126 |
+
"language_model.layers.12.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 127 |
+
"language_model.layers.12.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 128 |
+
"language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 129 |
+
"language_model.layers.12.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 130 |
+
"language_model.layers.12.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 131 |
+
"language_model.layers.12.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 132 |
+
"language_model.layers.12.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 133 |
+
"language_model.layers.12.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 134 |
+
"language_model.layers.12.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 135 |
+
"language_model.layers.12.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 136 |
+
"language_model.layers.13.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 137 |
+
"language_model.layers.13.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 138 |
+
"language_model.layers.13.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 139 |
+
"language_model.layers.13.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 140 |
+
"language_model.layers.13.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 141 |
+
"language_model.layers.13.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 142 |
+
"language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 143 |
+
"language_model.layers.13.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 144 |
+
"language_model.layers.13.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 145 |
+
"language_model.layers.13.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 146 |
+
"language_model.layers.13.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 147 |
+
"language_model.layers.13.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 148 |
+
"language_model.layers.13.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 149 |
+
"language_model.layers.13.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 150 |
+
"language_model.layers.14.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 151 |
+
"language_model.layers.14.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 152 |
+
"language_model.layers.14.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 153 |
+
"language_model.layers.14.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 154 |
+
"language_model.layers.14.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 155 |
+
"language_model.layers.14.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 156 |
+
"language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 157 |
+
"language_model.layers.14.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 158 |
+
"language_model.layers.14.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 159 |
+
"language_model.layers.14.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 160 |
+
"language_model.layers.14.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 161 |
+
"language_model.layers.14.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 162 |
+
"language_model.layers.14.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 163 |
+
"language_model.layers.14.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 164 |
+
"language_model.layers.15.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 165 |
+
"language_model.layers.15.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 166 |
+
"language_model.layers.15.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 167 |
+
"language_model.layers.15.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 168 |
+
"language_model.layers.15.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 169 |
+
"language_model.layers.15.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 170 |
+
"language_model.layers.15.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 171 |
+
"language_model.layers.15.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 172 |
+
"language_model.layers.15.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 173 |
+
"language_model.layers.15.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 174 |
+
"language_model.layers.15.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 175 |
+
"language_model.layers.16.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 176 |
+
"language_model.layers.16.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 177 |
+
"language_model.layers.16.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 178 |
+
"language_model.layers.16.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 179 |
+
"language_model.layers.16.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 180 |
+
"language_model.layers.16.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 181 |
+
"language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 182 |
+
"language_model.layers.16.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 183 |
+
"language_model.layers.16.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 184 |
+
"language_model.layers.16.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 185 |
+
"language_model.layers.16.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 186 |
+
"language_model.layers.16.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 187 |
+
"language_model.layers.16.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 188 |
+
"language_model.layers.16.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 189 |
+
"language_model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 190 |
+
"language_model.layers.17.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 191 |
+
"language_model.layers.17.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 192 |
+
"language_model.layers.17.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 193 |
+
"language_model.layers.17.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 194 |
+
"language_model.layers.17.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 195 |
+
"language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 196 |
+
"language_model.layers.17.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 197 |
+
"language_model.layers.17.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 198 |
+
"language_model.layers.17.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 199 |
+
"language_model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 200 |
+
"language_model.layers.17.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 201 |
+
"language_model.layers.17.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 202 |
+
"language_model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 203 |
+
"language_model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 204 |
+
"language_model.layers.18.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 205 |
+
"language_model.layers.18.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 206 |
+
"language_model.layers.18.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 207 |
+
"language_model.layers.18.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 208 |
+
"language_model.layers.18.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 209 |
+
"language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 210 |
+
"language_model.layers.18.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 211 |
+
"language_model.layers.18.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 212 |
+
"language_model.layers.18.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 213 |
+
"language_model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 214 |
+
"language_model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 215 |
+
"language_model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 216 |
+
"language_model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 217 |
+
"language_model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 218 |
+
"language_model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 219 |
+
"language_model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 220 |
+
"language_model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 221 |
+
"language_model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 222 |
+
"language_model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
|
| 223 |
+
"language_model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
|
| 224 |
+
"language_model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
|
| 225 |
+
"language_model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
|
| 226 |
+
"language_model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
|
| 227 |
+
"language_model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
|
| 228 |
+
"language_model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 229 |
+
"language_model.layers.2.linear_attn.A_log": "model-00001-of-00005.safetensors",
|
| 230 |
+
"language_model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00005.safetensors",
|
| 231 |
+
"language_model.layers.2.linear_attn.dt_bias": "model-00001-of-00005.safetensors",
|
| 232 |
+
"language_model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00005.safetensors",
|
| 233 |
+
"language_model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00005.safetensors",
|
| 234 |
+
"language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00005.safetensors",
|
| 235 |
+
"language_model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00005.safetensors",
|
| 236 |
+
"language_model.layers.2.linear_attn.norm.weight": "model-00001-of-00005.safetensors",
|
| 237 |
+
"language_model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00005.safetensors",
|
| 238 |
+
"language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
|
| 239 |
+
"language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
|
| 240 |
+
"language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
|
| 241 |
+
"language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
|
| 242 |
+
"language_model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 243 |
+
"language_model.layers.20.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 244 |
+
"language_model.layers.20.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 245 |
+
"language_model.layers.20.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 246 |
+
"language_model.layers.20.linear_attn.in_proj_a.weight": "model-00003-of-00005.safetensors",
|
| 247 |
+
"language_model.layers.20.linear_attn.in_proj_b.weight": "model-00003-of-00005.safetensors",
|
| 248 |
+
"language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00003-of-00005.safetensors",
|
| 249 |
+
"language_model.layers.20.linear_attn.in_proj_z.weight": "model-00003-of-00005.safetensors",
|
| 250 |
+
"language_model.layers.20.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 251 |
+
"language_model.layers.20.linear_attn.out_proj.weight": "model-00003-of-00005.safetensors",
|
| 252 |
+
"language_model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
|
| 253 |
+
"language_model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
|
| 254 |
+
"language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
|
| 255 |
+
"language_model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
|
| 256 |
+
"language_model.layers.21.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 257 |
+
"language_model.layers.21.linear_attn.A_log": "model-00003-of-00005.safetensors",
|
| 258 |
+
"language_model.layers.21.linear_attn.conv1d.weight": "model-00003-of-00005.safetensors",
|
| 259 |
+
"language_model.layers.21.linear_attn.dt_bias": "model-00003-of-00005.safetensors",
|
| 260 |
+
"language_model.layers.21.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 261 |
+
"language_model.layers.21.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 262 |
+
"language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 263 |
+
"language_model.layers.21.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 264 |
+
"language_model.layers.21.linear_attn.norm.weight": "model-00003-of-00005.safetensors",
|
| 265 |
+
"language_model.layers.21.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 266 |
+
"language_model.layers.21.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 267 |
+
"language_model.layers.21.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 268 |
+
"language_model.layers.21.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 269 |
+
"language_model.layers.21.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 270 |
+
"language_model.layers.22.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 271 |
+
"language_model.layers.22.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 272 |
+
"language_model.layers.22.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 273 |
+
"language_model.layers.22.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 274 |
+
"language_model.layers.22.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 275 |
+
"language_model.layers.22.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 276 |
+
"language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 277 |
+
"language_model.layers.22.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 278 |
+
"language_model.layers.22.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 279 |
+
"language_model.layers.22.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 280 |
+
"language_model.layers.22.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 281 |
+
"language_model.layers.22.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 282 |
+
"language_model.layers.22.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 283 |
+
"language_model.layers.22.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 284 |
+
"language_model.layers.23.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 285 |
+
"language_model.layers.23.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 286 |
+
"language_model.layers.23.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 287 |
+
"language_model.layers.23.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 288 |
+
"language_model.layers.23.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 289 |
+
"language_model.layers.23.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 290 |
+
"language_model.layers.23.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 291 |
+
"language_model.layers.23.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 292 |
+
"language_model.layers.23.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 293 |
+
"language_model.layers.23.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 294 |
+
"language_model.layers.23.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 295 |
+
"language_model.layers.24.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 296 |
+
"language_model.layers.24.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 297 |
+
"language_model.layers.24.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 298 |
+
"language_model.layers.24.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 299 |
+
"language_model.layers.24.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 300 |
+
"language_model.layers.24.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 301 |
+
"language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 302 |
+
"language_model.layers.24.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 303 |
+
"language_model.layers.24.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 304 |
+
"language_model.layers.24.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 305 |
+
"language_model.layers.24.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 306 |
+
"language_model.layers.24.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 307 |
+
"language_model.layers.24.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 308 |
+
"language_model.layers.24.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 309 |
+
"language_model.layers.25.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 310 |
+
"language_model.layers.25.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 311 |
+
"language_model.layers.25.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 312 |
+
"language_model.layers.25.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 313 |
+
"language_model.layers.25.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 314 |
+
"language_model.layers.25.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 315 |
+
"language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 316 |
+
"language_model.layers.25.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 317 |
+
"language_model.layers.25.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 318 |
+
"language_model.layers.25.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 319 |
+
"language_model.layers.25.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 320 |
+
"language_model.layers.25.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 321 |
+
"language_model.layers.25.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 322 |
+
"language_model.layers.25.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 323 |
+
"language_model.layers.26.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 324 |
+
"language_model.layers.26.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 325 |
+
"language_model.layers.26.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 326 |
+
"language_model.layers.26.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 327 |
+
"language_model.layers.26.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 328 |
+
"language_model.layers.26.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 329 |
+
"language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 330 |
+
"language_model.layers.26.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 331 |
+
"language_model.layers.26.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 332 |
+
"language_model.layers.26.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 333 |
+
"language_model.layers.26.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 334 |
+
"language_model.layers.26.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 335 |
+
"language_model.layers.26.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 336 |
+
"language_model.layers.26.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 337 |
+
"language_model.layers.27.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 338 |
+
"language_model.layers.27.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 339 |
+
"language_model.layers.27.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 340 |
+
"language_model.layers.27.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 341 |
+
"language_model.layers.27.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 342 |
+
"language_model.layers.27.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
|
| 343 |
+
"language_model.layers.27.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
|
| 344 |
+
"language_model.layers.27.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
|
| 345 |
+
"language_model.layers.27.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
|
| 346 |
+
"language_model.layers.27.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
|
| 347 |
+
"language_model.layers.27.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
|
| 348 |
+
"language_model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 349 |
+
"language_model.layers.28.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 350 |
+
"language_model.layers.28.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 351 |
+
"language_model.layers.28.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 352 |
+
"language_model.layers.28.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 353 |
+
"language_model.layers.28.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 354 |
+
"language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 355 |
+
"language_model.layers.28.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 356 |
+
"language_model.layers.28.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 357 |
+
"language_model.layers.28.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 358 |
+
"language_model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
|
| 359 |
+
"language_model.layers.28.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 360 |
+
"language_model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 361 |
+
"language_model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
|
| 362 |
+
"language_model.layers.29.input_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 363 |
+
"language_model.layers.29.linear_attn.A_log": "model-00004-of-00005.safetensors",
|
| 364 |
+
"language_model.layers.29.linear_attn.conv1d.weight": "model-00004-of-00005.safetensors",
|
| 365 |
+
"language_model.layers.29.linear_attn.dt_bias": "model-00004-of-00005.safetensors",
|
| 366 |
+
"language_model.layers.29.linear_attn.in_proj_a.weight": "model-00004-of-00005.safetensors",
|
| 367 |
+
"language_model.layers.29.linear_attn.in_proj_b.weight": "model-00004-of-00005.safetensors",
|
| 368 |
+
"language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00004-of-00005.safetensors",
|
| 369 |
+
"language_model.layers.29.linear_attn.in_proj_z.weight": "model-00004-of-00005.safetensors",
|
| 370 |
+
"language_model.layers.29.linear_attn.norm.weight": "model-00004-of-00005.safetensors",
|
| 371 |
+
"language_model.layers.29.linear_attn.out_proj.weight": "model-00004-of-00005.safetensors",
|
| 372 |
+
"language_model.layers.29.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
|
| 373 |
+
"language_model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
|
| 374 |
+
"language_model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
|
| 375 |
+
"language_model.layers.29.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 376 |
+
"language_model.layers.3.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 377 |
+
"language_model.layers.3.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 378 |
+
"language_model.layers.3.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 379 |
+
"language_model.layers.3.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 380 |
+
"language_model.layers.3.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 381 |
+
"language_model.layers.3.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 382 |
+
"language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
|
| 383 |
+
"language_model.layers.3.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 384 |
+
"language_model.layers.3.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 385 |
+
"language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
|
| 386 |
+
"language_model.layers.3.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 387 |
+
"language_model.layers.30.input_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 388 |
+
"language_model.layers.30.linear_attn.A_log": "model-00005-of-00005.safetensors",
|
| 389 |
+
"language_model.layers.30.linear_attn.conv1d.weight": "model-00005-of-00005.safetensors",
|
| 390 |
+
"language_model.layers.30.linear_attn.dt_bias": "model-00005-of-00005.safetensors",
|
| 391 |
+
"language_model.layers.30.linear_attn.in_proj_a.weight": "model-00005-of-00005.safetensors",
|
| 392 |
+
"language_model.layers.30.linear_attn.in_proj_b.weight": "model-00005-of-00005.safetensors",
|
| 393 |
+
"language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00005-of-00005.safetensors",
|
| 394 |
+
"language_model.layers.30.linear_attn.in_proj_z.weight": "model-00005-of-00005.safetensors",
|
| 395 |
+
"language_model.layers.30.linear_attn.norm.weight": "model-00005-of-00005.safetensors",
|
| 396 |
+
"language_model.layers.30.linear_attn.out_proj.weight": "model-00005-of-00005.safetensors",
|
| 397 |
+
"language_model.layers.30.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
|
| 398 |
+
"language_model.layers.30.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
|
| 399 |
+
"language_model.layers.30.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
|
| 400 |
+
"language_model.layers.30.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 401 |
+
"language_model.layers.31.input_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 402 |
+
"language_model.layers.31.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
|
| 403 |
+
"language_model.layers.31.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
|
| 404 |
+
"language_model.layers.31.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
|
| 405 |
+
"language_model.layers.31.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
|
| 406 |
+
"language_model.layers.31.self_attn.k_norm.weight": "model-00005-of-00005.safetensors",
|
| 407 |
+
"language_model.layers.31.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
|
| 408 |
+
"language_model.layers.31.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
|
| 409 |
+
"language_model.layers.31.self_attn.q_norm.weight": "model-00005-of-00005.safetensors",
|
| 410 |
+
"language_model.layers.31.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
|
| 411 |
+
"language_model.layers.31.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
|
| 412 |
+
"language_model.layers.4.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 413 |
+
"language_model.layers.4.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 414 |
+
"language_model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 415 |
+
"language_model.layers.4.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 416 |
+
"language_model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 417 |
+
"language_model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 418 |
+
"language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 419 |
+
"language_model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 420 |
+
"language_model.layers.4.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 421 |
+
"language_model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 422 |
+
"language_model.layers.4.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 423 |
+
"language_model.layers.4.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 424 |
+
"language_model.layers.4.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 425 |
+
"language_model.layers.4.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 426 |
+
"language_model.layers.5.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 427 |
+
"language_model.layers.5.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 428 |
+
"language_model.layers.5.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 429 |
+
"language_model.layers.5.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 430 |
+
"language_model.layers.5.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 431 |
+
"language_model.layers.5.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 432 |
+
"language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 433 |
+
"language_model.layers.5.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 434 |
+
"language_model.layers.5.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 435 |
+
"language_model.layers.5.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 436 |
+
"language_model.layers.5.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 437 |
+
"language_model.layers.5.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 438 |
+
"language_model.layers.5.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 439 |
+
"language_model.layers.5.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 440 |
+
"language_model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 441 |
+
"language_model.layers.6.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 442 |
+
"language_model.layers.6.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 443 |
+
"language_model.layers.6.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 444 |
+
"language_model.layers.6.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 445 |
+
"language_model.layers.6.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 446 |
+
"language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 447 |
+
"language_model.layers.6.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 448 |
+
"language_model.layers.6.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 449 |
+
"language_model.layers.6.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 450 |
+
"language_model.layers.6.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 451 |
+
"language_model.layers.6.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 452 |
+
"language_model.layers.6.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 453 |
+
"language_model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 454 |
+
"language_model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 455 |
+
"language_model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 456 |
+
"language_model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 457 |
+
"language_model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 458 |
+
"language_model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 459 |
+
"language_model.layers.7.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
|
| 460 |
+
"language_model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
|
| 461 |
+
"language_model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
|
| 462 |
+
"language_model.layers.7.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
|
| 463 |
+
"language_model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
|
| 464 |
+
"language_model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
|
| 465 |
+
"language_model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 466 |
+
"language_model.layers.8.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 467 |
+
"language_model.layers.8.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 468 |
+
"language_model.layers.8.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 469 |
+
"language_model.layers.8.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 470 |
+
"language_model.layers.8.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 471 |
+
"language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 472 |
+
"language_model.layers.8.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 473 |
+
"language_model.layers.8.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 474 |
+
"language_model.layers.8.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 475 |
+
"language_model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 476 |
+
"language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 477 |
+
"language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 478 |
+
"language_model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 479 |
+
"language_model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 480 |
+
"language_model.layers.9.linear_attn.A_log": "model-00002-of-00005.safetensors",
|
| 481 |
+
"language_model.layers.9.linear_attn.conv1d.weight": "model-00002-of-00005.safetensors",
|
| 482 |
+
"language_model.layers.9.linear_attn.dt_bias": "model-00002-of-00005.safetensors",
|
| 483 |
+
"language_model.layers.9.linear_attn.in_proj_a.weight": "model-00002-of-00005.safetensors",
|
| 484 |
+
"language_model.layers.9.linear_attn.in_proj_b.weight": "model-00002-of-00005.safetensors",
|
| 485 |
+
"language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00002-of-00005.safetensors",
|
| 486 |
+
"language_model.layers.9.linear_attn.in_proj_z.weight": "model-00002-of-00005.safetensors",
|
| 487 |
+
"language_model.layers.9.linear_attn.norm.weight": "model-00002-of-00005.safetensors",
|
| 488 |
+
"language_model.layers.9.linear_attn.out_proj.weight": "model-00002-of-00005.safetensors",
|
| 489 |
+
"language_model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
|
| 490 |
+
"language_model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
|
| 491 |
+
"language_model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
|
| 492 |
+
"language_model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
|
| 493 |
+
"language_model.norm.weight": "model-00005-of-00005.safetensors",
|
| 494 |
+
"visual.blocks.0.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 495 |
+
"visual.blocks.0.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 496 |
+
"visual.blocks.0.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 497 |
+
"visual.blocks.0.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 498 |
+
"visual.blocks.0.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 499 |
+
"visual.blocks.0.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 500 |
+
"visual.blocks.0.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 501 |
+
"visual.blocks.0.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 502 |
+
"visual.blocks.0.norm1.bias": "model-00005-of-00005.safetensors",
|
| 503 |
+
"visual.blocks.0.norm1.weight": "model-00005-of-00005.safetensors",
|
| 504 |
+
"visual.blocks.0.norm2.bias": "model-00005-of-00005.safetensors",
|
| 505 |
+
"visual.blocks.0.norm2.weight": "model-00005-of-00005.safetensors",
|
| 506 |
+
"visual.blocks.1.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 507 |
+
"visual.blocks.1.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 508 |
+
"visual.blocks.1.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 509 |
+
"visual.blocks.1.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 510 |
+
"visual.blocks.1.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 511 |
+
"visual.blocks.1.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 512 |
+
"visual.blocks.1.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 513 |
+
"visual.blocks.1.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 514 |
+
"visual.blocks.1.norm1.bias": "model-00005-of-00005.safetensors",
|
| 515 |
+
"visual.blocks.1.norm1.weight": "model-00005-of-00005.safetensors",
|
| 516 |
+
"visual.blocks.1.norm2.bias": "model-00005-of-00005.safetensors",
|
| 517 |
+
"visual.blocks.1.norm2.weight": "model-00005-of-00005.safetensors",
|
| 518 |
+
"visual.blocks.10.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 519 |
+
"visual.blocks.10.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 520 |
+
"visual.blocks.10.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 521 |
+
"visual.blocks.10.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 522 |
+
"visual.blocks.10.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 523 |
+
"visual.blocks.10.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 524 |
+
"visual.blocks.10.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 525 |
+
"visual.blocks.10.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 526 |
+
"visual.blocks.10.norm1.bias": "model-00005-of-00005.safetensors",
|
| 527 |
+
"visual.blocks.10.norm1.weight": "model-00005-of-00005.safetensors",
|
| 528 |
+
"visual.blocks.10.norm2.bias": "model-00005-of-00005.safetensors",
|
| 529 |
+
"visual.blocks.10.norm2.weight": "model-00005-of-00005.safetensors",
|
| 530 |
+
"visual.blocks.11.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 531 |
+
"visual.blocks.11.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 532 |
+
"visual.blocks.11.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 533 |
+
"visual.blocks.11.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 534 |
+
"visual.blocks.11.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 535 |
+
"visual.blocks.11.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 536 |
+
"visual.blocks.11.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 537 |
+
"visual.blocks.11.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 538 |
+
"visual.blocks.11.norm1.bias": "model-00005-of-00005.safetensors",
|
| 539 |
+
"visual.blocks.11.norm1.weight": "model-00005-of-00005.safetensors",
|
| 540 |
+
"visual.blocks.11.norm2.bias": "model-00005-of-00005.safetensors",
|
| 541 |
+
"visual.blocks.11.norm2.weight": "model-00005-of-00005.safetensors",
|
| 542 |
+
"visual.blocks.12.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 543 |
+
"visual.blocks.12.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 544 |
+
"visual.blocks.12.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 545 |
+
"visual.blocks.12.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 546 |
+
"visual.blocks.12.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 547 |
+
"visual.blocks.12.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 548 |
+
"visual.blocks.12.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 549 |
+
"visual.blocks.12.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 550 |
+
"visual.blocks.12.norm1.bias": "model-00005-of-00005.safetensors",
|
| 551 |
+
"visual.blocks.12.norm1.weight": "model-00005-of-00005.safetensors",
|
| 552 |
+
"visual.blocks.12.norm2.bias": "model-00005-of-00005.safetensors",
|
| 553 |
+
"visual.blocks.12.norm2.weight": "model-00005-of-00005.safetensors",
|
| 554 |
+
"visual.blocks.13.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 555 |
+
"visual.blocks.13.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 556 |
+
"visual.blocks.13.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 557 |
+
"visual.blocks.13.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 558 |
+
"visual.blocks.13.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 559 |
+
"visual.blocks.13.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 560 |
+
"visual.blocks.13.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 561 |
+
"visual.blocks.13.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 562 |
+
"visual.blocks.13.norm1.bias": "model-00005-of-00005.safetensors",
|
| 563 |
+
"visual.blocks.13.norm1.weight": "model-00005-of-00005.safetensors",
|
| 564 |
+
"visual.blocks.13.norm2.bias": "model-00005-of-00005.safetensors",
|
| 565 |
+
"visual.blocks.13.norm2.weight": "model-00005-of-00005.safetensors",
|
| 566 |
+
"visual.blocks.14.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 567 |
+
"visual.blocks.14.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 568 |
+
"visual.blocks.14.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 569 |
+
"visual.blocks.14.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 570 |
+
"visual.blocks.14.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 571 |
+
"visual.blocks.14.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 572 |
+
"visual.blocks.14.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 573 |
+
"visual.blocks.14.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 574 |
+
"visual.blocks.14.norm1.bias": "model-00005-of-00005.safetensors",
|
| 575 |
+
"visual.blocks.14.norm1.weight": "model-00005-of-00005.safetensors",
|
| 576 |
+
"visual.blocks.14.norm2.bias": "model-00005-of-00005.safetensors",
|
| 577 |
+
"visual.blocks.14.norm2.weight": "model-00005-of-00005.safetensors",
|
| 578 |
+
"visual.blocks.15.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 579 |
+
"visual.blocks.15.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 580 |
+
"visual.blocks.15.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 581 |
+
"visual.blocks.15.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 582 |
+
"visual.blocks.15.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 583 |
+
"visual.blocks.15.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 584 |
+
"visual.blocks.15.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 585 |
+
"visual.blocks.15.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 586 |
+
"visual.blocks.15.norm1.bias": "model-00005-of-00005.safetensors",
|
| 587 |
+
"visual.blocks.15.norm1.weight": "model-00005-of-00005.safetensors",
|
| 588 |
+
"visual.blocks.15.norm2.bias": "model-00005-of-00005.safetensors",
|
| 589 |
+
"visual.blocks.15.norm2.weight": "model-00005-of-00005.safetensors",
|
| 590 |
+
"visual.blocks.16.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 591 |
+
"visual.blocks.16.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 592 |
+
"visual.blocks.16.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 593 |
+
"visual.blocks.16.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 594 |
+
"visual.blocks.16.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 595 |
+
"visual.blocks.16.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 596 |
+
"visual.blocks.16.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 597 |
+
"visual.blocks.16.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 598 |
+
"visual.blocks.16.norm1.bias": "model-00005-of-00005.safetensors",
|
| 599 |
+
"visual.blocks.16.norm1.weight": "model-00005-of-00005.safetensors",
|
| 600 |
+
"visual.blocks.16.norm2.bias": "model-00005-of-00005.safetensors",
|
| 601 |
+
"visual.blocks.16.norm2.weight": "model-00005-of-00005.safetensors",
|
| 602 |
+
"visual.blocks.17.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 603 |
+
"visual.blocks.17.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 604 |
+
"visual.blocks.17.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 605 |
+
"visual.blocks.17.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 606 |
+
"visual.blocks.17.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 607 |
+
"visual.blocks.17.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 608 |
+
"visual.blocks.17.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 609 |
+
"visual.blocks.17.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 610 |
+
"visual.blocks.17.norm1.bias": "model-00005-of-00005.safetensors",
|
| 611 |
+
"visual.blocks.17.norm1.weight": "model-00005-of-00005.safetensors",
|
| 612 |
+
"visual.blocks.17.norm2.bias": "model-00005-of-00005.safetensors",
|
| 613 |
+
"visual.blocks.17.norm2.weight": "model-00005-of-00005.safetensors",
|
| 614 |
+
"visual.blocks.18.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 615 |
+
"visual.blocks.18.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 616 |
+
"visual.blocks.18.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 617 |
+
"visual.blocks.18.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 618 |
+
"visual.blocks.18.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 619 |
+
"visual.blocks.18.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 620 |
+
"visual.blocks.18.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 621 |
+
"visual.blocks.18.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 622 |
+
"visual.blocks.18.norm1.bias": "model-00005-of-00005.safetensors",
|
| 623 |
+
"visual.blocks.18.norm1.weight": "model-00005-of-00005.safetensors",
|
| 624 |
+
"visual.blocks.18.norm2.bias": "model-00005-of-00005.safetensors",
|
| 625 |
+
"visual.blocks.18.norm2.weight": "model-00005-of-00005.safetensors",
|
| 626 |
+
"visual.blocks.19.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 627 |
+
"visual.blocks.19.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 628 |
+
"visual.blocks.19.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 629 |
+
"visual.blocks.19.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 630 |
+
"visual.blocks.19.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 631 |
+
"visual.blocks.19.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 632 |
+
"visual.blocks.19.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 633 |
+
"visual.blocks.19.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 634 |
+
"visual.blocks.19.norm1.bias": "model-00005-of-00005.safetensors",
|
| 635 |
+
"visual.blocks.19.norm1.weight": "model-00005-of-00005.safetensors",
|
| 636 |
+
"visual.blocks.19.norm2.bias": "model-00005-of-00005.safetensors",
|
| 637 |
+
"visual.blocks.19.norm2.weight": "model-00005-of-00005.safetensors",
|
| 638 |
+
"visual.blocks.2.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 639 |
+
"visual.blocks.2.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 640 |
+
"visual.blocks.2.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 641 |
+
"visual.blocks.2.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 642 |
+
"visual.blocks.2.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 643 |
+
"visual.blocks.2.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 644 |
+
"visual.blocks.2.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 645 |
+
"visual.blocks.2.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 646 |
+
"visual.blocks.2.norm1.bias": "model-00005-of-00005.safetensors",
|
| 647 |
+
"visual.blocks.2.norm1.weight": "model-00005-of-00005.safetensors",
|
| 648 |
+
"visual.blocks.2.norm2.bias": "model-00005-of-00005.safetensors",
|
| 649 |
+
"visual.blocks.2.norm2.weight": "model-00005-of-00005.safetensors",
|
| 650 |
+
"visual.blocks.20.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 651 |
+
"visual.blocks.20.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 652 |
+
"visual.blocks.20.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 653 |
+
"visual.blocks.20.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 654 |
+
"visual.blocks.20.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 655 |
+
"visual.blocks.20.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 656 |
+
"visual.blocks.20.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 657 |
+
"visual.blocks.20.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 658 |
+
"visual.blocks.20.norm1.bias": "model-00005-of-00005.safetensors",
|
| 659 |
+
"visual.blocks.20.norm1.weight": "model-00005-of-00005.safetensors",
|
| 660 |
+
"visual.blocks.20.norm2.bias": "model-00005-of-00005.safetensors",
|
| 661 |
+
"visual.blocks.20.norm2.weight": "model-00005-of-00005.safetensors",
|
| 662 |
+
"visual.blocks.21.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 663 |
+
"visual.blocks.21.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 664 |
+
"visual.blocks.21.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 665 |
+
"visual.blocks.21.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 666 |
+
"visual.blocks.21.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 667 |
+
"visual.blocks.21.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 668 |
+
"visual.blocks.21.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 669 |
+
"visual.blocks.21.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 670 |
+
"visual.blocks.21.norm1.bias": "model-00005-of-00005.safetensors",
|
| 671 |
+
"visual.blocks.21.norm1.weight": "model-00005-of-00005.safetensors",
|
| 672 |
+
"visual.blocks.21.norm2.bias": "model-00005-of-00005.safetensors",
|
| 673 |
+
"visual.blocks.21.norm2.weight": "model-00005-of-00005.safetensors",
|
| 674 |
+
"visual.blocks.22.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 675 |
+
"visual.blocks.22.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 676 |
+
"visual.blocks.22.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 677 |
+
"visual.blocks.22.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 678 |
+
"visual.blocks.22.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 679 |
+
"visual.blocks.22.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 680 |
+
"visual.blocks.22.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 681 |
+
"visual.blocks.22.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 682 |
+
"visual.blocks.22.norm1.bias": "model-00005-of-00005.safetensors",
|
| 683 |
+
"visual.blocks.22.norm1.weight": "model-00005-of-00005.safetensors",
|
| 684 |
+
"visual.blocks.22.norm2.bias": "model-00005-of-00005.safetensors",
|
| 685 |
+
"visual.blocks.22.norm2.weight": "model-00005-of-00005.safetensors",
|
| 686 |
+
"visual.blocks.23.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 687 |
+
"visual.blocks.23.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 688 |
+
"visual.blocks.23.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 689 |
+
"visual.blocks.23.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 690 |
+
"visual.blocks.23.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 691 |
+
"visual.blocks.23.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 692 |
+
"visual.blocks.23.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 693 |
+
"visual.blocks.23.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 694 |
+
"visual.blocks.23.norm1.bias": "model-00005-of-00005.safetensors",
|
| 695 |
+
"visual.blocks.23.norm1.weight": "model-00005-of-00005.safetensors",
|
| 696 |
+
"visual.blocks.23.norm2.bias": "model-00005-of-00005.safetensors",
|
| 697 |
+
"visual.blocks.23.norm2.weight": "model-00005-of-00005.safetensors",
|
| 698 |
+
"visual.blocks.3.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 699 |
+
"visual.blocks.3.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 700 |
+
"visual.blocks.3.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 701 |
+
"visual.blocks.3.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 702 |
+
"visual.blocks.3.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 703 |
+
"visual.blocks.3.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 704 |
+
"visual.blocks.3.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 705 |
+
"visual.blocks.3.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 706 |
+
"visual.blocks.3.norm1.bias": "model-00005-of-00005.safetensors",
|
| 707 |
+
"visual.blocks.3.norm1.weight": "model-00005-of-00005.safetensors",
|
| 708 |
+
"visual.blocks.3.norm2.bias": "model-00005-of-00005.safetensors",
|
| 709 |
+
"visual.blocks.3.norm2.weight": "model-00005-of-00005.safetensors",
|
| 710 |
+
"visual.blocks.4.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 711 |
+
"visual.blocks.4.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 712 |
+
"visual.blocks.4.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 713 |
+
"visual.blocks.4.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 714 |
+
"visual.blocks.4.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 715 |
+
"visual.blocks.4.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 716 |
+
"visual.blocks.4.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 717 |
+
"visual.blocks.4.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 718 |
+
"visual.blocks.4.norm1.bias": "model-00005-of-00005.safetensors",
|
| 719 |
+
"visual.blocks.4.norm1.weight": "model-00005-of-00005.safetensors",
|
| 720 |
+
"visual.blocks.4.norm2.bias": "model-00005-of-00005.safetensors",
|
| 721 |
+
"visual.blocks.4.norm2.weight": "model-00005-of-00005.safetensors",
|
| 722 |
+
"visual.blocks.5.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 723 |
+
"visual.blocks.5.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 724 |
+
"visual.blocks.5.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 725 |
+
"visual.blocks.5.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 726 |
+
"visual.blocks.5.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 727 |
+
"visual.blocks.5.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 728 |
+
"visual.blocks.5.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 729 |
+
"visual.blocks.5.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 730 |
+
"visual.blocks.5.norm1.bias": "model-00005-of-00005.safetensors",
|
| 731 |
+
"visual.blocks.5.norm1.weight": "model-00005-of-00005.safetensors",
|
| 732 |
+
"visual.blocks.5.norm2.bias": "model-00005-of-00005.safetensors",
|
| 733 |
+
"visual.blocks.5.norm2.weight": "model-00005-of-00005.safetensors",
|
| 734 |
+
"visual.blocks.6.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 735 |
+
"visual.blocks.6.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 736 |
+
"visual.blocks.6.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 737 |
+
"visual.blocks.6.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 738 |
+
"visual.blocks.6.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 739 |
+
"visual.blocks.6.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 740 |
+
"visual.blocks.6.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 741 |
+
"visual.blocks.6.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 742 |
+
"visual.blocks.6.norm1.bias": "model-00005-of-00005.safetensors",
|
| 743 |
+
"visual.blocks.6.norm1.weight": "model-00005-of-00005.safetensors",
|
| 744 |
+
"visual.blocks.6.norm2.bias": "model-00005-of-00005.safetensors",
|
| 745 |
+
"visual.blocks.6.norm2.weight": "model-00005-of-00005.safetensors",
|
| 746 |
+
"visual.blocks.7.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 747 |
+
"visual.blocks.7.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 748 |
+
"visual.blocks.7.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 749 |
+
"visual.blocks.7.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 750 |
+
"visual.blocks.7.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 751 |
+
"visual.blocks.7.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 752 |
+
"visual.blocks.7.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 753 |
+
"visual.blocks.7.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 754 |
+
"visual.blocks.7.norm1.bias": "model-00005-of-00005.safetensors",
|
| 755 |
+
"visual.blocks.7.norm1.weight": "model-00005-of-00005.safetensors",
|
| 756 |
+
"visual.blocks.7.norm2.bias": "model-00005-of-00005.safetensors",
|
| 757 |
+
"visual.blocks.7.norm2.weight": "model-00005-of-00005.safetensors",
|
| 758 |
+
"visual.blocks.8.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 759 |
+
"visual.blocks.8.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 760 |
+
"visual.blocks.8.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 761 |
+
"visual.blocks.8.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 762 |
+
"visual.blocks.8.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 763 |
+
"visual.blocks.8.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 764 |
+
"visual.blocks.8.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 765 |
+
"visual.blocks.8.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 766 |
+
"visual.blocks.8.norm1.bias": "model-00005-of-00005.safetensors",
|
| 767 |
+
"visual.blocks.8.norm1.weight": "model-00005-of-00005.safetensors",
|
| 768 |
+
"visual.blocks.8.norm2.bias": "model-00005-of-00005.safetensors",
|
| 769 |
+
"visual.blocks.8.norm2.weight": "model-00005-of-00005.safetensors",
|
| 770 |
+
"visual.blocks.9.attn.proj.bias": "model-00005-of-00005.safetensors",
|
| 771 |
+
"visual.blocks.9.attn.proj.weight": "model-00005-of-00005.safetensors",
|
| 772 |
+
"visual.blocks.9.attn.qkv.bias": "model-00005-of-00005.safetensors",
|
| 773 |
+
"visual.blocks.9.attn.qkv.weight": "model-00005-of-00005.safetensors",
|
| 774 |
+
"visual.blocks.9.mlp.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 775 |
+
"visual.blocks.9.mlp.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 776 |
+
"visual.blocks.9.mlp.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 777 |
+
"visual.blocks.9.mlp.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 778 |
+
"visual.blocks.9.norm1.bias": "model-00005-of-00005.safetensors",
|
| 779 |
+
"visual.blocks.9.norm1.weight": "model-00005-of-00005.safetensors",
|
| 780 |
+
"visual.blocks.9.norm2.bias": "model-00005-of-00005.safetensors",
|
| 781 |
+
"visual.blocks.9.norm2.weight": "model-00005-of-00005.safetensors",
|
| 782 |
+
"visual.merger.linear_fc1.bias": "model-00005-of-00005.safetensors",
|
| 783 |
+
"visual.merger.linear_fc1.weight": "model-00005-of-00005.safetensors",
|
| 784 |
+
"visual.merger.linear_fc2.bias": "model-00005-of-00005.safetensors",
|
| 785 |
+
"visual.merger.linear_fc2.weight": "model-00005-of-00005.safetensors",
|
| 786 |
+
"visual.merger.norm.bias": "model-00005-of-00005.safetensors",
|
| 787 |
+
"visual.merger.norm.weight": "model-00005-of-00005.safetensors",
|
| 788 |
+
"visual.patch_embed.proj.bias": "model-00005-of-00005.safetensors",
|
| 789 |
+
"visual.patch_embed.proj.weight": "model-00005-of-00005.safetensors",
|
| 790 |
+
"visual.pos_embed.weight": "model-00005-of-00005.safetensors"
|
| 791 |
+
}
|
| 792 |
+
}
|
modular_gs1.py
ADDED
|
@@ -0,0 +1,971 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GS1: native hybrid text/vision decision model with attention residuals."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
from torch.nn import functional as F
|
| 9 |
+
from torch.nn.attention import SDPBackend, sdpa_kernel
|
| 10 |
+
from torch.utils.checkpoint import checkpoint
|
| 11 |
+
from transformers import AutoConfig, AutoModel, Qwen3_5Config
|
| 12 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import (
|
| 13 |
+
Qwen3_5Attention,
|
| 14 |
+
Qwen3_5DynamicCache,
|
| 15 |
+
Qwen3_5GatedDeltaNet,
|
| 16 |
+
Qwen3_5MLP,
|
| 17 |
+
Qwen3_5PreTrainedModel,
|
| 18 |
+
Qwen3_5RMSNorm,
|
| 19 |
+
Qwen3_5TextRotaryEmbedding,
|
| 20 |
+
Qwen3_5VisionModel,
|
| 21 |
+
apply_rotary_pos_emb,
|
| 22 |
+
)
|
| 23 |
+
from transformers.utils import ModelOutput
|
| 24 |
+
|
| 25 |
+
from .config import INFERENCE_LIMITS, CalibrationConfig, DecisionConfig
|
| 26 |
+
from .util import named_seed
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class GS1Config(Qwen3_5Config):
|
| 30 |
+
model_type = "gs1"
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
decision=None,
|
| 35 |
+
calibration=None,
|
| 36 |
+
initialization_seed=12,
|
| 37 |
+
head_compute_dtype="float32",
|
| 38 |
+
inference_limits=None,
|
| 39 |
+
**kwargs,
|
| 40 |
+
):
|
| 41 |
+
kwargs.pop("model_type", None)
|
| 42 |
+
kwargs.pop("architectures", None)
|
| 43 |
+
super().__init__(**kwargs)
|
| 44 |
+
self.decision = DecisionConfig.model_validate(decision or {}).model_dump()
|
| 45 |
+
self.calibration = CalibrationConfig.model_validate(calibration or {}).model_dump()
|
| 46 |
+
self.initialization_seed = initialization_seed
|
| 47 |
+
self.inference_limits = {**INFERENCE_LIMITS, **(inference_limits or {})}
|
| 48 |
+
if self.inference_limits.keys() != INFERENCE_LIMITS.keys() or any(
|
| 49 |
+
type(value) is not int or value < 1 for value in self.inference_limits.values()
|
| 50 |
+
):
|
| 51 |
+
raise ValueError("Invalid GS1 inference limits")
|
| 52 |
+
if head_compute_dtype not in {"float32", "bfloat16"}:
|
| 53 |
+
raise ValueError("head_compute_dtype must be float32 or bfloat16")
|
| 54 |
+
self.head_compute_dtype = head_compute_dtype
|
| 55 |
+
self.architectures = ["GS1Model"]
|
| 56 |
+
self.tie_word_embeddings = False
|
| 57 |
+
self.text_config.use_cache = False
|
| 58 |
+
self.text_config._attn_implementation = "sdpa"
|
| 59 |
+
self.vision_config._attn_implementation = "sdpa"
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass
|
| 63 |
+
class GS1Output(ModelOutput):
|
| 64 |
+
logits: torch.Tensor | None = None
|
| 65 |
+
last_hidden_state: torch.Tensor | None = None
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@dataclass
|
| 69 |
+
class GS1TextOutput(ModelOutput):
|
| 70 |
+
last_hidden_state: torch.Tensor | None = None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@dataclass(frozen=True)
|
| 74 |
+
class GDNState:
|
| 75 |
+
"""Native GDN prefix, including the last complete scan-chunk boundary."""
|
| 76 |
+
|
| 77 |
+
conv: torch.Tensor
|
| 78 |
+
recurrent: torch.Tensor
|
| 79 |
+
boundary_conv: torch.Tensor | None
|
| 80 |
+
boundary_recurrent: torch.Tensor | None
|
| 81 |
+
tail: torch.Tensor
|
| 82 |
+
position: int
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class GS1DecoderLayer(nn.Module):
|
| 86 |
+
"""A hybrid decoder block; only its full-attention input can read depth history."""
|
| 87 |
+
|
| 88 |
+
def __init__(self, config, index):
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.layer_type = config.layer_types[index]
|
| 91 |
+
self.config = config
|
| 92 |
+
self.index = index
|
| 93 |
+
if self.layer_type == "full_attention":
|
| 94 |
+
self.self_attn = Qwen3_5Attention(config, index)
|
| 95 |
+
elif self.layer_type == "linear_attention":
|
| 96 |
+
self.linear_attn = Qwen3_5GatedDeltaNet(config, index)
|
| 97 |
+
else:
|
| 98 |
+
raise ValueError(f"Unsupported decoder layer type: {self.layer_type}")
|
| 99 |
+
self.mlp = Qwen3_5MLP(config, config.intermediate_size)
|
| 100 |
+
self.input_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 101 |
+
self.post_attention_layernorm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 102 |
+
|
| 103 |
+
def forward(self, carry, positional, mask, read, terminal_query, valid_lengths=None, cohorts=None):
|
| 104 |
+
if self.layer_type == "full_attention":
|
| 105 |
+
mixed, _ = self.self_attn(
|
| 106 |
+
self.input_layernorm(read), position_embeddings=positional, attention_mask=mask
|
| 107 |
+
)
|
| 108 |
+
elif valid_lengths is not None and min(valid_lengths) != carry.shape[1]:
|
| 109 |
+
# GDN scans only real tokens. Equal-length cohorts retain native
|
| 110 |
+
# batched kernels and their original chunk boundaries; FA and FFN
|
| 111 |
+
# still execute over the complete decoder batch.
|
| 112 |
+
mixed = torch.zeros_like(carry)
|
| 113 |
+
if cohorts is None:
|
| 114 |
+
cohorts = tuple(
|
| 115 |
+
(
|
| 116 |
+
length,
|
| 117 |
+
torch.tensor(
|
| 118 |
+
[i for i, value in enumerate(valid_lengths) if value == length],
|
| 119 |
+
device=carry.device,
|
| 120 |
+
),
|
| 121 |
+
)
|
| 122 |
+
for length in sorted(set(valid_lengths))
|
| 123 |
+
)
|
| 124 |
+
for length, rows in cohorts:
|
| 125 |
+
value = carry.index_select(0, rows)[:, -length:]
|
| 126 |
+
normalized = self.input_layernorm(value)
|
| 127 |
+
if terminal_query:
|
| 128 |
+
cache = Qwen3_5DynamicCache(self.config)
|
| 129 |
+
cache.last_linear_layer = self.index
|
| 130 |
+
context = self.linear_attn(
|
| 131 |
+
normalized[:, :-1],
|
| 132 |
+
cache_params=cache,
|
| 133 |
+
cache_position=torch.arange(length - 1, device=carry.device),
|
| 134 |
+
)
|
| 135 |
+
query, _ = _linear_context(
|
| 136 |
+
self.linear_attn,
|
| 137 |
+
normalized[:, -1:],
|
| 138 |
+
(cache.conv_states[self.index], cache.recurrent_states[self.index]),
|
| 139 |
+
self.config,
|
| 140 |
+
)
|
| 141 |
+
value = torch.cat((context, query), 1)
|
| 142 |
+
else:
|
| 143 |
+
value = self.linear_attn(normalized)
|
| 144 |
+
mixed = mixed.index_copy(
|
| 145 |
+
0, rows, F.pad(value, (0, 0, carry.shape[1] - length, 0)).to(mixed.dtype)
|
| 146 |
+
)
|
| 147 |
+
elif terminal_query:
|
| 148 |
+
# Keep the trained context scan and differentiable one-token continuation.
|
| 149 |
+
cache = Qwen3_5DynamicCache(self.config)
|
| 150 |
+
cache.last_linear_layer = self.index
|
| 151 |
+
context = carry[:, :-1]
|
| 152 |
+
context_mixed = self.linear_attn(
|
| 153 |
+
self.input_layernorm(context),
|
| 154 |
+
cache_params=cache,
|
| 155 |
+
cache_position=torch.arange(context.shape[1], device=carry.device),
|
| 156 |
+
)
|
| 157 |
+
query_mixed, _ = _linear_context(
|
| 158 |
+
self.linear_attn,
|
| 159 |
+
self.input_layernorm(carry[:, -1:]),
|
| 160 |
+
(cache.conv_states[self.index], cache.recurrent_states[self.index]),
|
| 161 |
+
self.config,
|
| 162 |
+
)
|
| 163 |
+
context = context + context_mixed
|
| 164 |
+
query = carry[:, -1:] + query_mixed
|
| 165 |
+
context_ffn = self.mlp(self.post_attention_layernorm(context))
|
| 166 |
+
query_ffn = self.mlp(self.post_attention_layernorm(query))
|
| 167 |
+
return (
|
| 168 |
+
torch.cat((context + context_ffn, query + query_ffn), 1),
|
| 169 |
+
torch.cat((context_mixed, query_mixed), 1),
|
| 170 |
+
torch.cat((context_ffn, query_ffn), 1),
|
| 171 |
+
)
|
| 172 |
+
else:
|
| 173 |
+
mixed = self.linear_attn(self.input_layernorm(carry))
|
| 174 |
+
hidden = carry + mixed
|
| 175 |
+
ffn = self.mlp(self.post_attention_layernorm(hidden))
|
| 176 |
+
return hidden + ffn, mixed, ffn
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class CrossAttentionDecisionHead(nn.Module):
|
| 180 |
+
"""Read state at backbone width; project only the final dot-product vectors."""
|
| 181 |
+
|
| 182 |
+
def __init__(self, hidden_size, decision=None, seed=12):
|
| 183 |
+
super().__init__()
|
| 184 |
+
self.config = DecisionConfig.model_validate(decision or {})
|
| 185 |
+
if hidden_size % self.config.cross_attention_heads:
|
| 186 |
+
raise ValueError("cross_attention_heads must divide the backbone hidden size")
|
| 187 |
+
width = self.config.projection_size
|
| 188 |
+
eps = self.config.norm_eps
|
| 189 |
+
self.bf16_compute = False
|
| 190 |
+
with torch.random.fork_rng(devices=[]):
|
| 191 |
+
torch.manual_seed(named_seed(seed, "initialization/cross_attention_head"))
|
| 192 |
+
self.memory_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 193 |
+
self.memory_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 194 |
+
self.question_seed = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 195 |
+
self.question_attention = nn.MultiheadAttention(
|
| 196 |
+
hidden_size, self.config.cross_attention_heads, batch_first=True
|
| 197 |
+
)
|
| 198 |
+
self.decision_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 199 |
+
self.question_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 200 |
+
self.question_evidence_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 201 |
+
self.context_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 202 |
+
self.option_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 203 |
+
self.condition_projection = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 204 |
+
self.option_norm = nn.LayerNorm(hidden_size, eps=eps)
|
| 205 |
+
self.routing_attention = nn.ModuleList(
|
| 206 |
+
nn.MultiheadAttention(hidden_size, self.config.cross_attention_heads, batch_first=True)
|
| 207 |
+
for _ in range(self.config.cross_attention_layers)
|
| 208 |
+
)
|
| 209 |
+
self.routing_attention_norm = nn.ModuleList(
|
| 210 |
+
nn.LayerNorm(hidden_size, eps=eps) for _ in range(self.config.cross_attention_layers)
|
| 211 |
+
)
|
| 212 |
+
self.routing_ffn = nn.ModuleList(
|
| 213 |
+
nn.Sequential(
|
| 214 |
+
nn.Linear(hidden_size, hidden_size),
|
| 215 |
+
nn.GELU(),
|
| 216 |
+
nn.Linear(hidden_size, hidden_size),
|
| 217 |
+
)
|
| 218 |
+
for _ in range(self.config.cross_attention_layers)
|
| 219 |
+
)
|
| 220 |
+
self.routing_output_norm = nn.ModuleList(
|
| 221 |
+
nn.LayerNorm(hidden_size, eps=eps) for _ in range(self.config.cross_attention_layers)
|
| 222 |
+
)
|
| 223 |
+
self.query_mlp = nn.Sequential(
|
| 224 |
+
nn.Linear(hidden_size, hidden_size), nn.GELU(), nn.Linear(hidden_size, width)
|
| 225 |
+
)
|
| 226 |
+
self.key_mlp = nn.Sequential(
|
| 227 |
+
nn.Linear(hidden_size, hidden_size), nn.GELU(), nn.Linear(hidden_size, width)
|
| 228 |
+
)
|
| 229 |
+
self.query_norm = nn.LayerNorm(width, eps=eps)
|
| 230 |
+
self.key_norm = nn.LayerNorm(width, eps=eps)
|
| 231 |
+
|
| 232 |
+
def prepare_memory(self, state):
|
| 233 |
+
"""Prepare immutable state memory and each attention module's own K/V."""
|
| 234 |
+
dtype = torch.bfloat16 if self.bf16_compute else self.memory_projection.weight.dtype
|
| 235 |
+
with torch.autocast(device_type=state.device.type, dtype=torch.bfloat16, enabled=self.bf16_compute):
|
| 236 |
+
memory = self.memory_projection(self.memory_norm(state.to(dtype))).unsqueeze(0)
|
| 237 |
+
projections = []
|
| 238 |
+
for attention in (self.question_attention, *self.routing_attention):
|
| 239 |
+
width = attention.embed_dim
|
| 240 |
+
bias = attention.in_proj_bias[width:] if attention.in_proj_bias is not None else None
|
| 241 |
+
key, value = F.linear(memory, attention.in_proj_weight[width:], bias).chunk(2, -1)
|
| 242 |
+
shape = (1, memory.shape[1], attention.num_heads, width // attention.num_heads)
|
| 243 |
+
projections.append((key.reshape(shape).transpose(1, 2), value.reshape(shape).transpose(1, 2)))
|
| 244 |
+
return memory, tuple(projections)
|
| 245 |
+
|
| 246 |
+
@staticmethod
|
| 247 |
+
def _attend(attention, query, memory, projected=None):
|
| 248 |
+
if projected is None:
|
| 249 |
+
return attention(query, memory, memory, need_weights=False)[0]
|
| 250 |
+
width = attention.embed_dim
|
| 251 |
+
bias = attention.in_proj_bias[:width] if attention.in_proj_bias is not None else None
|
| 252 |
+
query = F.linear(query, attention.in_proj_weight[:width], bias)
|
| 253 |
+
batch, length = query.shape[:2]
|
| 254 |
+
query = query.reshape(batch, length, attention.num_heads, width // attention.num_heads).transpose(
|
| 255 |
+
1, 2
|
| 256 |
+
)
|
| 257 |
+
value = F.scaled_dot_product_attention(query, *projected, dropout_p=0.0)
|
| 258 |
+
value = value.transpose(1, 2).reshape(batch, length, width)
|
| 259 |
+
return F.linear(value, attention.out_proj.weight, attention.out_proj.bias)
|
| 260 |
+
|
| 261 |
+
def forward(self, state, question, decision, options, *, prepared_memory=None):
|
| 262 |
+
if state.ndim != 2 or question.ndim != 1 or decision.ndim != 1 or options.ndim != 2:
|
| 263 |
+
raise ValueError(
|
| 264 |
+
"Cross-attention readout expects state [T,D], question/decision [D], options [K,D]"
|
| 265 |
+
)
|
| 266 |
+
if not len(state) or not len(options):
|
| 267 |
+
raise ValueError("Cross-attention readout requires state and candidates")
|
| 268 |
+
dtype = torch.bfloat16 if self.bf16_compute else self.memory_projection.weight.dtype
|
| 269 |
+
with torch.autocast(device_type=state.device.type, dtype=torch.bfloat16, enabled=self.bf16_compute):
|
| 270 |
+
state = state.to(dtype)
|
| 271 |
+
question = question.to(dtype)
|
| 272 |
+
decision = decision.to(dtype)
|
| 273 |
+
options = options.to(dtype)
|
| 274 |
+
memory, projected = (
|
| 275 |
+
(self.memory_projection(self.memory_norm(state)).unsqueeze(0), None)
|
| 276 |
+
if prepared_memory is None
|
| 277 |
+
else prepared_memory
|
| 278 |
+
)
|
| 279 |
+
question_query = self.question_seed(question)[None, None]
|
| 280 |
+
question_evidence = self._attend(
|
| 281 |
+
self.question_attention, question_query, memory, None if projected is None else projected[0]
|
| 282 |
+
)[0, 0]
|
| 283 |
+
context = self.context_norm(
|
| 284 |
+
self.decision_projection(decision)
|
| 285 |
+
+ self.question_projection(question)
|
| 286 |
+
+ self.question_evidence_projection(question_evidence)
|
| 287 |
+
)
|
| 288 |
+
routed = self.option_norm(self.option_projection(options) + self.condition_projection(context))
|
| 289 |
+
for layer_index, (attention, attn_norm, ffn, output_norm) in enumerate(
|
| 290 |
+
zip(
|
| 291 |
+
self.routing_attention,
|
| 292 |
+
self.routing_attention_norm,
|
| 293 |
+
self.routing_ffn,
|
| 294 |
+
self.routing_output_norm,
|
| 295 |
+
strict=True,
|
| 296 |
+
)
|
| 297 |
+
):
|
| 298 |
+
evidence = self._attend(
|
| 299 |
+
attention, routed[None], memory, None if projected is None else projected[layer_index + 1]
|
| 300 |
+
)[0]
|
| 301 |
+
updated = attn_norm(routed + evidence)
|
| 302 |
+
routed = output_norm(updated + ffn(updated))
|
| 303 |
+
query = self.query_norm(self.query_mlp(context))
|
| 304 |
+
keys = self.key_norm(self.key_mlp(routed))
|
| 305 |
+
with torch.autocast(device_type=state.device.type, enabled=False):
|
| 306 |
+
return (keys.float() @ query.float()) / math.sqrt(self.config.projection_size)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
class FADepthRouter(nn.Module):
|
| 310 |
+
def __init__(self, hidden_size, rms_norm_eps=1e-6):
|
| 311 |
+
super().__init__()
|
| 312 |
+
self.query = nn.Parameter(torch.zeros(hidden_size))
|
| 313 |
+
self.rms_norm_eps = rms_norm_eps
|
| 314 |
+
|
| 315 |
+
def forward(self, carry, sources):
|
| 316 |
+
"""Return m * sum(alpha_j * v_j), with m equal to the source count.
|
| 317 |
+
|
| 318 |
+
Centering the weights around 1/m uses the existing carry as the sum of
|
| 319 |
+
sources. This equivalent form avoids regrouping low-precision residual
|
| 320 |
+
additions at initialization: query=0 gives exactly r=carry. Neither
|
| 321 |
+
keys nor values are detached; all depth paths retain their gradients.
|
| 322 |
+
"""
|
| 323 |
+
with torch.autocast(device_type=carry.device.type, enabled=False):
|
| 324 |
+
values = torch.stack(sources).float()
|
| 325 |
+
keys = values * torch.rsqrt(values.square().mean(-1, keepdim=True) + self.rms_norm_eps)
|
| 326 |
+
logits = torch.einsum("d,nbtd->nbt", self.query.float(), keys)
|
| 327 |
+
alpha = logits.softmax(dim=0)
|
| 328 |
+
count = len(sources)
|
| 329 |
+
centered = (alpha - 1.0 / count) * count
|
| 330 |
+
correction = torch.einsum("nbt,nbtd->btd", centered, values)
|
| 331 |
+
return (carry.float() + correction).to(carry.dtype)
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _attention(module, hidden, positional, previous, *, bidirectional=False):
|
| 335 |
+
batch, length = hidden.shape[:2]
|
| 336 |
+
shape = (batch, length, -1, module.head_dim)
|
| 337 |
+
query, gate = module.q_proj(hidden).view(batch, length, -1, 2 * module.head_dim).chunk(2, dim=-1)
|
| 338 |
+
query = module.q_norm(query).transpose(1, 2)
|
| 339 |
+
key = module.k_norm(module.k_proj(hidden).view(shape)).transpose(1, 2)
|
| 340 |
+
value = module.v_proj(hidden).view(shape).transpose(1, 2)
|
| 341 |
+
query, key = apply_rotary_pos_emb(query, key, *positional)
|
| 342 |
+
prefix_length = previous[0].shape[2] if previous else 0
|
| 343 |
+
if previous:
|
| 344 |
+
key = torch.cat((previous[0], key), dim=2)
|
| 345 |
+
value = torch.cat((previous[1], value), dim=2)
|
| 346 |
+
# Keep compact GQA cache tensors while avoiding a quadratic math fallback
|
| 347 |
+
# in SDPA implementations that do not fuse grouped-query attention.
|
| 348 |
+
groups = query.shape[1] // key.shape[1]
|
| 349 |
+
attention_query = query.to(value.dtype)
|
| 350 |
+
attention_key = key.to(value.dtype)
|
| 351 |
+
attention_value = value
|
| 352 |
+
if groups > 1:
|
| 353 |
+
attention_key = attention_key.repeat_interleave(groups, dim=1)
|
| 354 |
+
attention_value = attention_value.repeat_interleave(groups, dim=1)
|
| 355 |
+
padding = (-module.head_dim) % 8
|
| 356 |
+
if padding:
|
| 357 |
+
attention_query, attention_key, attention_value = (
|
| 358 |
+
F.pad(tensor, (0, padding)) for tensor in (attention_query, attention_key, attention_value)
|
| 359 |
+
)
|
| 360 |
+
mask = None
|
| 361 |
+
if prefix_length and not bidirectional:
|
| 362 |
+
q_positions = torch.arange(length, device=hidden.device)[:, None]
|
| 363 |
+
k_positions = torch.arange(prefix_length + length, device=hidden.device)[None, :]
|
| 364 |
+
allowed = k_positions <= prefix_length + q_positions
|
| 365 |
+
width = math.ceil((prefix_length + length) / 8) * 8
|
| 366 |
+
mask = attention_query.new_zeros((length, width))[:, : prefix_length + length]
|
| 367 |
+
mask.masked_fill_(~allowed, float("-inf"))
|
| 368 |
+
with (
|
| 369 |
+
torch.autocast(device_type=hidden.device.type, enabled=False),
|
| 370 |
+
sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]),
|
| 371 |
+
):
|
| 372 |
+
attended = F.scaled_dot_product_attention(
|
| 373 |
+
attention_query,
|
| 374 |
+
attention_key,
|
| 375 |
+
attention_value,
|
| 376 |
+
attn_mask=mask,
|
| 377 |
+
is_causal=not prefix_length and not bidirectional,
|
| 378 |
+
dropout_p=0.0,
|
| 379 |
+
scale=module.scaling,
|
| 380 |
+
)[..., : module.head_dim]
|
| 381 |
+
attended = attended.transpose(1, 2).reshape(batch, length, -1)
|
| 382 |
+
output = module.o_proj(attended * gate.reshape(batch, length, -1).sigmoid())
|
| 383 |
+
return output, (key, value)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _linear_context(module, hidden, previous, config):
|
| 387 |
+
if previous is None:
|
| 388 |
+
cache = Qwen3_5DynamicCache(config)
|
| 389 |
+
cache.last_linear_layer = module.layer_idx
|
| 390 |
+
output = module(hidden, cache_params=cache)
|
| 391 |
+
return output, (cache.conv_states[module.layer_idx], cache.recurrent_states[module.layer_idx])
|
| 392 |
+
|
| 393 |
+
batch, length = hidden.shape[:2]
|
| 394 |
+
raw = module.in_proj_qkv(hidden).transpose(1, 2)
|
| 395 |
+
window = torch.cat((previous[0], raw), dim=-1)
|
| 396 |
+
# The cached window holds kernel_size raw values. Its oldest value precedes
|
| 397 |
+
# the first suffix output's receptive field; exclude that value from conv.
|
| 398 |
+
if module.causal_conv1d_fn is not None:
|
| 399 |
+
# Match the upstream fused convolution+SiLU rounding in BF16. Applying
|
| 400 |
+
# the same causal kernel to the cached window preserves continuation
|
| 401 |
+
# gradients and avoids an extra low-precision convolution round trip.
|
| 402 |
+
mixed = module.causal_conv1d_fn(
|
| 403 |
+
x=window,
|
| 404 |
+
weight=module.conv1d.weight.squeeze(1),
|
| 405 |
+
bias=module.conv1d.bias,
|
| 406 |
+
activation=module.activation,
|
| 407 |
+
)[:, :, -length:]
|
| 408 |
+
else:
|
| 409 |
+
mixed = F.silu(
|
| 410 |
+
F.conv1d(window[:, :, 1:], module.conv1d.weight, module.conv1d.bias, groups=raw.shape[1])
|
| 411 |
+
)
|
| 412 |
+
mixed = mixed.transpose(1, 2)
|
| 413 |
+
query, key, value = mixed.split((module.key_dim, module.key_dim, module.value_dim), dim=-1)
|
| 414 |
+
query = query.reshape(batch, length, -1, module.head_k_dim)
|
| 415 |
+
key = key.reshape(batch, length, -1, module.head_k_dim)
|
| 416 |
+
value = value.reshape(batch, length, -1, module.head_v_dim)
|
| 417 |
+
repeat = module.num_v_heads // module.num_k_heads
|
| 418 |
+
if repeat > 1:
|
| 419 |
+
query = query.repeat_interleave(repeat, dim=2)
|
| 420 |
+
key = key.repeat_interleave(repeat, dim=2)
|
| 421 |
+
beta = module.in_proj_b(hidden).sigmoid()
|
| 422 |
+
decay = -module.A_log.float().exp() * F.softplus(module.in_proj_a(hidden).float() + module.dt_bias)
|
| 423 |
+
mixed, recurrent = module.chunk_gated_delta_rule(
|
| 424 |
+
query,
|
| 425 |
+
key,
|
| 426 |
+
value,
|
| 427 |
+
g=decay,
|
| 428 |
+
beta=beta,
|
| 429 |
+
initial_state=previous[1],
|
| 430 |
+
output_final_state=True,
|
| 431 |
+
use_qk_l2norm_in_kernel=True,
|
| 432 |
+
)
|
| 433 |
+
gate = module.in_proj_z(hidden).reshape(-1, module.head_v_dim)
|
| 434 |
+
mixed = module.norm(mixed.reshape(-1, module.head_v_dim), gate).reshape(batch, length, -1)
|
| 435 |
+
return module.out_proj(mixed), (window[:, :, -module.conv_kernel_size :].clone(), recurrent)
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def _continue_gdn(module, hidden, previous, config, *, terminal=False, scan=None):
|
| 439 |
+
"""Continue on the native 64-token chunk grid, preserving terminal scans."""
|
| 440 |
+
scan = scan or _linear_context
|
| 441 |
+
chunk_size = 64
|
| 442 |
+
|
| 443 |
+
if torch.is_grad_enabled() or module.training:
|
| 444 |
+
raise ValueError("Chunk-aware prefix state is inference-only")
|
| 445 |
+
if previous is not None and not isinstance(previous, GDNState):
|
| 446 |
+
raise ValueError("GS1 continuation requires chunk-aware native prefix state")
|
| 447 |
+
if terminal:
|
| 448 |
+
# The trained GS1 reference scans context first, then the terminal token
|
| 449 |
+
# from that final state. Do not merge the terminal back into the chunk.
|
| 450 |
+
if previous is None or hidden.shape[1] != 1:
|
| 451 |
+
raise ValueError("A terminal segment requires a prefix and exactly one token")
|
| 452 |
+
return scan(module, hidden, (previous.conv, previous.recurrent), config)
|
| 453 |
+
boundary = (
|
| 454 |
+
(previous.boundary_conv, previous.boundary_recurrent)
|
| 455 |
+
if previous is not None and previous.boundary_conv is not None
|
| 456 |
+
else None
|
| 457 |
+
)
|
| 458 |
+
tail = previous.tail if previous is not None else hidden[:, :0]
|
| 459 |
+
combined = torch.cat((tail, hidden), dim=1) if tail.shape[1] else hidden
|
| 460 |
+
result = scan(module, combined, boundary, config)
|
| 461 |
+
output, final = result[:2]
|
| 462 |
+
exported = result[2] if len(result) == 3 else None
|
| 463 |
+
completed = combined.shape[1] // chunk_size * chunk_size
|
| 464 |
+
if completed:
|
| 465 |
+
if completed == combined.shape[1]:
|
| 466 |
+
boundary = final
|
| 467 |
+
elif exported is not None:
|
| 468 |
+
boundary = exported
|
| 469 |
+
else:
|
| 470 |
+
boundary = scan(module, combined[:, :completed], boundary, config)[1]
|
| 471 |
+
state = GDNState(
|
| 472 |
+
*final,
|
| 473 |
+
*(boundary if boundary is not None else (None, None)),
|
| 474 |
+
combined[:, completed:].clone(),
|
| 475 |
+
(previous.position if previous is not None else 0) + hidden.shape[1],
|
| 476 |
+
)
|
| 477 |
+
return output[:, tail.shape[1] :], state
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
class GS1PretrainedModel(Qwen3_5PreTrainedModel):
|
| 481 |
+
"""Shared initialization and strict native serialization for GS1 modules."""
|
| 482 |
+
|
| 483 |
+
config_class = GS1Config
|
| 484 |
+
base_model_prefix = ""
|
| 485 |
+
_no_split_modules = ["GS1DecoderLayer", "Qwen3_5VisionBlock"]
|
| 486 |
+
_tied_weights_keys = {}
|
| 487 |
+
|
| 488 |
+
def get_output_embeddings(self):
|
| 489 |
+
return None
|
| 490 |
+
|
| 491 |
+
def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
|
| 492 |
+
for module in self.modules():
|
| 493 |
+
if isinstance(module, GS1TextModel):
|
| 494 |
+
module.gradient_checkpointing = True
|
| 495 |
+
|
| 496 |
+
def gradient_checkpointing_disable(self):
|
| 497 |
+
for module in self.modules():
|
| 498 |
+
if isinstance(module, GS1TextModel):
|
| 499 |
+
module.gradient_checkpointing = False
|
| 500 |
+
|
| 501 |
+
def save_pretrained(self, save_directory, *, state_dict=None, **kwargs):
|
| 502 |
+
from .util import export_state
|
| 503 |
+
|
| 504 |
+
self.config.decision = self.decision_config.model_dump()
|
| 505 |
+
if hasattr(self, "calibrator"):
|
| 506 |
+
self.config.calibration = self.calibrator.config.model_dump()
|
| 507 |
+
if hasattr(self, "decision_head"):
|
| 508 |
+
self.config.head_compute_dtype = "bfloat16" if self.decision_head.bf16_compute else "float32"
|
| 509 |
+
kwargs.pop("safe_serialization", None)
|
| 510 |
+
return super().save_pretrained(
|
| 511 |
+
save_directory,
|
| 512 |
+
state_dict=export_state(self, state_dict),
|
| 513 |
+
safe_serialization=True,
|
| 514 |
+
**kwargs,
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
@classmethod
|
| 518 |
+
def from_pretrained(cls, source, *, device="cpu", dtype=None, **kwargs):
|
| 519 |
+
from .util import load_native
|
| 520 |
+
|
| 521 |
+
return load_native(source, device=device, dtype=dtype, model_class=cls, **kwargs)
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
class GS1TextModel(GS1PretrainedModel):
|
| 525 |
+
"""Native hybrid text decoder with GS1 depth routing and segmented attention."""
|
| 526 |
+
|
| 527 |
+
def __init__(self, config):
|
| 528 |
+
super().__init__(config)
|
| 529 |
+
text = config.text_config
|
| 530 |
+
self.decision_config = DecisionConfig.model_validate(config.decision)
|
| 531 |
+
self.initialization_seed = config.initialization_seed
|
| 532 |
+
self.full_layers = tuple(i for i, kind in enumerate(text.layer_types) if kind == "full_attention")
|
| 533 |
+
if text.linear_num_value_heads % text.linear_num_key_heads:
|
| 534 |
+
raise ValueError("GDN value heads must be a multiple of key heads")
|
| 535 |
+
if text.num_attention_heads % text.num_key_value_heads:
|
| 536 |
+
raise ValueError("Attention heads must be a multiple of key/value heads")
|
| 537 |
+
for selected in (self.decision_config.attnres_layers, self.decision_config.bidirectional_layers):
|
| 538 |
+
if selected is not None and not set(selected) <= set(self.full_layers):
|
| 539 |
+
raise ValueError("Selected layers must be full-attention decoder indices")
|
| 540 |
+
self.embed_tokens = nn.Embedding(text.vocab_size, text.hidden_size, text.pad_token_id)
|
| 541 |
+
self.layers = nn.ModuleList(GS1DecoderLayer(text, i) for i in range(text.num_hidden_layers))
|
| 542 |
+
self.norm = Qwen3_5RMSNorm(text.hidden_size, eps=text.rms_norm_eps)
|
| 543 |
+
self.rotary_emb = Qwen3_5TextRotaryEmbedding(text)
|
| 544 |
+
self.gradient_checkpointing = False
|
| 545 |
+
self.post_init()
|
| 546 |
+
receivers = self.decision_config.attnres_layers or self.full_layers
|
| 547 |
+
self.attnres_routers = nn.ModuleDict(
|
| 548 |
+
{
|
| 549 |
+
str(i): FADepthRouter(text.hidden_size, self.decision_config.attnres_eps)
|
| 550 |
+
for i in receivers
|
| 551 |
+
if self.decision_config.attnres
|
| 552 |
+
}
|
| 553 |
+
)
|
| 554 |
+
|
| 555 |
+
def get_input_embeddings(self):
|
| 556 |
+
return self.embed_tokens
|
| 557 |
+
|
| 558 |
+
def set_input_embeddings(self, value):
|
| 559 |
+
self.embed_tokens = value
|
| 560 |
+
|
| 561 |
+
def is_bidirectional(self, index):
|
| 562 |
+
selected = self.decision_config.bidirectional_layers
|
| 563 |
+
return self.decision_config.bidirectional and (selected is None or index in selected)
|
| 564 |
+
|
| 565 |
+
def _depth_input(self, index, carry, bank, partial):
|
| 566 |
+
if str(index) not in self.attnres_routers:
|
| 567 |
+
return carry
|
| 568 |
+
sources = bank if partial is None else (*bank, partial)
|
| 569 |
+
return self.attnres_routers[str(index)](carry, sources)
|
| 570 |
+
|
| 571 |
+
@staticmethod
|
| 572 |
+
def _advance_depth(bank, partial, mixed, ffn, full):
|
| 573 |
+
increment = mixed + ffn
|
| 574 |
+
partial = increment if partial is None else partial + increment
|
| 575 |
+
return ((*bank, partial), None) if full else (bank, partial)
|
| 576 |
+
|
| 577 |
+
def prepare_encode(self, hidden, position_ids, roles=None, *, terminal_query=False, attention_mask=None):
|
| 578 |
+
"""Validate immutable batch geometry before tensor execution or CUDA capture."""
|
| 579 |
+
batch, length = hidden.shape[:2]
|
| 580 |
+
if not batch or not length:
|
| 581 |
+
raise ValueError("GS1 expects nonempty branch rows")
|
| 582 |
+
valid = (
|
| 583 |
+
torch.ones((batch, length), dtype=torch.bool, device=hidden.device)
|
| 584 |
+
if attention_mask is None
|
| 585 |
+
else attention_mask.bool()
|
| 586 |
+
)
|
| 587 |
+
if valid.shape != (batch, length) or not valid[:, -1].all() or (valid[:, :-1] & ~valid[:, 1:]).any():
|
| 588 |
+
raise ValueError("GS1 batches require nonempty left-padded rows")
|
| 589 |
+
if terminal_query and (valid.sum(-1) < 2).any():
|
| 590 |
+
raise ValueError("A decision requires context before its terminal token")
|
| 591 |
+
if roles is None:
|
| 592 |
+
roles = torch.zeros((batch, length), dtype=torch.long, device=hidden.device)
|
| 593 |
+
elif roles.shape == (length,):
|
| 594 |
+
roles = roles[None].expand(batch, -1)
|
| 595 |
+
if roles.shape != (batch, length):
|
| 596 |
+
raise ValueError("Token roles must match the branch rows")
|
| 597 |
+
keep = valid.unsqueeze(-1)
|
| 598 |
+
valid_lengths = tuple(valid.sum(-1).tolist())
|
| 599 |
+
hidden = hidden * keep
|
| 600 |
+
masks = {}
|
| 601 |
+
eye = torch.eye(length, dtype=torch.bool, device=hidden.device)[None]
|
| 602 |
+
for bidirectional in {self.is_bidirectional(i) for i in self.full_layers}:
|
| 603 |
+
if bidirectional:
|
| 604 |
+
allowed = (roles[:, :, None] >= roles[:, None, :]) & (roles[:, None, :] != 3)
|
| 605 |
+
allowed |= eye
|
| 606 |
+
else:
|
| 607 |
+
allowed = torch.ones((batch, length, length), dtype=torch.bool, device=hidden.device).tril_()
|
| 608 |
+
allowed &= valid[:, None, :]
|
| 609 |
+
allowed |= ~valid[:, :, None] & eye
|
| 610 |
+
masks[bidirectional] = hidden.new_zeros((batch, 1, length, length)).masked_fill_(
|
| 611 |
+
~allowed[:, None], torch.finfo(hidden.dtype).min
|
| 612 |
+
)
|
| 613 |
+
positional = self.rotary_emb(hidden, position_ids)
|
| 614 |
+
cohorts = tuple(
|
| 615 |
+
(
|
| 616 |
+
length,
|
| 617 |
+
torch.tensor(
|
| 618 |
+
[i for i, value in enumerate(valid_lengths) if value == length], device=hidden.device
|
| 619 |
+
),
|
| 620 |
+
)
|
| 621 |
+
for length in sorted(set(valid_lengths))
|
| 622 |
+
)
|
| 623 |
+
return keep, valid_lengths, masks, positional, cohorts
|
| 624 |
+
|
| 625 |
+
def encode(self, hidden, position_ids, roles=None, *, terminal_query=False, attention_mask=None):
|
| 626 |
+
layout = self.prepare_encode(
|
| 627 |
+
hidden, position_ids, roles, terminal_query=terminal_query, attention_mask=attention_mask
|
| 628 |
+
)
|
| 629 |
+
return self.encode_prepared(hidden, layout, terminal_query=terminal_query)
|
| 630 |
+
|
| 631 |
+
def encode_prepared(self, hidden, layout, *, terminal_query=False, execution_policy=None):
|
| 632 |
+
"""The unchanged decoder equations over already validated, static geometry."""
|
| 633 |
+
if execution_policy is not None:
|
| 634 |
+
return execution_policy.encode(self, hidden, layout, terminal_query=terminal_query)
|
| 635 |
+
keep, valid_lengths, masks, positional, cohorts = layout
|
| 636 |
+
hidden = hidden * keep
|
| 637 |
+
bank, partial = (hidden,), None
|
| 638 |
+
|
| 639 |
+
def run(carry, layer, index, bank, partial):
|
| 640 |
+
read = self._depth_input(index, carry, bank, partial)
|
| 641 |
+
mask = masks.get(self.is_bidirectional(index))
|
| 642 |
+
output, mixed, ffn = layer(carry, positional, mask, read, terminal_query, valid_lengths, cohorts)
|
| 643 |
+
# Prefix PADs start from zero GDN state and remain zero after every
|
| 644 |
+
# layer. Valid tokens retain the ordinary convolution/time scan.
|
| 645 |
+
output, mixed, ffn = output * keep, mixed * keep, ffn * keep
|
| 646 |
+
if self.attnres_routers:
|
| 647 |
+
bank, partial = self._advance_depth(
|
| 648 |
+
bank, partial, mixed, ffn, layer.layer_type == "full_attention"
|
| 649 |
+
)
|
| 650 |
+
return output, bank, partial
|
| 651 |
+
|
| 652 |
+
for index, layer in enumerate(self.layers):
|
| 653 |
+
if self.training and self.gradient_checkpointing:
|
| 654 |
+
hidden, bank, partial = checkpoint(
|
| 655 |
+
run, hidden, layer, index, bank, partial, use_reentrant=False
|
| 656 |
+
)
|
| 657 |
+
else:
|
| 658 |
+
hidden, bank, partial = run(hidden, layer, index, bank, partial)
|
| 659 |
+
return self.norm(hidden) * keep
|
| 660 |
+
|
| 661 |
+
def segment_forward(
|
| 662 |
+
self, input_ids, start_position=0, prefix=None, *, terminal_query=False, execution_policy=None
|
| 663 |
+
):
|
| 664 |
+
"""Inference-only segment continuation, including GDN convolution/state caches."""
|
| 665 |
+
if torch.is_grad_enabled() or self.training:
|
| 666 |
+
raise ValueError("Shared segment caches require eval mode without gradients")
|
| 667 |
+
if execution_policy is not None:
|
| 668 |
+
return execution_policy.segment(
|
| 669 |
+
self, input_ids, start_position, prefix, terminal_query=terminal_query
|
| 670 |
+
)
|
| 671 |
+
hidden = self.embed_tokens(input_ids)
|
| 672 |
+
positions = torch.arange(start_position, start_position + hidden.shape[1], device=hidden.device)[
|
| 673 |
+
None
|
| 674 |
+
].expand(hidden.shape[0], -1)
|
| 675 |
+
positional = self.rotary_emb(hidden, positions)
|
| 676 |
+
bank, partial, states = (hidden,), None, []
|
| 677 |
+
for index, layer in enumerate(self.layers):
|
| 678 |
+
read = self._depth_input(index, hidden, bank, partial)
|
| 679 |
+
normalized = layer.input_layernorm(read)
|
| 680 |
+
previous = prefix[index] if prefix is not None else None
|
| 681 |
+
if layer.layer_type == "full_attention":
|
| 682 |
+
mixed, state = _attention(
|
| 683 |
+
layer.self_attn,
|
| 684 |
+
normalized,
|
| 685 |
+
positional,
|
| 686 |
+
previous,
|
| 687 |
+
bidirectional=self.is_bidirectional(index),
|
| 688 |
+
)
|
| 689 |
+
else:
|
| 690 |
+
mixed, state = _continue_gdn(
|
| 691 |
+
layer.linear_attn, normalized, previous, self.config.text_config, terminal=terminal_query
|
| 692 |
+
)
|
| 693 |
+
hidden = hidden + mixed
|
| 694 |
+
ffn = layer.mlp(layer.post_attention_layernorm(hidden))
|
| 695 |
+
hidden = hidden + ffn
|
| 696 |
+
if self.attnres_routers:
|
| 697 |
+
bank, partial = self._advance_depth(
|
| 698 |
+
bank, partial, mixed, ffn, layer.layer_type == "full_attention"
|
| 699 |
+
)
|
| 700 |
+
states.append(state)
|
| 701 |
+
return self.norm(hidden), tuple(states)
|
| 702 |
+
|
| 703 |
+
def forward(
|
| 704 |
+
self,
|
| 705 |
+
input_ids=None,
|
| 706 |
+
*,
|
| 707 |
+
inputs_embeds=None,
|
| 708 |
+
position_ids=None,
|
| 709 |
+
roles=None,
|
| 710 |
+
terminal_query=False,
|
| 711 |
+
attention_mask=None,
|
| 712 |
+
prepared_layout=None,
|
| 713 |
+
execution_policy=None,
|
| 714 |
+
):
|
| 715 |
+
if (input_ids is None) == (inputs_embeds is None):
|
| 716 |
+
raise ValueError("Specify exactly one of input_ids or inputs_embeds")
|
| 717 |
+
hidden = self.embed_tokens(input_ids) if inputs_embeds is None else inputs_embeds
|
| 718 |
+
if position_ids is None:
|
| 719 |
+
position_ids = torch.arange(hidden.shape[1], device=hidden.device)[None].expand(
|
| 720 |
+
hidden.shape[0], -1
|
| 721 |
+
)
|
| 722 |
+
output = (
|
| 723 |
+
self.encode(
|
| 724 |
+
hidden, position_ids, roles, terminal_query=terminal_query, attention_mask=attention_mask
|
| 725 |
+
)
|
| 726 |
+
if prepared_layout is None
|
| 727 |
+
else self.encode_prepared(
|
| 728 |
+
hidden, prepared_layout, terminal_query=terminal_query, execution_policy=execution_policy
|
| 729 |
+
)
|
| 730 |
+
)
|
| 731 |
+
return GS1TextOutput(last_hidden_state=output)
|
| 732 |
+
|
| 733 |
+
|
| 734 |
+
class GS1Model(GS1PretrainedModel):
|
| 735 |
+
"""Complete GS1: native text, upstream vision and one decision head."""
|
| 736 |
+
|
| 737 |
+
def __init__(self, config):
|
| 738 |
+
super().__init__(config)
|
| 739 |
+
self.initialization_seed = config.initialization_seed
|
| 740 |
+
self.language_model = GS1TextModel(config)
|
| 741 |
+
self.decision_config = self.language_model.decision_config
|
| 742 |
+
self.visual = Qwen3_5VisionModel._from_config(config.vision_config)
|
| 743 |
+
self.post_init()
|
| 744 |
+
self.decision_head = CrossAttentionDecisionHead(
|
| 745 |
+
config.text_config.hidden_size, self.decision_config, self.initialization_seed
|
| 746 |
+
)
|
| 747 |
+
self.decision_head.bf16_compute = config.head_compute_dtype == "bfloat16"
|
| 748 |
+
from .schema import Calibrator
|
| 749 |
+
|
| 750 |
+
self.calibrator = Calibrator(config.calibration)
|
| 751 |
+
|
| 752 |
+
def get_input_embeddings(self):
|
| 753 |
+
return self.language_model.get_input_embeddings()
|
| 754 |
+
|
| 755 |
+
def set_input_embeddings(self, value):
|
| 756 |
+
self.language_model.set_input_embeddings(value)
|
| 757 |
+
|
| 758 |
+
def _vision_positions(self, input_ids, types, image_grid, video_grid=None):
|
| 759 |
+
positions = torch.zeros(3, *input_ids.shape, device=input_ids.device, dtype=torch.long)
|
| 760 |
+
grids = {
|
| 761 |
+
1: iter(image_grid) if image_grid is not None else None,
|
| 762 |
+
2: iter(video_grid) if video_grid is not None else None,
|
| 763 |
+
}
|
| 764 |
+
merge = self.config.vision_config.spatial_merge_size
|
| 765 |
+
for row in range(input_ids.shape[0]):
|
| 766 |
+
begin, offset = 0, 0
|
| 767 |
+
while begin < input_ids.shape[1]:
|
| 768 |
+
kind = int(types[row, begin])
|
| 769 |
+
end = begin + 1
|
| 770 |
+
while end < input_ids.shape[1] and int(types[row, end]) == kind:
|
| 771 |
+
end += 1
|
| 772 |
+
if kind == 0:
|
| 773 |
+
piece = torch.arange(end - begin, device=input_ids.device)[None].expand(3, -1) + offset
|
| 774 |
+
offset += end - begin
|
| 775 |
+
else:
|
| 776 |
+
t, h, w = map(int, next(grids[kind]))
|
| 777 |
+
h, w = h // merge, w // merge
|
| 778 |
+
piece = torch.stack(
|
| 779 |
+
(
|
| 780 |
+
torch.full((t * h * w,), offset, device=input_ids.device),
|
| 781 |
+
torch.arange(offset, offset + h, device=input_ids.device).repeat_interleave(
|
| 782 |
+
w * t
|
| 783 |
+
),
|
| 784 |
+
torch.arange(offset, offset + w, device=input_ids.device).repeat(h * t),
|
| 785 |
+
)
|
| 786 |
+
)
|
| 787 |
+
offset += max(h, w)
|
| 788 |
+
if piece.shape[1] != end - begin:
|
| 789 |
+
raise ValueError("Vision grid and placeholder token count differ")
|
| 790 |
+
positions[:, row, begin:end] = piece
|
| 791 |
+
begin = end
|
| 792 |
+
return positions
|
| 793 |
+
|
| 794 |
+
def forward(
|
| 795 |
+
self,
|
| 796 |
+
input_ids,
|
| 797 |
+
*,
|
| 798 |
+
branch=None,
|
| 799 |
+
state_length=None,
|
| 800 |
+
branches=None,
|
| 801 |
+
state_lengths=None,
|
| 802 |
+
position_ids=None,
|
| 803 |
+
pixel_values=None,
|
| 804 |
+
image_grid_thw=None,
|
| 805 |
+
mm_token_type_ids=None,
|
| 806 |
+
attention_mask=None,
|
| 807 |
+
readout_callback=None,
|
| 808 |
+
**kwargs,
|
| 809 |
+
):
|
| 810 |
+
hidden = self.get_input_embeddings()(input_ids)
|
| 811 |
+
if pixel_values is not None:
|
| 812 |
+
features = self.visual(
|
| 813 |
+
pixel_values.to(self.visual.dtype), grid_thw=image_grid_thw, return_dict=True
|
| 814 |
+
).pooler_output.to(hidden.device, hidden.dtype)
|
| 815 |
+
mask = (input_ids == self.config.image_token_id).unsqueeze(-1).expand_as(hidden)
|
| 816 |
+
if hidden[mask].numel() != features.numel():
|
| 817 |
+
raise ValueError("Image features and placeholder tokens differ")
|
| 818 |
+
hidden = hidden.masked_scatter(mask, features)
|
| 819 |
+
if position_ids is None:
|
| 820 |
+
if mm_token_type_ids is None:
|
| 821 |
+
raise ValueError("Visual branches require modality token types")
|
| 822 |
+
position_ids = self._vision_positions(input_ids, mm_token_type_ids, image_grid_thw)
|
| 823 |
+
if position_ids is None:
|
| 824 |
+
position_ids = torch.arange(hidden.shape[1], device=hidden.device)[None].expand(
|
| 825 |
+
hidden.shape[0], -1
|
| 826 |
+
)
|
| 827 |
+
if branch is not None and branches is not None:
|
| 828 |
+
raise ValueError("Specify one branch or batched branch metadata")
|
| 829 |
+
rows = branches if branches is not None else [branch] if branch is not None else None
|
| 830 |
+
lengths = state_lengths if branches is not None else [state_length] if branch is not None else None
|
| 831 |
+
roles = None
|
| 832 |
+
starts = []
|
| 833 |
+
if rows is not None:
|
| 834 |
+
if len(rows) != input_ids.shape[0] or lengths is None or len(lengths) != len(rows):
|
| 835 |
+
raise ValueError("Compiled branch metadata differs from batch rows")
|
| 836 |
+
roles = torch.full_like(input_ids, -1)
|
| 837 |
+
for row, (item, offset) in enumerate(zip(rows, lengths, strict=True)):
|
| 838 |
+
total = offset + len(item.suffix)
|
| 839 |
+
begin = input_ids.shape[1] - total
|
| 840 |
+
if begin < 0 or (attention_mask is not None and int(attention_mask[row].sum()) != total):
|
| 841 |
+
raise ValueError("Compiled branch length differs from model input")
|
| 842 |
+
if branch is not None and begin:
|
| 843 |
+
raise ValueError("Single compiled branches must be unpadded")
|
| 844 |
+
starts.append(begin)
|
| 845 |
+
roles[row, begin : begin + offset] = 0
|
| 846 |
+
roles[row, begin + offset :] = torch.tensor(
|
| 847 |
+
[
|
| 848 |
+
1 if r == "question_context" else 2 if r == "candidate_context" else 3
|
| 849 |
+
for r in item.token_roles
|
| 850 |
+
],
|
| 851 |
+
device=hidden.device,
|
| 852 |
+
)
|
| 853 |
+
hidden = self.language_model(
|
| 854 |
+
inputs_embeds=hidden,
|
| 855 |
+
position_ids=position_ids,
|
| 856 |
+
roles=roles,
|
| 857 |
+
terminal_query=rows is not None,
|
| 858 |
+
attention_mask=attention_mask,
|
| 859 |
+
).last_hidden_state
|
| 860 |
+
logits = None
|
| 861 |
+
if rows is not None:
|
| 862 |
+
values = [
|
| 863 |
+
(readout_callback or self.readout)(hidden[row, begin:], item, offset)
|
| 864 |
+
for row, (begin, item, offset) in enumerate(zip(starts, rows, lengths, strict=True))
|
| 865 |
+
]
|
| 866 |
+
width = max(value.numel() for value in values)
|
| 867 |
+
logits = (
|
| 868 |
+
values[0]
|
| 869 |
+
if branch is not None
|
| 870 |
+
else torch.stack(
|
| 871 |
+
[F.pad(value, (0, width - value.numel()), value=-torch.inf) for value in values]
|
| 872 |
+
)
|
| 873 |
+
)
|
| 874 |
+
return GS1Output(logits=logits, last_hidden_state=hidden)
|
| 875 |
+
|
| 876 |
+
def readout(self, hidden, branch, offset, *, prepared_memory=None, state_hidden=None):
|
| 877 |
+
def pool(start, span, end):
|
| 878 |
+
content = hidden[offset + span[0] : offset + span[1]].mean(0)
|
| 879 |
+
closing = hidden[offset + end]
|
| 880 |
+
return (
|
| 881 |
+
(hidden[offset + start] + content + closing) / 3
|
| 882 |
+
if self.decision_config.bidirectional
|
| 883 |
+
else (content + closing) / 2
|
| 884 |
+
)
|
| 885 |
+
|
| 886 |
+
question = pool(branch.question_start_position, branch.question_span, branch.question_end_position)
|
| 887 |
+
options = torch.stack(
|
| 888 |
+
[
|
| 889 |
+
pool(start, span, end)
|
| 890 |
+
for start, span, end in zip(
|
| 891 |
+
branch.option_starts, branch.option_spans, branch.option_ends, strict=True
|
| 892 |
+
)
|
| 893 |
+
]
|
| 894 |
+
)
|
| 895 |
+
return self.decision_head(
|
| 896 |
+
hidden[branch.state_evidence_start : offset] if state_hidden is None else state_hidden,
|
| 897 |
+
question,
|
| 898 |
+
hidden[offset + branch.decision_position],
|
| 899 |
+
options,
|
| 900 |
+
prepared_memory=prepared_memory,
|
| 901 |
+
)
|
| 902 |
+
|
| 903 |
+
def probabilities(self, logits, candidate_mask=None, *, calibration_active=True):
|
| 904 |
+
from .schema import probabilities
|
| 905 |
+
|
| 906 |
+
return probabilities(logits, candidate_mask, self.calibrator if calibration_active else None)
|
| 907 |
+
|
| 908 |
+
def decision_compiler(self, tokenizer=None, **overrides):
|
| 909 |
+
"""Build the native prompt compiler from standard pretrained assets."""
|
| 910 |
+
from .compiler import Compiler
|
| 911 |
+
|
| 912 |
+
if tokenizer is None:
|
| 913 |
+
from transformers import AutoTokenizer
|
| 914 |
+
|
| 915 |
+
if not hasattr(self, "_decision_tokenizer"):
|
| 916 |
+
self._decision_tokenizer = AutoTokenizer.from_pretrained(
|
| 917 |
+
self._processor_source, local_files_only=True
|
| 918 |
+
)
|
| 919 |
+
tokenizer = self._decision_tokenizer
|
| 920 |
+
source = getattr(self, "_processor_source", None)
|
| 921 |
+
limits = {
|
| 922 |
+
key: self.config.inference_limits[key]
|
| 923 |
+
for key in ("max_context", "max_questions", "max_request_tokens", "max_branches")
|
| 924 |
+
}
|
| 925 |
+
for key, value in overrides.items():
|
| 926 |
+
limits[key] = min(limits[key], value)
|
| 927 |
+
compiler = Compiler(tokenizer, decision=self.decision_config, processor=source, **limits)
|
| 928 |
+
if compiler.controls != getattr(self.config, "structural_token_ids", {}):
|
| 929 |
+
raise ValueError("Structural token identity mismatch")
|
| 930 |
+
return compiler
|
| 931 |
+
|
| 932 |
+
@torch.inference_mode()
|
| 933 |
+
def decide(self, state, questions, *, tokenizer=None, precision="bf16"):
|
| 934 |
+
"""Return typed decisions using the standalone eager reference path.
|
| 935 |
+
|
| 936 |
+
This method needs only the files in this model's Hub repository. The
|
| 937 |
+
separately installed gs1 runtime provides qualified CUDA graph execution.
|
| 938 |
+
"""
|
| 939 |
+
from .compiler import branch_logits
|
| 940 |
+
from .schema import DecisionRequest, DecisionResponse, Usage, make_answer
|
| 941 |
+
|
| 942 |
+
if precision not in {"fp32", "bf16"}:
|
| 943 |
+
raise ValueError("precision must be fp32 or bf16")
|
| 944 |
+
if self.training:
|
| 945 |
+
raise ValueError("Call model.eval() before making decisions")
|
| 946 |
+
compiler = self.decision_compiler(tokenizer)
|
| 947 |
+
tokenizer = compiler.tokenizer
|
| 948 |
+
name = getattr(self.config, "_name_or_path", "") or "GS1"
|
| 949 |
+
request = DecisionRequest(state=state, questions=questions, model=name)
|
| 950 |
+
compiled = compiler.compile(request)
|
| 951 |
+
device = self.get_input_embeddings().weight.device
|
| 952 |
+
answers = {}
|
| 953 |
+
with torch.autocast(device.type, dtype=torch.bfloat16, enabled=precision == "bf16"):
|
| 954 |
+
for branch in compiled.branches:
|
| 955 |
+
logits = branch_logits(
|
| 956 |
+
self, [(compiled.state, branch)], pad_token_id=tokenizer.pad_token_id or 0
|
| 957 |
+
)[0]
|
| 958 |
+
values = self.probabilities(logits).p_final.tolist()
|
| 959 |
+
answers[branch.question_id] = make_answer(request.questions[branch.question_id], values)
|
| 960 |
+
response = DecisionResponse(
|
| 961 |
+
model=name, answers=answers, usage=Usage(input_tokens=compiled.input_tokens, output_tokens=0)
|
| 962 |
+
)
|
| 963 |
+
return response.model_dump(mode="json")
|
| 964 |
+
|
| 965 |
+
|
| 966 |
+
# Native discovery resolves the complete text/vision/decision model.
|
| 967 |
+
AutoConfig.register("gs1", GS1Config, exist_ok=True)
|
| 968 |
+
AutoModel.register(GS1Config, GS1Model, exist_ok=True)
|
| 969 |
+
|
| 970 |
+
GS1Config.register_for_auto_class("AutoConfig")
|
| 971 |
+
GS1Model.register_for_auto_class("AutoModel")
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 16777216,
|
| 4 |
+
"shortest_edge": 65536
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"image_processor_type": "Qwen2VLImageProcessorFast"
|
| 21 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Standalone default inference; no gs1 repository installation is required.
|
| 2 |
+
torch==2.9.1
|
| 3 |
+
torchvision==0.24.1
|
| 4 |
+
transformers==5.3.0
|
| 5 |
+
safetensors>=0.5,<1
|
| 6 |
+
accelerate>=1.6,<2
|
| 7 |
+
huggingface-hub>=0.30,<2
|
| 8 |
+
pydantic>=2.8,<3
|
| 9 |
+
typesafe-sdk==0.7.0
|
| 10 |
+
pillow>=11,<13
|
schema.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Typed decisions, FP32 probabilities, calibration and confidence."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from typing import Annotated, Any, Literal
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
| 10 |
+
from torch import nn
|
| 11 |
+
from typesafe_sdk import Choice as Choice
|
| 12 |
+
from typesafe_sdk import Noul as Noul
|
| 13 |
+
from typesafe_sdk import Question as Question
|
| 14 |
+
from typesafe_sdk import Score as Score
|
| 15 |
+
|
| 16 |
+
from .config import CalibrationConfig
|
| 17 |
+
|
| 18 |
+
Description = str | dict[str, JsonValue] | list[JsonValue]
|
| 19 |
+
CONFIDENCE_VERSION = "typesafe_v1"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class StrictModel(BaseModel):
|
| 23 |
+
model_config = ConfigDict(extra="forbid", strict=True)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class DecisionRequest(StrictModel):
|
| 27 |
+
state: Description
|
| 28 |
+
model: str = Field(min_length=1)
|
| 29 |
+
questions: dict[str, Question] = Field(min_length=1)
|
| 30 |
+
|
| 31 |
+
@field_validator("questions", mode="before")
|
| 32 |
+
@classmethod
|
| 33 |
+
def sdk_primitives(cls, value):
|
| 34 |
+
if not isinstance(value, dict):
|
| 35 |
+
raise ValueError("questions must be an object")
|
| 36 |
+
kinds = {"choice": Choice, "noul": Noul, "score": Score}
|
| 37 |
+
parsed = {}
|
| 38 |
+
for qid, question in value.items():
|
| 39 |
+
if isinstance(question, dict):
|
| 40 |
+
kind = question.get("type")
|
| 41 |
+
if kind not in kinds:
|
| 42 |
+
raise ValueError("Question type must be choice, noul, or score")
|
| 43 |
+
question = kinds[kind].model_validate(question, strict=True)
|
| 44 |
+
parsed[qid] = question
|
| 45 |
+
return parsed
|
| 46 |
+
|
| 47 |
+
@model_validator(mode="after")
|
| 48 |
+
def server_contract(self):
|
| 49 |
+
# Keep SDK-native primitives, but enforce the narrower public serving contract.
|
| 50 |
+
# SDK 0.7.0 permits omitted instructions and singleton Score questions.
|
| 51 |
+
for question in self.questions.values():
|
| 52 |
+
if not isinstance(question.instructions, (str, dict, list)):
|
| 53 |
+
raise ValueError("Every question requires text, object, or array instructions")
|
| 54 |
+
if question.type == "choice":
|
| 55 |
+
if not 1 <= len(question.criteria) <= 255:
|
| 56 |
+
raise ValueError("Choice requires between 1 and 255 candidates")
|
| 57 |
+
descriptions = [v for v in question.criteria.values() if v is not None]
|
| 58 |
+
elif question.type == "score":
|
| 59 |
+
if not 2 <= len(question.criteria) <= 10:
|
| 60 |
+
raise ValueError("Score requires between 2 and 10 levels")
|
| 61 |
+
descriptions = question.criteria
|
| 62 |
+
else:
|
| 63 |
+
descriptions = list(noul_descriptions(question))
|
| 64 |
+
if any(not isinstance(v, (str, dict, list)) for v in descriptions):
|
| 65 |
+
raise ValueError("Criteria descriptions must be text, objects, or arrays")
|
| 66 |
+
# Reject NaN/Infinity at every depth, including structured descriptions.
|
| 67 |
+
json.dumps(self.model_dump(mode="json"), allow_nan=False)
|
| 68 |
+
return self
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def noul_descriptions(question: Question) -> tuple[Description, Description]:
|
| 72 |
+
criteria = question.criteria or {}
|
| 73 |
+
return (
|
| 74 |
+
criteria["false"]
|
| 75 |
+
if criteria.get("false") is not None
|
| 76 |
+
else "The proposition is false under the question's stated semantics.",
|
| 77 |
+
criteria["true"]
|
| 78 |
+
if criteria.get("true") is not None
|
| 79 |
+
else "The proposition is true under the question's stated semantics.",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class Usage(StrictModel):
|
| 84 |
+
input_tokens: int = Field(ge=0)
|
| 85 |
+
output_tokens: int = Field(ge=0)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class ChoiceAnswer(StrictModel):
|
| 89 |
+
type: Literal["choice"] = "choice"
|
| 90 |
+
choice: str
|
| 91 |
+
probabilities: dict[str, float]
|
| 92 |
+
confidence: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class NoulAnswer(StrictModel):
|
| 96 |
+
type: Literal["noul"] = "noul"
|
| 97 |
+
noul: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class ScoreAnswer(StrictModel):
|
| 101 |
+
type: Literal["score"] = "score"
|
| 102 |
+
score: float = Field(ge=0, allow_inf_nan=False)
|
| 103 |
+
legend: dict[str, Description]
|
| 104 |
+
probabilities: dict[str, float]
|
| 105 |
+
confidence: float = Field(ge=0, le=1, allow_inf_nan=False)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
Answer = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")]
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
class DecisionResponse(StrictModel):
|
| 112 |
+
model: str
|
| 113 |
+
answers: dict[str, Answer]
|
| 114 |
+
usage: Usage
|
| 115 |
+
|
| 116 |
+
@model_validator(mode="after")
|
| 117 |
+
def numerical_validity(self):
|
| 118 |
+
for answer in self.answers.values():
|
| 119 |
+
if isinstance(answer, NoulAnswer):
|
| 120 |
+
continue
|
| 121 |
+
p = answer.probabilities
|
| 122 |
+
if not p or any(not math.isfinite(v) or not 0 <= v <= 1 for v in p.values()):
|
| 123 |
+
raise ValueError("Probabilities must be finite and within [0, 1]")
|
| 124 |
+
if abs(sum(p.values()) - 1) > 1e-6:
|
| 125 |
+
raise ValueError("Probabilities must sum to one")
|
| 126 |
+
if isinstance(answer, ChoiceAnswer):
|
| 127 |
+
if answer.choice not in p or p[answer.choice] != max(p.values()):
|
| 128 |
+
raise ValueError("Choice must be an argmax")
|
| 129 |
+
else:
|
| 130 |
+
if set(p) != set(answer.legend) or set(p) != {str(k) for k in range(len(p))}:
|
| 131 |
+
raise ValueError("Score indices and legend must align")
|
| 132 |
+
if abs(answer.score - sum(int(k) * v for k, v in p.items())) > 1e-6:
|
| 133 |
+
raise ValueError("Score must equal its probability-weighted level")
|
| 134 |
+
return self
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def candidate_ids(question: Question) -> list[str]:
|
| 138 |
+
if question.type == "choice":
|
| 139 |
+
return list(question.criteria)
|
| 140 |
+
if question.type == "noul":
|
| 141 |
+
return ["false", "true"]
|
| 142 |
+
return [str(k) for k in range(len(question.criteria))]
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def validate_response(request: DecisionRequest, response: DecisionResponse):
|
| 146 |
+
if response.answers.keys() != request.questions.keys():
|
| 147 |
+
raise ValueError("Response question identities do not match the request")
|
| 148 |
+
for qid, question in request.questions.items():
|
| 149 |
+
answer = response.answers[qid]
|
| 150 |
+
if answer.type != question.type:
|
| 151 |
+
raise ValueError(f"Primitive mismatch for {qid}")
|
| 152 |
+
if answer.type != "noul" and set(answer.probabilities) != set(candidate_ids(question)):
|
| 153 |
+
raise ValueError(f"Candidate mismatch for {qid}")
|
| 154 |
+
if answer.type == "score" and answer.legend != dict(zip(candidate_ids(question), question.criteria)):
|
| 155 |
+
raise ValueError(f"Score legend mismatch for {qid}")
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _bounded_confidence(value: float) -> float:
|
| 159 |
+
if not math.isfinite(value) or value < -1e-6 or value > 1 + 1e-6:
|
| 160 |
+
raise FloatingPointError("Invalid confidence")
|
| 161 |
+
return max(0.0, min(1.0, value))
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# TypeSafe formulas: https://docs.typesafe.ai/confidence
|
| 165 |
+
def noul_confidence(probability: float) -> float:
|
| 166 |
+
"""Optional TypeSafe confidence measure; Noul answers expose only p(true)."""
|
| 167 |
+
return _bounded_confidence(abs(2 * probability - 1))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def choice_confidence(probabilities: list[float]) -> float:
|
| 171 |
+
"""TypeSafe's top probability above an even split, with singleton support."""
|
| 172 |
+
n = len(probabilities)
|
| 173 |
+
if n == 1:
|
| 174 |
+
return 1.0
|
| 175 |
+
return _bounded_confidence((max(probabilities) - 1 / n) / (1 - 1 / n))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def score_confidence(probabilities: list[float]) -> float:
|
| 179 |
+
"""TypeSafe's spread around the first most likely level, in rubric order."""
|
| 180 |
+
n = len(probabilities)
|
| 181 |
+
m = probabilities.index(max(probabilities))
|
| 182 |
+
spread = sum(p * abs(i - m) for i, p in enumerate(probabilities))
|
| 183 |
+
even_spread = sum(abs(i - (n - 1) / 2) for i in range(n)) / n
|
| 184 |
+
return _bounded_confidence(max(0.0, 1 - spread / even_spread))
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def make_answer(question: Question, probabilities: list[float]) -> Any:
|
| 188 |
+
mapping = dict(zip(candidate_ids(question), probabilities, strict=True))
|
| 189 |
+
if question.type == "noul":
|
| 190 |
+
return NoulAnswer(noul=probabilities[1])
|
| 191 |
+
if question.type == "choice":
|
| 192 |
+
return ChoiceAnswer(
|
| 193 |
+
choice=max(mapping, key=mapping.get),
|
| 194 |
+
probabilities=mapping,
|
| 195 |
+
confidence=choice_confidence(probabilities),
|
| 196 |
+
)
|
| 197 |
+
return ScoreAnswer(
|
| 198 |
+
score=sum(k * p for k, p in enumerate(probabilities)),
|
| 199 |
+
legend=dict(zip(candidate_ids(question), question.criteria)),
|
| 200 |
+
probabilities=mapping,
|
| 201 |
+
confidence=score_confidence(probabilities),
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def aligned_probabilities(question: Question, answer: Answer) -> list[float]:
|
| 206 |
+
if answer.type == "noul":
|
| 207 |
+
return [1 - answer.noul, answer.noul]
|
| 208 |
+
return [answer.probabilities[k] for k in candidate_ids(question)]
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class Calibrator(nn.Module):
|
| 212 |
+
def __init__(self, config=None):
|
| 213 |
+
super().__init__()
|
| 214 |
+
self.config = CalibrationConfig.model_validate(config or {})
|
| 215 |
+
if self.config.mode == "scalar_temperature":
|
| 216 |
+
self.beta = nn.Parameter(torch.zeros(()))
|
| 217 |
+
|
| 218 |
+
def forward(self):
|
| 219 |
+
if self.config.mode == "identity":
|
| 220 |
+
return 1.0
|
| 221 |
+
if not torch.isfinite(self.beta):
|
| 222 |
+
raise FloatingPointError("Nonfinite scalar temperature")
|
| 223 |
+
return (self.config.log_temperature_bound * self.beta.float().tanh()).exp()
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
@dataclass
|
| 227 |
+
class Probabilities:
|
| 228 |
+
candidate_mask: torch.Tensor
|
| 229 |
+
log_p_raw: torch.Tensor
|
| 230 |
+
p_raw: torch.Tensor
|
| 231 |
+
entropy: torch.Tensor
|
| 232 |
+
margin: torch.Tensor
|
| 233 |
+
temperature: torch.Tensor
|
| 234 |
+
log_p_final: torch.Tensor
|
| 235 |
+
p_final: torch.Tensor
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def probabilities(logits, candidate_mask=None, calibrator=None):
|
| 239 |
+
if logits.ndim < 1 or logits.shape[-1] == 0:
|
| 240 |
+
raise ValueError("Expected a nonempty candidate axis")
|
| 241 |
+
with torch.autocast(device_type=logits.device.type, enabled=False):
|
| 242 |
+
logits = logits.float()
|
| 243 |
+
mask = torch.ones_like(logits, dtype=torch.bool) if candidate_mask is None else candidate_mask
|
| 244 |
+
if mask.dtype != torch.bool or mask.shape != logits.shape or not mask.any(-1).all():
|
| 245 |
+
raise ValueError("Candidate mask must match logits and contain nonempty support")
|
| 246 |
+
if not torch.isfinite(logits[mask]).all():
|
| 247 |
+
raise FloatingPointError("Nonfinite valid decision logits")
|
| 248 |
+
log_raw = logits.masked_fill(~mask, -torch.inf).log_softmax(-1)
|
| 249 |
+
if not torch.isfinite(log_raw[mask]).all():
|
| 250 |
+
raise FloatingPointError("Valid log probabilities exceed FP32 range")
|
| 251 |
+
raw = log_raw.exp()
|
| 252 |
+
entropy = -(raw * log_raw.masked_fill(~mask, 0)).sum(-1)
|
| 253 |
+
top = raw.topk(min(2, logits.shape[-1]), dim=-1).values
|
| 254 |
+
margin = top[..., 0] - (top[..., 1] if top.shape[-1] > 1 else 0)
|
| 255 |
+
value = calibrator() if calibrator is not None else 1.0
|
| 256 |
+
temperature = torch.ones_like(entropy) * value
|
| 257 |
+
temperature = torch.where(mask.sum(-1) == 1, torch.ones_like(temperature), temperature)
|
| 258 |
+
if calibrator is None or calibrator.config.mode == "identity":
|
| 259 |
+
final_log = log_raw
|
| 260 |
+
else:
|
| 261 |
+
final_log = (
|
| 262 |
+
(logits.masked_fill(~mask, 0) / temperature.unsqueeze(-1))
|
| 263 |
+
.masked_fill(~mask, -torch.inf)
|
| 264 |
+
.log_softmax(-1)
|
| 265 |
+
)
|
| 266 |
+
return Probabilities(mask, log_raw, raw, entropy, margin, temperature, final_log, final_log.exp())
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0579ce9ee8ce5d0c244b069842aac4f8e1e65babd8ebfe4d98fe4fbf406831f1
|
| 3 |
+
size 19990645
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"audio_bos_token": "<|audio_start|>",
|
| 4 |
+
"audio_eos_token": "<|audio_end|>",
|
| 5 |
+
"audio_token": "<|audio_pad|>",
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": null,
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"extra_special_tokens": [
|
| 12 |
+
"<state>",
|
| 13 |
+
"</state>",
|
| 14 |
+
"<question>",
|
| 15 |
+
"</question>",
|
| 16 |
+
"<option>",
|
| 17 |
+
"</option>",
|
| 18 |
+
"<|decide|>"
|
| 19 |
+
],
|
| 20 |
+
"image_token": "<|image_pad|>",
|
| 21 |
+
"is_local": true,
|
| 22 |
+
"model_max_length": 262144,
|
| 23 |
+
"model_specific_special_tokens": {
|
| 24 |
+
"audio_bos_token": "<|audio_start|>",
|
| 25 |
+
"audio_eos_token": "<|audio_end|>",
|
| 26 |
+
"audio_token": "<|audio_pad|>",
|
| 27 |
+
"image_token": "<|image_pad|>",
|
| 28 |
+
"video_token": "<|video_pad|>",
|
| 29 |
+
"vision_bos_token": "<|vision_start|>",
|
| 30 |
+
"vision_eos_token": "<|vision_end|>"
|
| 31 |
+
},
|
| 32 |
+
"pad_token": "<|endoftext|>",
|
| 33 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 34 |
+
"split_special_tokens": false,
|
| 35 |
+
"tokenizer_class": "TokenizersBackend",
|
| 36 |
+
"unk_token": null,
|
| 37 |
+
"video_token": "<|video_pad|>",
|
| 38 |
+
"vision_bos_token": "<|vision_start|>",
|
| 39 |
+
"vision_eos_token": "<|vision_end|>"
|
| 40 |
+
}
|
util.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Native checkpoint loading, serialization and deterministic identities."""
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from accelerate import init_empty_weights
|
| 9 |
+
from safetensors.torch import load_file
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def canonical(value) -> str:
|
| 13 |
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def digest(value) -> str:
|
| 17 |
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def file_hash(path: Path) -> str:
|
| 21 |
+
h = hashlib.sha256()
|
| 22 |
+
with path.open("rb") as stream:
|
| 23 |
+
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
|
| 24 |
+
h.update(chunk)
|
| 25 |
+
return h.hexdigest()
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def named_seed(seed, name):
|
| 29 |
+
return int.from_bytes(hashlib.sha256(canonical(["gs1.rng", seed, name]).encode()).digest()[:8], "big") % (
|
| 30 |
+
2**63
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _weight_files(directory):
|
| 35 |
+
directory = Path(directory)
|
| 36 |
+
index = directory / "model.safetensors.index.json"
|
| 37 |
+
if index.is_file():
|
| 38 |
+
weight_map = json.loads(index.read_text())["weight_map"]
|
| 39 |
+
files = sorted(set(weight_map.values()))
|
| 40 |
+
else:
|
| 41 |
+
files = sorted(p.name for p in directory.glob("*.safetensors"))
|
| 42 |
+
weight_map = None
|
| 43 |
+
if not files:
|
| 44 |
+
raise ValueError("Checkpoint contains no safetensors weights")
|
| 45 |
+
if any(Path(name).is_absolute() or ".." in Path(name).parts for name in files):
|
| 46 |
+
raise ValueError("Checkpoint shard is outside its directory")
|
| 47 |
+
# Hub snapshots link to shared cache blobs; validate shard names rather
|
| 48 |
+
# than rejecting those normal cache symlinks after resolving their targets.
|
| 49 |
+
return [directory / name for name in files], weight_map
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def checkpoint_files(directory):
|
| 53 |
+
"""Files that define the weights, native configuration and input encoding."""
|
| 54 |
+
directory = Path(directory)
|
| 55 |
+
names = (
|
| 56 |
+
"config.json",
|
| 57 |
+
"model.safetensors.index.json",
|
| 58 |
+
"tokenizer.json",
|
| 59 |
+
"tokenizer_config.json",
|
| 60 |
+
"special_tokens_map.json",
|
| 61 |
+
"added_tokens.json",
|
| 62 |
+
"vocab.json",
|
| 63 |
+
"vocab.txt",
|
| 64 |
+
"merges.txt",
|
| 65 |
+
"tokenizer.model",
|
| 66 |
+
"chat_template.jinja",
|
| 67 |
+
"preprocessor_config.json",
|
| 68 |
+
"video_preprocessor_config.json",
|
| 69 |
+
)
|
| 70 |
+
weights, _ = _weight_files(directory)
|
| 71 |
+
return sorted({*weights, *(directory / name for name in names if (directory / name).is_file())})
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def read_weights(directory):
|
| 75 |
+
files, weight_map = _weight_files(directory)
|
| 76 |
+
state = {}
|
| 77 |
+
for path in files:
|
| 78 |
+
shard = load_file(path, device="cpu")
|
| 79 |
+
if state.keys() & shard.keys():
|
| 80 |
+
raise ValueError("Duplicate checkpoint parameter")
|
| 81 |
+
state.update(shard)
|
| 82 |
+
if weight_map is not None and state.keys() != weight_map.keys():
|
| 83 |
+
raise ValueError("Checkpoint index differs from its parameter contents")
|
| 84 |
+
return state
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _construct(config, model_class=None):
|
| 88 |
+
from .modular_gs1 import GS1Model
|
| 89 |
+
|
| 90 |
+
# Nonpersistent rotary buffers must remain real tensors, not meta placeholders.
|
| 91 |
+
with init_empty_weights(include_buffers=False):
|
| 92 |
+
return (model_class or GS1Model)(config)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _assign(model, state):
|
| 96 |
+
expected = model.state_dict()
|
| 97 |
+
if state.keys() != expected.keys():
|
| 98 |
+
raise ValueError(
|
| 99 |
+
f"Incomplete GS1 weights: missing={sorted(expected.keys() - state.keys())}, "
|
| 100 |
+
f"unexpected={sorted(state.keys() - expected.keys())}"
|
| 101 |
+
)
|
| 102 |
+
for name, value in state.items():
|
| 103 |
+
if value.shape != expected[name].shape:
|
| 104 |
+
raise ValueError(f"GS1 parameter shape mismatch: {name}")
|
| 105 |
+
model.load_state_dict(state, strict=True, assign=True)
|
| 106 |
+
return model
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def resolve_source(
|
| 110 |
+
source, *, revision=None, local_files_only=False, cache_dir=None, token=None, force_download=False
|
| 111 |
+
):
|
| 112 |
+
"""Resolve a native directory or an immutable Hub snapshot."""
|
| 113 |
+
path = Path(source).expanduser()
|
| 114 |
+
if path.is_dir():
|
| 115 |
+
return path.resolve()
|
| 116 |
+
from huggingface_hub import snapshot_download
|
| 117 |
+
|
| 118 |
+
return Path(
|
| 119 |
+
snapshot_download(
|
| 120 |
+
str(source),
|
| 121 |
+
revision=revision,
|
| 122 |
+
local_files_only=local_files_only,
|
| 123 |
+
cache_dir=cache_dir,
|
| 124 |
+
token=token,
|
| 125 |
+
force_download=force_download,
|
| 126 |
+
allow_patterns=["*.json", "*.safetensors", "*.jinja", "*.py", "*.model", "*.txt"],
|
| 127 |
+
)
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def load_native(
|
| 132 |
+
source,
|
| 133 |
+
*,
|
| 134 |
+
device="cpu",
|
| 135 |
+
dtype=None,
|
| 136 |
+
model_class=None,
|
| 137 |
+
config=None,
|
| 138 |
+
revision=None,
|
| 139 |
+
local_files_only=False,
|
| 140 |
+
cache_dir=None,
|
| 141 |
+
token=None,
|
| 142 |
+
force_download=False,
|
| 143 |
+
**kwargs,
|
| 144 |
+
):
|
| 145 |
+
from .modular_gs1 import GS1Config
|
| 146 |
+
|
| 147 |
+
for name in (
|
| 148 |
+
"_from_auto",
|
| 149 |
+
"_from_pipeline",
|
| 150 |
+
"_commit_hash",
|
| 151 |
+
"trust_remote_code",
|
| 152 |
+
"code_revision",
|
| 153 |
+
"weights_only",
|
| 154 |
+
"use_safetensors",
|
| 155 |
+
"adapter_kwargs",
|
| 156 |
+
):
|
| 157 |
+
kwargs.pop(name, None)
|
| 158 |
+
if "torch_dtype" in kwargs:
|
| 159 |
+
if dtype is not None:
|
| 160 |
+
raise ValueError("Use only dtype or torch_dtype")
|
| 161 |
+
dtype = kwargs.pop("torch_dtype")
|
| 162 |
+
device_map = kwargs.pop("device_map", None)
|
| 163 |
+
if device_map is not None:
|
| 164 |
+
if isinstance(device_map, dict) and set(device_map) == {""}:
|
| 165 |
+
device = device_map[""]
|
| 166 |
+
elif isinstance(device_map, str) and (
|
| 167 |
+
device_map in {"cpu", "cuda"} or device_map.startswith("cuda:")
|
| 168 |
+
):
|
| 169 |
+
device = device_map
|
| 170 |
+
else:
|
| 171 |
+
raise ValueError(
|
| 172 |
+
"GS1 supports a single device; use .to(device) instead of automatic device mapping"
|
| 173 |
+
)
|
| 174 |
+
if kwargs:
|
| 175 |
+
raise TypeError(f"Unsupported GS1 loading options: {sorted(kwargs)}")
|
| 176 |
+
directory = resolve_source(
|
| 177 |
+
source,
|
| 178 |
+
revision=revision,
|
| 179 |
+
local_files_only=local_files_only,
|
| 180 |
+
cache_dir=cache_dir,
|
| 181 |
+
token=token,
|
| 182 |
+
force_download=force_download,
|
| 183 |
+
)
|
| 184 |
+
raw = json.loads((directory / "config.json").read_text())
|
| 185 |
+
if raw.get("model_type") != "gs1":
|
| 186 |
+
raise ValueError("Expected native GS1 weights")
|
| 187 |
+
config = config or GS1Config(**raw)
|
| 188 |
+
config._name_or_path = str(source)
|
| 189 |
+
model = _construct(config, model_class)
|
| 190 |
+
model = _assign(model, read_weights(directory))
|
| 191 |
+
model._processor_source = directory
|
| 192 |
+
if dtype is not None and dtype != "auto":
|
| 193 |
+
if isinstance(dtype, str):
|
| 194 |
+
dtype = getattr(torch, dtype)
|
| 195 |
+
model.to(dtype=dtype)
|
| 196 |
+
return model.to(device).eval()
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def export_state(model, state=None):
|
| 200 |
+
"""Save the native inference parameter tree without dtype conversion."""
|
| 201 |
+
state = model.state_dict() if state is None else state
|
| 202 |
+
return {name: value.detach().contiguous() for name, value in state.items()}
|
video_preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 25165824,
|
| 4 |
+
"shortest_edge": 4096
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"video_processor_type": "Qwen3VLVideoProcessor"
|
| 21 |
+
}
|