Commit ·
df03673
0
Parent(s):
Duplicate from Adaop/SGI-Preview
Browse files- .gitattributes +35 -0
- LICENSE +202 -0
- README.md +201 -0
- chat_template.jinja +52 -0
- config.json +202 -0
- configuration_ricursive.py +123 -0
- inference/server/README.md +75 -0
- inference/server/model.py +512 -0
- inference/server/ripple_format.py +288 -0
- inference/server/serve.py +68 -0
- model-00001-of-00016.safetensors +3 -0
- model-00002-of-00016.safetensors +3 -0
- model-00003-of-00016.safetensors +3 -0
- model-00004-of-00016.safetensors +3 -0
- model-00005-of-00016.safetensors +3 -0
- model-00006-of-00016.safetensors +3 -0
- model-00007-of-00016.safetensors +3 -0
- model-00008-of-00016.safetensors +3 -0
- model-00009-of-00016.safetensors +3 -0
- model-00010-of-00016.safetensors +3 -0
- model-00011-of-00016.safetensors +3 -0
- model-00012-of-00016.safetensors +3 -0
- model-00013-of-00016.safetensors +3 -0
- model-00014-of-00016.safetensors +3 -0
- model-00015-of-00016.safetensors +3 -0
- model-00016-of-00016.safetensors +3 -0
- model.safetensors.index.json +0 -0
- modeling_ricursive.py +606 -0
- special_tokens_map.json +1 -0
- tokenizer.json +0 -0
- tokenizer_config.json +190 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
---
|
| 4 |
+
# Ripple-65B Preview
|
| 5 |
+
|
| 6 |
+
<div align="center">
|
| 7 |
+
<picture>
|
| 8 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://framerusercontent.com/images/rSaJGoPP1Saln5kpScZNGCw.png" width="30%">
|
| 9 |
+
<source media="(prefers-color-scheme: light)" srcset="https://framerusercontent.com/images/3rFANSmoZuSBhjRwxZolyEU5guo.png" width="30%">
|
| 10 |
+
<img alt="Ricursive Logo" src="https://framerusercontent.com/images/3rFANSmoZuSBhjRwxZolyEU5guo.png" width="30%">
|
| 11 |
+
</picture>
|
| 12 |
+
</div>
|
| 13 |
+
|
| 14 |
+
<p align="center">
|
| 15 |
+
<a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview">BF16</a> |
|
| 16 |
+
<a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview-NVFP4">NVFP4</a> |
|
| 17 |
+
<a href="https://huggingface.co/ricursiveAI/Ripple-65B-Base-Preview">Base Model</a> |
|
| 18 |
+
<a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview-DFlash2">DFlash2</a> |
|
| 19 |
+
<a href="https://www.ricursive.com/ripple">Launch Blog</a> |
|
| 20 |
+
License: <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview/blob/main/LICENSE">Apache 2.0</a>
|
| 21 |
+
</p>
|
| 22 |
+
|
| 23 |
+
Ricursive Intelligence introduces **Ripple-65B Preview**, a 65B parameter dense Transformer model pretrained on 12 trillion tokens with support for up to 1M context length.
|
| 24 |
+
Ripple-65B Preview delivers strong results on agentic coding, scoring **80.52 on TerminalBench 2.1** and
|
| 25 |
+
surpassing significantly-larger US open-weight models such as Reflection Beam 501B (**80.1**), Meta Muse Spark Open 816B (**76.4**),
|
| 26 |
+
Thinking Machines Inkling 975B (**63.8**), and NVIDIA Nemotron 3 Ultra 550B (**56.4**).
|
| 27 |
+
|
| 28 |
+
Our Preview release includes an instruction-tuned model in BF16 and NVFP4 numerics, and a BF16 base model. **This repo contains the BF16 instruction-tuned model.**
|
| 29 |
+
|
| 30 |
+
Starting from an initial Olmo 3 baseline, we iteratively improved all aspects of the training recipe including pretraining scaling laws, model architecture, data, and posttraining stack.
|
| 31 |
+
Our final Ripple v1 model ladder improves Compute Efficiency (CE) on code by **16x** relative to the Olmo 3 ladder, with the advantage further increasing with scale up to
|
| 32 |
+
**55x** for the full hero run.
|
| 33 |
+
|
| 34 |
+
Ripple-65B was codesigned to be trained and served efficiently on NVIDIA hardware.
|
| 35 |
+
For our first model, a key objective was to make design decisions which reduce pretraining and posttraining
|
| 36 |
+
infrastructure complexity while still enabling strong reasoning and agentic performance.
|
| 37 |
+
Therefore, we chose to train a modernized dense Transformer model rather than use Mixture-of-Experts (MoE) like other labs.
|
| 38 |
+
This decision quickly paid off, and we were able to train a strong model from scratch with a small team that scaled from 1 to 5 people during development,
|
| 39 |
+
while maintaining solid MFU (40%) throughout pretraining and posttraining with simple sharding, no custom kernels, and no quantization (BF16).
|
| 40 |
+
|
| 41 |
+
Ripple-65B uses a **thin-and-deep** model aspect ratio, with a narrow d_model (5120) and 80 total layers.
|
| 42 |
+
We switch from the Olmo 3 llama-like architecture to a Gemma 3N-like architecture to improve training stability at scale.
|
| 43 |
+
To improve reasoning performance, we **overprovision the number of attention heads** while increasing the SWA-to-global layer ratio
|
| 44 |
+
and reducing SWA window size to compensate for the increased cost.
|
| 45 |
+
In addition, we added FFN activation clipping, gated attention, global NoPE, and value residuals.
|
| 46 |
+
|
| 47 |
+
The end result is a model that is especially capable at reasoning and agentic tasks given its model size, matching or exceeding other
|
| 48 |
+
US open-weight models with more than 10x the number of total parameters.
|
| 49 |
+
At only 65B total parameters, the model can fit on consumer hardware such as the NVIDIA DGX Spark.
|
| 50 |
+
To encourage local model development, we provide an NVFP4 checkpoint and colab to help you get started.
|
| 51 |
+
The model is also fast at token generation when running on datacenter GPUs, reaching 250 tok/s on 2xB200 batch=8 prior to quantization or custom kernels.
|
| 52 |
+
|
| 53 |
+
**Pre-training.** Ripple-65B Preview is pretrained from scratch on our 12T text-only predominantly-English and code corpus at 8K sequence length.
|
| 54 |
+
Ripple-65B Base Preview is a **pretrain-only checkpoint** without any midtraining or synthetic QA data.
|
| 55 |
+
|
| 56 |
+
**Post-training.** We use a standard post-training recipe consisting of midtraining, long context, SFT, long context SFT, and mixed domain RL.
|
| 57 |
+
Our Preview release has gone through relatively light posttraining, with a focus on agentic coding.
|
| 58 |
+
|
| 59 |
+
We show a brief overview of each stage below:
|
| 60 |
+
|
| 61 |
+
<div align="center">
|
| 62 |
+
|
| 63 |
+
| Stage | Tokens | Step | Seqlen | Batch Size |
|
| 64 |
+
| ----- | ------ | ------ | ---- | ----- |
|
| 65 |
+
| Pretrain | 12T | 877,500 | 8K | 1408 -> 3072 |
|
| 66 |
+
| Midtrain | 200B | 8,000 | 64K | 384 |
|
| 67 |
+
| Long Context | 6.5B | 389 | 2M | 8 |
|
| 68 |
+
| SFT | 183B | 7,274 | 128K | 192 |
|
| 69 |
+
| LC SFT | 13.5B | 802 | 2M | 8 |
|
| 70 |
+
| RL | N/A | 700 | 128K | 2048 |
|
| 71 |
+
|
| 72 |
+
</div>
|
| 73 |
+
|
| 74 |
+
## Instruct Model Benchmarks
|
| 75 |
+
|
| 76 |
+
We compare Ripple-65B Preview against currently-available US open-weight frontier models.
|
| 77 |
+
All models are tested at full BF16 precision with maximum effort levels.
|
| 78 |
+
|
| 79 |
+
Each lab typically only provides a limited subset of available benchmarks.
|
| 80 |
+
Our table below shows official model card scores where available, and we use our own in-house eval flow for missing evals (italized).
|
| 81 |
+
For SWE-bench Pro v2.0 Hard and DeepSWE 1.1, we use Terminus 2 for Ripple-65B Preview.
|
| 82 |
+
In other cases where the harness is not specified, we use OpenCode.
|
| 83 |
+
A higher number is better for all benchmarks.
|
| 84 |
+
|
| 85 |
+
Replicating benchmark scores has been challenging, particularly for modern agentic evals, due to the number of factors which can cause
|
| 86 |
+
eval scores to shift.
|
| 87 |
+
For example, we found that Terminal-Bench 2.1 scores were universally lower (with the exception of Muse Spark Open)
|
| 88 |
+
when using the official Terminus 2 harness at 128K sequence length with summarization.
|
| 89 |
+
Therefore, our results represent a best-effort attempt at a fair comparison.
|
| 90 |
+
|
| 91 |
+
<div align="center">
|
| 92 |
+
|
| 93 |
+
| Benchmark | Ripple-65B<br>Preview | Nemotron 3<br>Ultra 550B | Inkling<br>975B | Muse Spark<br>Open 816B | Reflection<br>Beam 501B |
|
| 94 |
+
|---|---|---|---|---|---|
|
| 95 |
+
| **Model** | | | | | |
|
| 96 |
+
| Architecture | Dense | MoE | MoE | MoE | MoE |
|
| 97 |
+
| Total params | 65B | 550B | **975B** | 816B | 501B |
|
| 98 |
+
| Active params | **65B** | 55B | 41B | 42B | 23B |
|
| 99 |
+
| Pre-training tokens | 12T | 20T | **45T** | N/A | 23.8T |
|
| 100 |
+
| **Knowledge** | | | | | |
|
| 101 |
+
| MMLU | 89.6 | *92.0* | *91.3* | ***92.6*** | *92.4* |
|
| 102 |
+
| MMLU-Pro | 84.2 | 86.8 | *87.3* | ***87.9*** | *76.5* |
|
| 103 |
+
| GPQA Diamond | 85.2 | 87.0 | 87.2 | 89.0 | **90.6** |
|
| 104 |
+
| **Math** | | | | | |
|
| 105 |
+
| AIME 2024 | 97.1 | *94.2* | *97.5* | ***97.9*** | *96.7* |
|
| 106 |
+
| AIME 2025 | 98.3 | *94.2* | *91.7* | ***100.0*** | *99.6* |
|
| 107 |
+
| AIME 2026 | 95.0 | *95.4* | 97.1 | ***98.3*** | 97.7 |
|
| 108 |
+
| HMMT Feb 2026 | 90.5 | *80.3* | 86.3 | ***93.9*** | ***93.9*** |
|
| 109 |
+
| **Coding & Agentic** | | | | | |
|
| 110 |
+
| Terminal-Bench 2.1 (self-reported) | **80.5** | 56.4 | 63.8 | 76.4 | 80.1 |
|
| 111 |
+
| Terminal-Bench 2.1 (Terminus 2) | **80.5** | *55.1* | *56.2* | *78.7* | *70.8* |
|
| 112 |
+
| SWE-bench Verified | 76.5 | 71.9 | 77.6 | *78.7* | **81.6** |
|
| 113 |
+
| SWE-bench Pro v1.0 | 61.42 | *46.9* | 54.3 | *58.2* | **66.1** |
|
| 114 |
+
| SWE-bench Pro v2.0 Hard | **79.1** | *30.7* | *2.0* | *52.3* | 76.5 |
|
| 115 |
+
| DeepSWE 1.1 | 44.3 | *0.2* | *1.8* | **60.0** | 43.3 |
|
| 116 |
+
| SciCode (Inspect AI) | 39.4 | 44.6 | 46.1 | **58.5** | 49.2 |
|
| 117 |
+
| τ³-bench banking | 31.27 | 22.6 | 23.7 | *14.4* | **36.1** |
|
| 118 |
+
| AutomationBench (Public) | 36.4 | *8.4* | *14.6* | *42.4* | 39.3 |
|
| 119 |
+
| MCP Atlas | 71.67 | *62.2* | 76.0 | **89.2** | 80.8 |
|
| 120 |
+
| **Safety** | | | | | |
|
| 121 |
+
| FORTRESS Adversarial | 83.8 | *79.3* | 78.0 | **94.2** | *92.1* |
|
| 122 |
+
| FORTRESS Benign | 95.5 | *89.1* | **95.9** | *82.6* | *86.4* |
|
| 123 |
+
| StrongREJECT | 97.5 | *98.2* | **98.6** | *94.7* | *95.8* |
|
| 124 |
+
| **Instruction Following** | | | | | |
|
| 125 |
+
| IFBench | 79.2 | **81.7** | 79.8 | *78.6* | 79.6 |
|
| 126 |
+
| **Long Context** | | | | | |
|
| 127 |
+
| AA-LCR | 75.0 | 65.4 | *78.3* | ***80.7*** | 78.2 |
|
| 128 |
+
|
| 129 |
+
</div>
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
## Base Model Benchmarks
|
| 133 |
+
|
| 134 |
+
We show traditional pretraining evals comparing Ripple-65B Base Preview against a selection of open-weight base models. Although we would have liked to
|
| 135 |
+
compare against the same set of models throughout the model card, many companies are now choosing to only open-source their instruction-tuned models.
|
| 136 |
+
|
| 137 |
+
With the rise of the importance of posttraining, many modern LLM recipes now blend posttraining data earlier into the training process, including during pretraining.
|
| 138 |
+
For our first model, we kept our pretraining data clean with minimal synthetic data, focusing on high quality natural data rather than aggressively targeting evals.
|
| 139 |
+
We instead added significantly more synthetic data during midtraining, which allowed us to quickly iterate on our synthetic data recipe without blocking pretraining.
|
| 140 |
+
|
| 141 |
+
Ripple-65B Base Preview is a pretrain-only checkpoint. The base model overall beats Poolside Laguna M.1,
|
| 142 |
+
and is comparable with DeepSeek V3.1 and V4 Flash on reasoning, math, and code.
|
| 143 |
+
Nemotron 3 Ultra dominates on traditional pretraining evals, likely due to their heavy use of [task-seeded synthetic data](https://huggingface.co/blog/nvidia/task-seeded-sdg).
|
| 144 |
+
However, its pretraining eval performance did not translate to modern agentic tasks, likely due to their synthetic data being out of distribution relative to these tasks.
|
| 145 |
+
|
| 146 |
+
We report official model results where available.
|
| 147 |
+
For our own model and for missing evals, we use our internal fork of [Olmes](https://github.com/allenai/olmes).
|
| 148 |
+
|
| 149 |
+
<div align="center">
|
| 150 |
+
|
| 151 |
+
| Benchmark (Metric) | # Shots | Ripple-65B<br>Preview | Laguna<br>M.1 225B | DeepSeek<br>V3.1 671B | DeepSeek<br>V4 Flash 284B | Nemotron 3<br>Ultra 550B |
|
| 152 |
+
|---|---|---|---|---|---|---|
|
| 153 |
+
| **Model** | | | | | | |
|
| 154 |
+
| Architecture | — | Dense | MoE | MoE | MoE | MoE |
|
| 155 |
+
| Total params | — | 65B | 225B | **671B** | 284B | 550B |
|
| 156 |
+
| Active params | — | **65B** | 23B | 37B | 13B | 55B |
|
| 157 |
+
| Training tokens (T) | — | 12T | 30T+ | 15.6T | **32T+** | 20T |
|
| 158 |
+
| **World Knowledge** | | | | | | |
|
| 159 |
+
| MMLU (MC acc) | 5-shot | *83.7* | *76.6* | *88.3* | 88.7 | **89.1** |
|
| 160 |
+
| MMLU-Pro (CoT EM) | 5-shot | *61.5* | *52.2* | *57.8* | 68.3 | **79.1** |
|
| 161 |
+
| AGIEval-En (acc) | 3-5-shot | *74.4* | *66.2* | *81.9* | **83.9** | 78.7 |
|
| 162 |
+
| GPQA main (CoT EM) | 5-shot | *37.7* | *31.7* | *30.8* | *44.0* | ***49.6*** |
|
| 163 |
+
| GPQA-Diamond (CoT EM) | 5-shot | *38.4* | *27.8* | *37.4* | *44.4* | ***53.0*** |
|
| 164 |
+
| C-Eval (MC acc) | 5-shot | *69.6* | *59.3* | *90.7* | **92.1** | *82.2* |
|
| 165 |
+
| SuperGPQA (MC acc) | 5-shot | *34.2* | *24.5* | *43.1* | 46.5 | ***52.9*** |
|
| 166 |
+
| SimpleQA-Verified (EM) | 25-shot | *10.1* | *10.8* | *17.3* | **30.1** | *27.1* |
|
| 167 |
+
| **Language & Reasoning** | | | | | | |
|
| 168 |
+
| BBH (CoT EM) | 3-shot | *85.6* | *79.6* | *87.2* | 86.9 | ***88.0*** |
|
| 169 |
+
| BBEH (EM) | 1-shot | *17.9* | *14.2* | *18.4* | **25.4** | *21.2* |
|
| 170 |
+
| DROP (F1) | 1-shot | *77.8* | *66.8* | *81.2* | **88.6** | *87.6* |
|
| 171 |
+
| HellaSwag (acc_norm) | 0-shot | *84.6* | *86.2* | *86.8* | 85.7 | ***89.1*** |
|
| 172 |
+
| HellaSwag (acc_norm) | 10-shot | *87.2* | *88.1* | *89.4* | *88.8* | **90.5** |
|
| 173 |
+
| ARC-Challenge (MC acc) | 25-shot | *95.7* | *90.8* | *95.1* | *95.8* | **97.4** |
|
| 174 |
+
| OpenBookQA (acc_norm) | 0-shot | *47.4* | *48.6* | ***50.4*** | *48.0* | 48.6 |
|
| 175 |
+
| PIQA (acc_norm) | 0-shot | *83.6* | *84.8* | ***85.2*** | *84.7* | 83.8 |
|
| 176 |
+
| WinoGrande (acc) | 5-shot | ***83.4*** | *83.2* | *83.2* | *81.4* | 79.3 |
|
| 177 |
+
| **Code & Math** | | | | | | |
|
| 178 |
+
| HumanEval (pass@1, n=20, T=0.2) | 0-shot | *70.9* | *83.4* | *69.8* | 69.5 | **83.8** |
|
| 179 |
+
| MBPP Sanitized (pass@1, n=32, T=0.2) | 3-shot | *81.5* | *81.1* | *75.2* | *74.6* | **86.0** |
|
| 180 |
+
| BigCodeBench (pass@1, n=5, T=0.8) | 3-shot | *49.2* | *50.5* | ***59.9*** | 56.8 | *55.6* |
|
| 181 |
+
| GSM8K (CoT EM) | 8-shot | *88.5* | *80.4* | *79.6* | **90.8** | 88.1 |
|
| 182 |
+
| GSM1K (CoT EM) | 8-shot | *88.4* | *79.7* | *81.8* | *83.2* | ***90.2*** |
|
| 183 |
+
| MATH (flex EM) | 4-shot | *59.1* | *52.5* | *59.7* | 57.4 | **82.0** |
|
| 184 |
+
| MGSM (CoT EM, 11-lang avg) | 8-shot | *78.2* | *65.4* | *81.7* | 85.7 | **87.7** |
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
</div>
|
| 188 |
+
|
| 189 |
+
## Citation
|
| 190 |
+
|
| 191 |
+
```bibtex
|
| 192 |
+
@misc{ricursive2026ripple65b,
|
| 193 |
+
title={Ripple-65B: Pushing the Frontier of Local Intelligence},
|
| 194 |
+
author={Ricursive Intelligence},
|
| 195 |
+
year={2026},
|
| 196 |
+
}
|
| 197 |
+
```
|
| 198 |
+
|
| 199 |
+
## Contact
|
| 200 |
+
|
| 201 |
+
For questions and comments, please contact us at [info@ricursive.com](mailto:info@ricursive.com). To learn more, please visit our [website](https://www.ricursive.com).
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 4 |
+
{{- messages[0]['content'] + '\n\n' }}
|
| 5 |
+
{%- endif %}
|
| 6 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
|
| 7 |
+
{%- for tool in tools %}
|
| 8 |
+
{{- "\n" }}
|
| 9 |
+
{{- tool | tojson }}
|
| 10 |
+
{%- endfor %}
|
| 11 |
+
{{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
|
| 12 |
+
{%- else %}
|
| 13 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 14 |
+
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
| 15 |
+
{%- endif %}
|
| 16 |
+
{%- endif %}
|
| 17 |
+
{%- for message in messages %}
|
| 18 |
+
{%- set content = message.content or '' %}
|
| 19 |
+
{%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
|
| 20 |
+
{%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
|
| 21 |
+
{%- endif %}
|
| 22 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
| 23 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 24 |
+
{%- elif message.role == "assistant" %}
|
| 25 |
+
{{- '<|im_start|>' + message.role }}
|
| 26 |
+
{%- if content %}
|
| 27 |
+
{{- '\n' + content }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{%- for tool_call in message.tool_calls %}
|
| 30 |
+
{%- if tool_call.function is defined %}
|
| 31 |
+
{%- set tool_call = tool_call.function %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '\n<tool_call>\n{"name": ' }}
|
| 34 |
+
{{- tool_call.name | tojson }}
|
| 35 |
+
{{- ', "arguments": ' }}
|
| 36 |
+
{{- tool_call.arguments | tojson }}
|
| 37 |
+
{{- '}\n</tool_call>' }}
|
| 38 |
+
{%- endfor %}
|
| 39 |
+
{{- '<|im_end|>\n' }}
|
| 40 |
+
{%- elif message.role == "tool" %}
|
| 41 |
+
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
| 42 |
+
{{- '<|im_start|>user' }}
|
| 43 |
+
{%- endif %}
|
| 44 |
+
{{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
|
| 45 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 46 |
+
{{- '<|im_end|>\n' }}
|
| 47 |
+
{%- endif %}
|
| 48 |
+
{%- endif %}
|
| 49 |
+
{%- endfor %}
|
| 50 |
+
{%- if add_generation_prompt %}
|
| 51 |
+
{{- '<|im_start|>assistant\n' }}
|
| 52 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"RicursiveForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "configuration_ricursive.RicursiveConfig",
|
| 7 |
+
"AutoModel": "modeling_ricursive.RicursiveModel",
|
| 8 |
+
"AutoModelForCausalLM": "modeling_ricursive.RicursiveForCausalLM"
|
| 9 |
+
},
|
| 10 |
+
"bos_token_id": 100257,
|
| 11 |
+
"embed_scale": 71.55417527999327,
|
| 12 |
+
"eos_token_id": [
|
| 13 |
+
100265,
|
| 14 |
+
100257
|
| 15 |
+
],
|
| 16 |
+
"ffn_clamp_threshold": 20.0,
|
| 17 |
+
"gate_full_precision": true,
|
| 18 |
+
"head_dim": 128,
|
| 19 |
+
"hidden_act": "silu",
|
| 20 |
+
"hidden_size": 5120,
|
| 21 |
+
"hybrid_layer_pattern": [
|
| 22 |
+
1,
|
| 23 |
+
1,
|
| 24 |
+
1,
|
| 25 |
+
1,
|
| 26 |
+
0,
|
| 27 |
+
1,
|
| 28 |
+
1,
|
| 29 |
+
1,
|
| 30 |
+
1,
|
| 31 |
+
0,
|
| 32 |
+
1,
|
| 33 |
+
1,
|
| 34 |
+
1,
|
| 35 |
+
1,
|
| 36 |
+
0,
|
| 37 |
+
1,
|
| 38 |
+
1,
|
| 39 |
+
1,
|
| 40 |
+
1,
|
| 41 |
+
0,
|
| 42 |
+
1,
|
| 43 |
+
1,
|
| 44 |
+
1,
|
| 45 |
+
1,
|
| 46 |
+
0,
|
| 47 |
+
1,
|
| 48 |
+
1,
|
| 49 |
+
1,
|
| 50 |
+
1,
|
| 51 |
+
0,
|
| 52 |
+
1,
|
| 53 |
+
1,
|
| 54 |
+
1,
|
| 55 |
+
1,
|
| 56 |
+
0,
|
| 57 |
+
1,
|
| 58 |
+
1,
|
| 59 |
+
1,
|
| 60 |
+
1,
|
| 61 |
+
0,
|
| 62 |
+
1,
|
| 63 |
+
1,
|
| 64 |
+
1,
|
| 65 |
+
1,
|
| 66 |
+
0,
|
| 67 |
+
1,
|
| 68 |
+
1,
|
| 69 |
+
1,
|
| 70 |
+
1,
|
| 71 |
+
0,
|
| 72 |
+
1,
|
| 73 |
+
1,
|
| 74 |
+
1,
|
| 75 |
+
1,
|
| 76 |
+
0,
|
| 77 |
+
1,
|
| 78 |
+
1,
|
| 79 |
+
1,
|
| 80 |
+
1,
|
| 81 |
+
0,
|
| 82 |
+
1,
|
| 83 |
+
1,
|
| 84 |
+
1,
|
| 85 |
+
1,
|
| 86 |
+
0,
|
| 87 |
+
1,
|
| 88 |
+
1,
|
| 89 |
+
1,
|
| 90 |
+
1,
|
| 91 |
+
0,
|
| 92 |
+
1,
|
| 93 |
+
1,
|
| 94 |
+
1,
|
| 95 |
+
1,
|
| 96 |
+
0,
|
| 97 |
+
1,
|
| 98 |
+
1,
|
| 99 |
+
1,
|
| 100 |
+
1,
|
| 101 |
+
0
|
| 102 |
+
],
|
| 103 |
+
"intermediate_size": 40960,
|
| 104 |
+
"is_hybrid_swa": true,
|
| 105 |
+
"layer_types": [
|
| 106 |
+
"sliding_attention",
|
| 107 |
+
"sliding_attention",
|
| 108 |
+
"sliding_attention",
|
| 109 |
+
"sliding_attention",
|
| 110 |
+
"full_attention",
|
| 111 |
+
"sliding_attention",
|
| 112 |
+
"sliding_attention",
|
| 113 |
+
"sliding_attention",
|
| 114 |
+
"sliding_attention",
|
| 115 |
+
"full_attention",
|
| 116 |
+
"sliding_attention",
|
| 117 |
+
"sliding_attention",
|
| 118 |
+
"sliding_attention",
|
| 119 |
+
"sliding_attention",
|
| 120 |
+
"full_attention",
|
| 121 |
+
"sliding_attention",
|
| 122 |
+
"sliding_attention",
|
| 123 |
+
"sliding_attention",
|
| 124 |
+
"sliding_attention",
|
| 125 |
+
"full_attention",
|
| 126 |
+
"sliding_attention",
|
| 127 |
+
"sliding_attention",
|
| 128 |
+
"sliding_attention",
|
| 129 |
+
"sliding_attention",
|
| 130 |
+
"full_attention",
|
| 131 |
+
"sliding_attention",
|
| 132 |
+
"sliding_attention",
|
| 133 |
+
"sliding_attention",
|
| 134 |
+
"sliding_attention",
|
| 135 |
+
"full_attention",
|
| 136 |
+
"sliding_attention",
|
| 137 |
+
"sliding_attention",
|
| 138 |
+
"sliding_attention",
|
| 139 |
+
"sliding_attention",
|
| 140 |
+
"full_attention",
|
| 141 |
+
"sliding_attention",
|
| 142 |
+
"sliding_attention",
|
| 143 |
+
"sliding_attention",
|
| 144 |
+
"sliding_attention",
|
| 145 |
+
"full_attention",
|
| 146 |
+
"sliding_attention",
|
| 147 |
+
"sliding_attention",
|
| 148 |
+
"sliding_attention",
|
| 149 |
+
"sliding_attention",
|
| 150 |
+
"full_attention",
|
| 151 |
+
"sliding_attention",
|
| 152 |
+
"sliding_attention",
|
| 153 |
+
"sliding_attention",
|
| 154 |
+
"sliding_attention",
|
| 155 |
+
"full_attention",
|
| 156 |
+
"sliding_attention",
|
| 157 |
+
"sliding_attention",
|
| 158 |
+
"sliding_attention",
|
| 159 |
+
"sliding_attention",
|
| 160 |
+
"full_attention",
|
| 161 |
+
"sliding_attention",
|
| 162 |
+
"sliding_attention",
|
| 163 |
+
"sliding_attention",
|
| 164 |
+
"sliding_attention",
|
| 165 |
+
"full_attention",
|
| 166 |
+
"sliding_attention",
|
| 167 |
+
"sliding_attention",
|
| 168 |
+
"sliding_attention",
|
| 169 |
+
"sliding_attention",
|
| 170 |
+
"full_attention",
|
| 171 |
+
"sliding_attention",
|
| 172 |
+
"sliding_attention",
|
| 173 |
+
"sliding_attention",
|
| 174 |
+
"sliding_attention",
|
| 175 |
+
"full_attention",
|
| 176 |
+
"sliding_attention",
|
| 177 |
+
"sliding_attention",
|
| 178 |
+
"sliding_attention",
|
| 179 |
+
"sliding_attention",
|
| 180 |
+
"full_attention",
|
| 181 |
+
"sliding_attention",
|
| 182 |
+
"sliding_attention",
|
| 183 |
+
"sliding_attention",
|
| 184 |
+
"sliding_attention",
|
| 185 |
+
"full_attention"
|
| 186 |
+
],
|
| 187 |
+
"max_position_embeddings": 2097152,
|
| 188 |
+
"model_type": "ricursive",
|
| 189 |
+
"num_attention_heads": 80,
|
| 190 |
+
"num_hidden_layers": 80,
|
| 191 |
+
"num_key_value_heads": 8,
|
| 192 |
+
"pad_token_id": 100277,
|
| 193 |
+
"rms_norm_eps": 1e-06,
|
| 194 |
+
"rope_theta": 10000.0,
|
| 195 |
+
"sliding_window": 1024,
|
| 196 |
+
"tie_word_embeddings": false,
|
| 197 |
+
"transformers_version": "4.57.6",
|
| 198 |
+
"use_cache": true,
|
| 199 |
+
"use_gated_attention": true,
|
| 200 |
+
"use_valres": true,
|
| 201 |
+
"vocab_size": 100352
|
| 202 |
+
}
|
configuration_ricursive.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HuggingFace PretrainedConfig for the Ricursive transformer architecture."""
|
| 2 |
+
|
| 3 |
+
from transformers import PretrainedConfig
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class RicursiveConfig(PretrainedConfig):
|
| 7 |
+
"""
|
| 8 |
+
Configuration for the Ricursive transformer.
|
| 9 |
+
|
| 10 |
+
Key architectural features:
|
| 11 |
+
- PeriNorm blocks (4 layer norms per block instead of 2)
|
| 12 |
+
- Gated attention with elementwise sigmoid gate
|
| 13 |
+
- Mixed local/global layers: local layers use sliding window + RoPE,
|
| 14 |
+
global layers (every 5th) use full attention with no RoPE
|
| 15 |
+
- Per-head Q/K RMSNorm
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
model_type = "ricursive"
|
| 19 |
+
|
| 20 |
+
def __init__(
|
| 21 |
+
self,
|
| 22 |
+
vocab_size=100352,
|
| 23 |
+
hidden_size=3072,
|
| 24 |
+
intermediate_size=24576,
|
| 25 |
+
num_hidden_layers=50,
|
| 26 |
+
num_attention_heads=48,
|
| 27 |
+
num_key_value_heads=8,
|
| 28 |
+
head_dim=128,
|
| 29 |
+
max_position_embeddings=2048,
|
| 30 |
+
hidden_act="silu",
|
| 31 |
+
rms_norm_eps=1e-6,
|
| 32 |
+
rope_theta=10000.0,
|
| 33 |
+
sliding_window=1024,
|
| 34 |
+
layer_types=None,
|
| 35 |
+
use_gated_attention=True,
|
| 36 |
+
gate_full_precision=True,
|
| 37 |
+
use_valres=False,
|
| 38 |
+
ffn_clamp_threshold=None,
|
| 39 |
+
embed_scale=None,
|
| 40 |
+
tie_word_embeddings=False,
|
| 41 |
+
**kwargs,
|
| 42 |
+
):
|
| 43 |
+
self.vocab_size = vocab_size
|
| 44 |
+
self.hidden_size = hidden_size
|
| 45 |
+
self.intermediate_size = intermediate_size
|
| 46 |
+
self.num_hidden_layers = num_hidden_layers
|
| 47 |
+
self.num_attention_heads = num_attention_heads
|
| 48 |
+
self.num_key_value_heads = num_key_value_heads
|
| 49 |
+
self.head_dim = head_dim
|
| 50 |
+
self.max_position_embeddings = max_position_embeddings
|
| 51 |
+
self.hidden_act = hidden_act
|
| 52 |
+
self.rms_norm_eps = rms_norm_eps
|
| 53 |
+
self.rope_theta = rope_theta
|
| 54 |
+
self.sliding_window = sliding_window
|
| 55 |
+
self.use_gated_attention = use_gated_attention
|
| 56 |
+
self.gate_full_precision = gate_full_precision
|
| 57 |
+
self.use_valres = use_valres
|
| 58 |
+
self.ffn_clamp_threshold = ffn_clamp_threshold
|
| 59 |
+
# Hybrid-SWA metadata is derived below. Serialized copies must not override the
|
| 60 |
+
# architecture encoded by layer_types.
|
| 61 |
+
kwargs.pop("is_hybrid_swa", None)
|
| 62 |
+
kwargs.pop("hybrid_layer_pattern", None)
|
| 63 |
+
# Embedding scale convention (see modeling_ricursive.RicursiveModel.forward):
|
| 64 |
+
# - non-folded checkpoints set embed_scale (e.g. sqrt(hidden_size)) and the model
|
| 65 |
+
# scales the embeddings at runtime;
|
| 66 |
+
# - folded checkpoints bake the scale into embed_tokens.weight and leave this None.
|
| 67 |
+
# Either way the SGLang native port (sglang_ricursive) reads this same key, so the
|
| 68 |
+
# served and trained forwards agree.
|
| 69 |
+
self.embed_scale = embed_scale
|
| 70 |
+
|
| 71 |
+
# Per-layer type: "sliding_attention" (window + RoPE) or "full_attention" (no RoPE)
|
| 72 |
+
# These names follow the HuggingFace PretrainedConfig layer_types validation.
|
| 73 |
+
if layer_types is not None:
|
| 74 |
+
self.layer_types = layer_types
|
| 75 |
+
else:
|
| 76 |
+
# Default: every 5th layer is full_attention (indices 4, 9, 14, ...)
|
| 77 |
+
self.layer_types = [
|
| 78 |
+
"full_attention" if i % 5 == 4 else "sliding_attention"
|
| 79 |
+
for i in range(num_hidden_layers)
|
| 80 |
+
]
|
| 81 |
+
if len(self.layer_types) != num_hidden_layers:
|
| 82 |
+
raise ValueError(
|
| 83 |
+
f"layer_types has {len(self.layer_types)} entries, expected {num_hidden_layers}"
|
| 84 |
+
)
|
| 85 |
+
invalid_layer_types = set(self.layer_types) - {
|
| 86 |
+
"sliding_attention",
|
| 87 |
+
"full_attention",
|
| 88 |
+
}
|
| 89 |
+
if invalid_layer_types:
|
| 90 |
+
raise ValueError(f"unsupported Ricursive layer_types: {sorted(invalid_layer_types)}")
|
| 91 |
+
|
| 92 |
+
# SGLang's compact hybrid KV allocator uses 1 for sliding attention and 0 for full
|
| 93 |
+
# attention. Keep these fields fully derived so they cannot drift from model semantics.
|
| 94 |
+
self.hybrid_layer_pattern = [
|
| 95 |
+
1 if layer_type == "sliding_attention" else 0 for layer_type in self.layer_types
|
| 96 |
+
]
|
| 97 |
+
self.is_hybrid_swa = 0 in self.hybrid_layer_pattern and 1 in self.hybrid_layer_pattern
|
| 98 |
+
|
| 99 |
+
# Standard HF config fields expected by modeling code
|
| 100 |
+
self.use_cache = kwargs.pop("use_cache", True)
|
| 101 |
+
|
| 102 |
+
self.base_model_tp_plan = {
|
| 103 |
+
"layers.*.self_attn.q_proj": "colwise",
|
| 104 |
+
"layers.*.self_attn.k_proj": "colwise",
|
| 105 |
+
"layers.*.self_attn.v_proj": "colwise",
|
| 106 |
+
"layers.*.self_attn.o_proj": "rowwise",
|
| 107 |
+
"layers.*.self_attn.g_proj": "colwise",
|
| 108 |
+
"layers.*.mlp.gate_proj": "colwise",
|
| 109 |
+
"layers.*.mlp.up_proj": "colwise",
|
| 110 |
+
"layers.*.mlp.down_proj": "rowwise",
|
| 111 |
+
}
|
| 112 |
+
self.auto_map = {
|
| 113 |
+
"AutoConfig": "configuration_ricursive.RicursiveConfig",
|
| 114 |
+
"AutoModel": "modeling_ricursive.RicursiveModel",
|
| 115 |
+
"AutoModelForCausalLM": "modeling_ricursive.RicursiveForCausalLM",
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
| 119 |
+
|
| 120 |
+
# Must be set AFTER super().__init__ — HF's PretrainedConfig.__init__
|
| 121 |
+
# pops "architectures" from kwargs (default None) and assigns it to
|
| 122 |
+
# self.architectures, which would otherwise overwrite our value.
|
| 123 |
+
self.architectures = ["RicursiveForCausalLM"]
|
inference/server/README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Ripple on B200
|
| 2 |
+
|
| 3 |
+
BF16 Ripple-65B + native DFlash2 on two NVIDIA B200 GPUs, using **SGLang 0.5.19**.
|
| 4 |
+
Four files here: this README, `model.py`, `ripple_format.py`, and `serve.py`. Both
|
| 5 |
+
inference folders share `chat_template.jinja` at the repository root; keep that layout.
|
| 6 |
+
Uses the tools-aware ChatML template from our evals.
|
| 7 |
+
|
| 8 |
+
Requires Linux, CUDA Toolkit 13, a C++ compiler, and [uv](https://docs.astral.sh/uv/).
|
| 9 |
+
Use complete, unquantized BF16 target and matching `DFlash2DraftModel` checkpoints.
|
| 10 |
+
Run the commands below from `inference/server`.
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
export CUDA_HOME=/usr/local/cuda-13.0
|
| 14 |
+
export PATH="$CUDA_HOME/bin:$PATH"
|
| 15 |
+
|
| 16 |
+
CUDA_VISIBLE_DEVICES=0,1 uv run --script serve.py \
|
| 17 |
+
--model-path /path/to/target \
|
| 18 |
+
--speculative-draft-model-path /path/to/draft
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
Dependencies install automatically. The server listens on `127.0.0.1:30000`.
|
| 22 |
+
It uses the checkpoint's tokenizer and the repository-root chat template.
|
| 23 |
+
|
| 24 |
+
```bash
|
| 25 |
+
curl http://127.0.0.1:30000/v1/chat/completions \
|
| 26 |
+
-H 'Content-Type: application/json' \
|
| 27 |
+
-d '{"model":"ripple","messages":[{"role":"user","content":"Hello"}],"max_tokens":128}'
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
Defaults: TP=2, BF16 weights and KV cache, TRT-LLM target attention, Triton draft
|
| 31 |
+
attention, 16 draft tokens, and a **1Mi-token target context** (`--context-length 1048576`,
|
| 32 |
+
including prompt and generated tokens). Standard SGLang CLI flags apply.
|
| 33 |
+
The DFlash2 draft's last layer is global attention without RoPE. It was trained on
|
| 34 |
+
16Ki-token rows, so the draft keeps a **16Ki-token window** (`--speculative-draft-window-size 16384`);
|
| 35 |
+
smaller windows are refused.
|
| 36 |
+
The launcher defaults `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1` so stock SGLang can
|
| 37 |
+
extend the draft's positional cache beyond its 128Ki checkpoint limit. This does not
|
| 38 |
+
increase the draft attention window. The target checkpoint supports up to 2Mi positions.
|
| 39 |
+
For four GPUs, run a second copy with `CUDA_VISIBLE_DEVICES=2,3` and `--port 30001`.
|
| 40 |
+
This gives two independent replicas, each using two GPUs. Send eight concurrent
|
| 41 |
+
requests to each port for sixteen total.
|
| 42 |
+
|
| 43 |
+
The launcher enables Ripple's parsers from `ripple_format.py` (`--reasoning-parser ripple`
|
| 44 |
+
and `--tool-call-parser ripple`). Responses separate `message.reasoning_content`,
|
| 45 |
+
final-answer `message.content`, and structured `message.tool_calls`. Streaming uses the
|
| 46 |
+
corresponding `delta` fields. Allow enough output tokens for both reasoning and the answer.
|
| 47 |
+
|
| 48 |
+
Pass OpenAI-style `tools` and `tool_choice: "auto"` to request calls:
|
| 49 |
+
|
| 50 |
+
```json
|
| 51 |
+
{
|
| 52 |
+
"model": "ripple",
|
| 53 |
+
"messages": [{"role": "user", "content": "Use add_numbers to calculate 23 + 19."}],
|
| 54 |
+
"tools": [{"type": "function", "function": {
|
| 55 |
+
"name": "add_numbers", "description": "Add two integers.", "strict": true,
|
| 56 |
+
"parameters": {"type": "object", "properties": {
|
| 57 |
+
"a": {"type": "integer"}, "b": {"type": "integer"}
|
| 58 |
+
}, "required": ["a", "b"], "additionalProperties": false}
|
| 59 |
+
}}],
|
| 60 |
+
"tool_choice": "auto",
|
| 61 |
+
"temperature": 0,
|
| 62 |
+
"max_tokens": 1024
|
| 63 |
+
}
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
Your application executes the requested function. Append the returned assistant
|
| 67 |
+
message unchanged, then a `tool` message containing its `tool_call_id` and a string
|
| 68 |
+
`content` with the result, and send the history again. The template re-renders an
|
| 69 |
+
unchanged message to exactly the tokens the model generated. Previous
|
| 70 |
+
`reasoning_content` is retained by default (`chat_template_kwargs.clear_thinking=false`).
|
| 71 |
+
The template does not implement a thinking on/off switch or reasoning-effort levels.
|
| 72 |
+
|
| 73 |
+
For `tool_choice: "required"`, use `function.strict=true` and
|
| 74 |
+
`parameters.additionalProperties=false`, as above. Non-strict required calls
|
| 75 |
+
produced extra argument fields in validation with this SGLang version.
|
inference/server/model.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ricursive transformer implemented with SGLang layers.
|
| 2 |
+
BF16 weights with native DFlash2 feature capture.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Iterable, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from sglang.srt.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
| 9 |
+
from sglang.srt.layers.activation import SiluAndMul
|
| 10 |
+
from sglang.srt.layers.layernorm import RMSNorm
|
| 11 |
+
from sglang.srt.layers.linear import (
|
| 12 |
+
ColumnParallelLinear,
|
| 13 |
+
MergedColumnParallelLinear,
|
| 14 |
+
RowParallelLinear,
|
| 15 |
+
)
|
| 16 |
+
from sglang.srt.layers.logits_processor import LogitsProcessor
|
| 17 |
+
from sglang.srt.layers.radix_attention import RadixAttention
|
| 18 |
+
from sglang.srt.layers.rotary_embedding import get_rope
|
| 19 |
+
from sglang.srt.layers.vocab_parallel_embedding import (
|
| 20 |
+
ParallelLMHead,
|
| 21 |
+
VocabParallelEmbedding,
|
| 22 |
+
)
|
| 23 |
+
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
| 24 |
+
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
| 25 |
+
from sglang.srt.utils import add_prefix, make_layers
|
| 26 |
+
from torch import nn
|
| 27 |
+
from transformers import PretrainedConfig
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def rotary_cache_length(config) -> int:
|
| 31 |
+
"""Include DFlash lookahead beyond the admitted context boundary."""
|
| 32 |
+
from sglang.srt.server_args import get_global_server_args
|
| 33 |
+
|
| 34 |
+
args = get_global_server_args()
|
| 35 |
+
native = int(args.context_length or config.max_position_embeddings)
|
| 36 |
+
return native + (
|
| 37 |
+
int(args.speculative_num_draft_tokens) if args.speculative_algorithm == "DFLASH" else 0
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class RicursiveMLP(nn.Module):
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
hidden_size: int,
|
| 45 |
+
intermediate_size: int,
|
| 46 |
+
activation_clamp: Optional[float] = None,
|
| 47 |
+
prefix: str = "",
|
| 48 |
+
) -> None:
|
| 49 |
+
super().__init__()
|
| 50 |
+
self.activation_clamp = activation_clamp
|
| 51 |
+
self.gate_up_proj = MergedColumnParallelLinear(
|
| 52 |
+
hidden_size,
|
| 53 |
+
[intermediate_size] * 2,
|
| 54 |
+
bias=False,
|
| 55 |
+
prefix=add_prefix("gate_up_proj", prefix),
|
| 56 |
+
)
|
| 57 |
+
self.down_proj = RowParallelLinear(
|
| 58 |
+
intermediate_size,
|
| 59 |
+
hidden_size,
|
| 60 |
+
bias=False,
|
| 61 |
+
prefix=add_prefix("down_proj", prefix),
|
| 62 |
+
)
|
| 63 |
+
self.act_fn = SiluAndMul()
|
| 64 |
+
|
| 65 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 66 |
+
gate_up, _ = self.gate_up_proj(x)
|
| 67 |
+
if self.activation_clamp is not None:
|
| 68 |
+
gate, up = gate_up.chunk(2, dim=-1)
|
| 69 |
+
gate = gate.clamp(max=self.activation_clamp)
|
| 70 |
+
up = up.clamp(min=-self.activation_clamp, max=self.activation_clamp)
|
| 71 |
+
gate_up = torch.cat((gate, up), dim=-1)
|
| 72 |
+
x = self.act_fn(gate_up)
|
| 73 |
+
x, _ = self.down_proj(x)
|
| 74 |
+
return x
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class RicursiveAttention(nn.Module):
|
| 78 |
+
def __init__(
|
| 79 |
+
self,
|
| 80 |
+
config: PretrainedConfig,
|
| 81 |
+
layer_id: int = 0,
|
| 82 |
+
prefix: str = "",
|
| 83 |
+
) -> None:
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.config = config
|
| 86 |
+
hidden_size = config.hidden_size
|
| 87 |
+
tp_size = get_tensor_model_parallel_world_size()
|
| 88 |
+
self.total_num_heads = config.num_attention_heads
|
| 89 |
+
self.num_heads = self.total_num_heads // tp_size
|
| 90 |
+
self.total_num_kv_heads = config.num_key_value_heads
|
| 91 |
+
if self.total_num_heads % tp_size or self.total_num_kv_heads % tp_size:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
"Ricursive separate Q/K/V projections require heads divisible by attention TP"
|
| 94 |
+
)
|
| 95 |
+
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
| 96 |
+
self.head_dim = config.head_dim
|
| 97 |
+
self.q_size = self.num_heads * self.head_dim
|
| 98 |
+
self.kv_size = self.num_kv_heads * self.head_dim
|
| 99 |
+
self.scaling = self.head_dim**-0.5
|
| 100 |
+
|
| 101 |
+
self.q_proj = ColumnParallelLinear(
|
| 102 |
+
hidden_size,
|
| 103 |
+
self.total_num_heads * self.head_dim,
|
| 104 |
+
bias=False,
|
| 105 |
+
prefix=add_prefix("q_proj", prefix),
|
| 106 |
+
)
|
| 107 |
+
self.k_proj = ColumnParallelLinear(
|
| 108 |
+
hidden_size,
|
| 109 |
+
self.total_num_kv_heads * self.head_dim,
|
| 110 |
+
bias=False,
|
| 111 |
+
prefix=add_prefix("k_proj", prefix),
|
| 112 |
+
)
|
| 113 |
+
self.v_proj = ColumnParallelLinear(
|
| 114 |
+
hidden_size,
|
| 115 |
+
self.total_num_kv_heads * self.head_dim,
|
| 116 |
+
bias=False,
|
| 117 |
+
prefix=add_prefix("v_proj", prefix),
|
| 118 |
+
)
|
| 119 |
+
self.o_proj = RowParallelLinear(
|
| 120 |
+
self.total_num_heads * self.head_dim,
|
| 121 |
+
hidden_size,
|
| 122 |
+
bias=False,
|
| 123 |
+
prefix=add_prefix("o_proj", prefix),
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
if getattr(config, "use_gated_attention", True):
|
| 127 |
+
self.g_proj = ColumnParallelLinear(
|
| 128 |
+
hidden_size,
|
| 129 |
+
self.total_num_heads * self.head_dim,
|
| 130 |
+
bias=False,
|
| 131 |
+
prefix=add_prefix("g_proj", prefix),
|
| 132 |
+
)
|
| 133 |
+
else:
|
| 134 |
+
self.g_proj = None
|
| 135 |
+
|
| 136 |
+
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 137 |
+
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 138 |
+
|
| 139 |
+
self.use_valres = bool(
|
| 140 |
+
getattr(config, "use_valres", getattr(config, "use_value_residuals", False))
|
| 141 |
+
)
|
| 142 |
+
if self.use_valres:
|
| 143 |
+
self.value_res_lambda = nn.Parameter(torch.zeros(1))
|
| 144 |
+
|
| 145 |
+
layer_type = config.layer_types[layer_id]
|
| 146 |
+
self.is_sliding = layer_type == "sliding_attention"
|
| 147 |
+
sliding_window = (config.sliding_window - 1) if self.is_sliding else -1
|
| 148 |
+
|
| 149 |
+
self.rotary_emb = None
|
| 150 |
+
if self.is_sliding:
|
| 151 |
+
self.rotary_emb = get_rope(
|
| 152 |
+
self.head_dim,
|
| 153 |
+
rotary_dim=self.head_dim,
|
| 154 |
+
max_position=rotary_cache_length(config),
|
| 155 |
+
base=config.rope_theta,
|
| 156 |
+
is_neox_style=True,
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
self.attn = RadixAttention(
|
| 160 |
+
self.num_heads,
|
| 161 |
+
self.head_dim,
|
| 162 |
+
self.scaling,
|
| 163 |
+
num_kv_heads=self.num_kv_heads,
|
| 164 |
+
layer_id=layer_id,
|
| 165 |
+
sliding_window_size=sliding_window,
|
| 166 |
+
prefix=add_prefix("attn", prefix),
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 170 |
+
q = self.q_norm.forward_native(q.reshape(-1, self.head_dim)).reshape(-1, self.q_size)
|
| 171 |
+
k = self.k_norm.forward_native(k.reshape(-1, self.head_dim)).reshape(-1, self.kv_size)
|
| 172 |
+
return q, k
|
| 173 |
+
|
| 174 |
+
def forward(
|
| 175 |
+
self,
|
| 176 |
+
positions: torch.Tensor,
|
| 177 |
+
hidden_states: torch.Tensor,
|
| 178 |
+
forward_batch: ForwardBatch,
|
| 179 |
+
value_residual: Optional[torch.Tensor] = None,
|
| 180 |
+
) -> torch.Tensor:
|
| 181 |
+
value_input = hidden_states
|
| 182 |
+
if value_residual is not None and self.use_valres:
|
| 183 |
+
blend = torch.sigmoid(self.value_res_lambda[0]).to(dtype=hidden_states.dtype)
|
| 184 |
+
value_input = torch.lerp(
|
| 185 |
+
value_residual.to(dtype=hidden_states.dtype), hidden_states, blend
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
q, _ = self.q_proj(hidden_states)
|
| 189 |
+
k, _ = self.k_proj(hidden_states)
|
| 190 |
+
v, _ = self.v_proj(value_input)
|
| 191 |
+
|
| 192 |
+
q, k = self._apply_qk_norm(q, k)
|
| 193 |
+
|
| 194 |
+
if self.rotary_emb is not None:
|
| 195 |
+
q, k = self.rotary_emb(positions, q, k)
|
| 196 |
+
|
| 197 |
+
attn_output = self.attn(q, k, v, forward_batch)
|
| 198 |
+
|
| 199 |
+
if self.g_proj is not None:
|
| 200 |
+
gate, _ = self.g_proj(hidden_states)
|
| 201 |
+
gate = torch.sigmoid(gate.float()).to(attn_output.dtype)
|
| 202 |
+
attn_output = attn_output * gate
|
| 203 |
+
|
| 204 |
+
output, _ = self.o_proj(attn_output)
|
| 205 |
+
return output
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class RicursiveDecoderLayer(nn.Module):
|
| 209 |
+
"""PeriNorm decoder layer: 4 RMSNorms, post-norm inside the residual on both sublayers."""
|
| 210 |
+
|
| 211 |
+
def __init__(
|
| 212 |
+
self,
|
| 213 |
+
config: PretrainedConfig,
|
| 214 |
+
layer_id: int = 0,
|
| 215 |
+
prefix: str = "",
|
| 216 |
+
) -> None:
|
| 217 |
+
super().__init__()
|
| 218 |
+
self.self_attn = RicursiveAttention(
|
| 219 |
+
config, layer_id, prefix=add_prefix("self_attn", prefix)
|
| 220 |
+
)
|
| 221 |
+
self.mlp = RicursiveMLP(
|
| 222 |
+
hidden_size=config.hidden_size,
|
| 223 |
+
intermediate_size=config.intermediate_size,
|
| 224 |
+
activation_clamp=getattr(
|
| 225 |
+
config, "ffn_activation_clamp", getattr(config, "ffn_clamp_threshold", None)
|
| 226 |
+
),
|
| 227 |
+
prefix=add_prefix("mlp", prefix),
|
| 228 |
+
)
|
| 229 |
+
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 230 |
+
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 231 |
+
self.pre_feedforward_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 232 |
+
self.post_feedforward_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 233 |
+
|
| 234 |
+
def forward(
|
| 235 |
+
self,
|
| 236 |
+
positions: torch.Tensor,
|
| 237 |
+
hidden_states: torch.Tensor,
|
| 238 |
+
forward_batch: ForwardBatch,
|
| 239 |
+
value_residual: Optional[torch.Tensor] = None,
|
| 240 |
+
) -> torch.Tensor:
|
| 241 |
+
residual = hidden_states
|
| 242 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 243 |
+
hidden_states = self.self_attn(
|
| 244 |
+
positions, hidden_states, forward_batch, value_residual=value_residual
|
| 245 |
+
)
|
| 246 |
+
hidden_states = residual + self.post_attention_layernorm(hidden_states)
|
| 247 |
+
|
| 248 |
+
residual = hidden_states
|
| 249 |
+
hidden_states = self.pre_feedforward_layernorm(hidden_states)
|
| 250 |
+
hidden_states = self.mlp(hidden_states)
|
| 251 |
+
hidden_states = residual + self.post_feedforward_layernorm(hidden_states)
|
| 252 |
+
return hidden_states
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
class RicursiveModel(nn.Module):
|
| 256 |
+
def __init__(
|
| 257 |
+
self,
|
| 258 |
+
config: PretrainedConfig,
|
| 259 |
+
prefix: str = "",
|
| 260 |
+
) -> None:
|
| 261 |
+
super().__init__()
|
| 262 |
+
self.config = config
|
| 263 |
+
self.embed_tokens = VocabParallelEmbedding(
|
| 264 |
+
config.vocab_size,
|
| 265 |
+
config.hidden_size,
|
| 266 |
+
prefix=add_prefix("embed_tokens", prefix),
|
| 267 |
+
)
|
| 268 |
+
self.layers = make_layers(
|
| 269 |
+
config.num_hidden_layers,
|
| 270 |
+
lambda idx, prefix: RicursiveDecoderLayer(config=config, layer_id=idx, prefix=prefix),
|
| 271 |
+
prefix=add_prefix("layers", prefix),
|
| 272 |
+
)
|
| 273 |
+
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 274 |
+
self.embed_scale = getattr(config, "embed_scale", None)
|
| 275 |
+
self.use_valres = bool(getattr(config, "use_valres", False))
|
| 276 |
+
self.dflash_capture_layers: tuple[int, ...] = ()
|
| 277 |
+
|
| 278 |
+
def forward(
|
| 279 |
+
self,
|
| 280 |
+
input_ids: torch.Tensor,
|
| 281 |
+
positions: torch.Tensor,
|
| 282 |
+
forward_batch: ForwardBatch,
|
| 283 |
+
input_embeds: Optional[torch.Tensor] = None,
|
| 284 |
+
) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:
|
| 285 |
+
if input_embeds is None:
|
| 286 |
+
hidden_states = self.embed_tokens(input_ids)
|
| 287 |
+
else:
|
| 288 |
+
hidden_states = input_embeds
|
| 289 |
+
if self.embed_scale is not None:
|
| 290 |
+
hidden_states = hidden_states * self.embed_scale
|
| 291 |
+
value_residual = hidden_states if self.use_valres else None
|
| 292 |
+
|
| 293 |
+
captured = {}
|
| 294 |
+
for index, layer in enumerate(self.layers):
|
| 295 |
+
hidden_states = layer(
|
| 296 |
+
positions, hidden_states, forward_batch, value_residual=value_residual
|
| 297 |
+
)
|
| 298 |
+
if index in self.dflash_capture_layers:
|
| 299 |
+
captured[index] = hidden_states
|
| 300 |
+
|
| 301 |
+
hidden_states = self.norm(hidden_states)
|
| 302 |
+
if self.dflash_capture_layers:
|
| 303 |
+
aux_hidden_states = [captured[index] for index in self.dflash_capture_layers]
|
| 304 |
+
return hidden_states, aux_hidden_states
|
| 305 |
+
return hidden_states
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
class RicursiveForCausalLM(nn.Module):
|
| 309 |
+
def __init__(
|
| 310 |
+
self,
|
| 311 |
+
config: PretrainedConfig,
|
| 312 |
+
quant_config=None,
|
| 313 |
+
prefix: str = "",
|
| 314 |
+
) -> None:
|
| 315 |
+
super().__init__()
|
| 316 |
+
if quant_config is not None or getattr(config, "quantization_config", None):
|
| 317 |
+
raise ValueError("Only unquantized BF16 checkpoints are supported")
|
| 318 |
+
if torch.get_default_dtype() != torch.bfloat16:
|
| 319 |
+
raise ValueError("Only BF16 is supported; use --dtype bfloat16")
|
| 320 |
+
self.config = config
|
| 321 |
+
self.pp_group = get_pp_group()
|
| 322 |
+
self.model = RicursiveModel(config, prefix=add_prefix("model", prefix))
|
| 323 |
+
if config.tie_word_embeddings:
|
| 324 |
+
self.lm_head = self.model.embed_tokens
|
| 325 |
+
else:
|
| 326 |
+
self.lm_head = ParallelLMHead(
|
| 327 |
+
config.vocab_size,
|
| 328 |
+
config.hidden_size,
|
| 329 |
+
org_num_embeddings=config.vocab_size,
|
| 330 |
+
prefix=add_prefix("lm_head", prefix),
|
| 331 |
+
)
|
| 332 |
+
self.logits_processor = LogitsProcessor(config)
|
| 333 |
+
|
| 334 |
+
def get_input_embeddings(self) -> VocabParallelEmbedding:
|
| 335 |
+
"""Expose the target embedding table shared by the DFlash worker."""
|
| 336 |
+
return self.model.embed_tokens
|
| 337 |
+
|
| 338 |
+
def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None:
|
| 339 |
+
"""Capture post-block, pre-final-norm states in the drafter's configured order.
|
| 340 |
+
|
| 341 |
+
:param layer_ids: Zero-based target block indices from the draft configuration.
|
| 342 |
+
:raises ValueError: If indices are empty, duplicated, or outside the target model.
|
| 343 |
+
"""
|
| 344 |
+
if (
|
| 345 |
+
not layer_ids
|
| 346 |
+
or any(type(index) is not int for index in layer_ids)
|
| 347 |
+
or len(set(layer_ids)) != len(layer_ids)
|
| 348 |
+
or any(index < 0 or index >= len(self.model.layers) for index in layer_ids)
|
| 349 |
+
):
|
| 350 |
+
raise ValueError("DFlash requires unique, valid target layer indices")
|
| 351 |
+
self.model.dflash_capture_layers = tuple(layer_ids)
|
| 352 |
+
|
| 353 |
+
def get_attention_sliding_window_size(self) -> Optional[int]:
|
| 354 |
+
"""Return the window used by Ricursive local-attention layers.
|
| 355 |
+
|
| 356 |
+
SGLang's native ``--disable-hybrid-swa-memory`` flag controls compact KV allocation.
|
| 357 |
+
Per-layer attention semantics are architectural and must not depend on an environment
|
| 358 |
+
variable.
|
| 359 |
+
"""
|
| 360 |
+
layer_types = getattr(self.config, "layer_types", ()) or ()
|
| 361 |
+
if "sliding_attention" not in layer_types:
|
| 362 |
+
return None
|
| 363 |
+
sliding_window = getattr(self.config, "sliding_window", None)
|
| 364 |
+
if sliding_window is None:
|
| 365 |
+
return None
|
| 366 |
+
return sliding_window - 1
|
| 367 |
+
|
| 368 |
+
@torch.no_grad()
|
| 369 |
+
def forward(
|
| 370 |
+
self,
|
| 371 |
+
input_ids: torch.Tensor,
|
| 372 |
+
positions: torch.Tensor,
|
| 373 |
+
forward_batch: ForwardBatch,
|
| 374 |
+
input_embeds: Optional[torch.Tensor] = None,
|
| 375 |
+
) -> torch.Tensor:
|
| 376 |
+
hidden_states = self.model(
|
| 377 |
+
input_ids=input_ids,
|
| 378 |
+
positions=positions,
|
| 379 |
+
forward_batch=forward_batch,
|
| 380 |
+
input_embeds=input_embeds,
|
| 381 |
+
)
|
| 382 |
+
if self.model.dflash_capture_layers:
|
| 383 |
+
hidden_states, aux_hidden_states = hidden_states
|
| 384 |
+
return self.logits_processor(
|
| 385 |
+
input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
|
| 386 |
+
)
|
| 387 |
+
return self.logits_processor(input_ids, hidden_states, self.lm_head, forward_batch)
|
| 388 |
+
|
| 389 |
+
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
| 390 |
+
"""Map the checkpoint's separate gate/up tensors into SGLang's fused MLP."""
|
| 391 |
+
params = dict(self.named_parameters(remove_duplicate=False))
|
| 392 |
+
loaded, unused = set(), []
|
| 393 |
+
for name, weight in weights:
|
| 394 |
+
if "rotary_emb.inv_freq" in name:
|
| 395 |
+
continue
|
| 396 |
+
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
| 397 |
+
continue
|
| 398 |
+
for source, shard in (("gate_proj", 0), ("up_proj", 1)):
|
| 399 |
+
if source in name:
|
| 400 |
+
mapped = name.replace(source, "gate_up_proj")
|
| 401 |
+
if mapped in params:
|
| 402 |
+
params[mapped].weight_loader(params[mapped], weight, shard)
|
| 403 |
+
loaded.add(mapped)
|
| 404 |
+
else:
|
| 405 |
+
unused.append(name)
|
| 406 |
+
break
|
| 407 |
+
else:
|
| 408 |
+
if name in params:
|
| 409 |
+
param = params[name]
|
| 410 |
+
getattr(param, "weight_loader", default_weight_loader)(param, weight)
|
| 411 |
+
loaded.add(name)
|
| 412 |
+
else:
|
| 413 |
+
unused.append(name)
|
| 414 |
+
tied = {"lm_head.weight"} if self.config.tie_word_embeddings else set()
|
| 415 |
+
_raise_on_load_gaps("Ricursive target", unused, set(params) - loaded - tied)
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
class _NoRope(nn.Module):
|
| 419 |
+
"""Identity RoPE for DFlash2 draft layers listed in config.no_rope_layers."""
|
| 420 |
+
|
| 421 |
+
# SGLang's fused draft context-KV path applies one RoPE table to every layer; it
|
| 422 |
+
# only accepts neox-style RoPE, so this marker turns it off for drafts with NoPE layers.
|
| 423 |
+
is_neox_style = False
|
| 424 |
+
|
| 425 |
+
def forward(self, positions, q, k):
|
| 426 |
+
return q, k
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# DFlash2 drafts are trained on 16K-token rows (12,288 tokens of context + 4,096 predicted),
|
| 430 |
+
# so a global draft layer never attends further back than this.
|
| 431 |
+
DRAFT_TRAINING_CONTEXT = 16384
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def _patch_draft_attention():
|
| 435 |
+
"""Fit SGLang 0.5.19's DFlash draft attention to DFlash2 drafts with global layers.
|
| 436 |
+
|
| 437 |
+
It applies RoPE in every layer, ignoring no_rope_layers, and its draft window
|
| 438 |
+
(--speculative-draft-window-size) truncates every layer, including full attention,
|
| 439 |
+
so the window must cover the draft's training context.
|
| 440 |
+
"""
|
| 441 |
+
from sglang.srt.models.dflash import DFlashAttention
|
| 442 |
+
from sglang.srt.server_args import get_global_server_args
|
| 443 |
+
|
| 444 |
+
init = DFlashAttention.__init__
|
| 445 |
+
|
| 446 |
+
def __init__(self, config, layer_id, *args, **kwargs):
|
| 447 |
+
init(self, config, layer_id, *args, **kwargs)
|
| 448 |
+
if layer_id in (getattr(config, "no_rope_layers", None) or ()):
|
| 449 |
+
self.rotary_emb = _NoRope()
|
| 450 |
+
self.use_table_qk_norm_rope = False
|
| 451 |
+
layer_types = getattr(config, "layer_types", None) or ()
|
| 452 |
+
window = get_global_server_args().speculative_draft_window_size
|
| 453 |
+
if (
|
| 454 |
+
layer_id < len(layer_types)
|
| 455 |
+
and layer_types[layer_id] == "full_attention"
|
| 456 |
+
and window is not None
|
| 457 |
+
and window < DRAFT_TRAINING_CONTEXT
|
| 458 |
+
):
|
| 459 |
+
raise ValueError(
|
| 460 |
+
f"Draft layer {layer_id} attends globally over its {DRAFT_TRAINING_CONTEXT}-token "
|
| 461 |
+
f"training rows; use --speculative-draft-window-size >= {DRAFT_TRAINING_CONTEXT}"
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
DFlashAttention.__init__ = __init__
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
_patch_draft_attention()
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
def _raise_on_load_gaps(model: str, unused, unloaded) -> None:
|
| 471 |
+
"""Refuse checkpoint tensors nothing consumed and parameters nothing loaded."""
|
| 472 |
+
if unused or unloaded:
|
| 473 |
+
raise ValueError(f"{model}: unused tensors {sorted(unused)[:20]}, "
|
| 474 |
+
f"unloaded parameters {sorted(unloaded)[:20]}")
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def _strict_draft_loading():
|
| 478 |
+
"""SGLang 0.5.19's DFlash loader skips tensors it cannot place: refuse such drafts."""
|
| 479 |
+
from sglang.srt.models.dflash import DFlashDraftModel
|
| 480 |
+
|
| 481 |
+
load, shared = DFlashDraftModel.load_weights, ("inv_freq", "embed_tokens.weight", "lm_head.weight")
|
| 482 |
+
|
| 483 |
+
def load_weights(self, weights):
|
| 484 |
+
if (getattr(self.config, "dflash_config", None) or {}).get("block_norm", "pre") != "pre":
|
| 485 |
+
raise ValueError("Only pre-norm DFlash drafts are supported")
|
| 486 |
+
params, calls, unused = dict(self.named_parameters()), [], []
|
| 487 |
+
own = {n: p.weight_loader for n, p in params.items() if "weight_loader" in p.__dict__}
|
| 488 |
+
for n, p in params.items(): # record which parameters receive a tensor
|
| 489 |
+
f = own.get(n, default_weight_loader)
|
| 490 |
+
p.weight_loader = lambda q, *a, _f=f, _n=n: (calls.append(_n), _f(q, *a))[1]
|
| 491 |
+
|
| 492 |
+
def watch(stream): # a tensor that reached no weight_loader was skipped
|
| 493 |
+
for name, weight in stream:
|
| 494 |
+
before = len(calls)
|
| 495 |
+
yield name, weight
|
| 496 |
+
if len(calls) == before and not name.endswith(shared):
|
| 497 |
+
unused.append(name)
|
| 498 |
+
|
| 499 |
+
try:
|
| 500 |
+
load(self, watch(weights))
|
| 501 |
+
finally:
|
| 502 |
+
for n, p in params.items():
|
| 503 |
+
del p.weight_loader
|
| 504 |
+
if n in own:
|
| 505 |
+
p.weight_loader = own[n]
|
| 506 |
+
unloaded = {n for n in params if not n.endswith(("k_scale", "v_scale"))} - set(calls)
|
| 507 |
+
_raise_on_load_gaps("DFlash draft", unused, unloaded)
|
| 508 |
+
|
| 509 |
+
DFlashDraftModel.load_weights = load_weights
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
_strict_draft_loading()
|
inference/server/ripple_format.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ripple's chat format for SGLang: reasoning parser, tool-call parser and tokenizer check.
|
| 2 |
+
|
| 3 |
+
The model generates the layout chat_template.jinja renders:
|
| 4 |
+
|
| 5 |
+
<think>\\n{reasoning}\\n</think>\\n{content}
|
| 6 |
+
<think>\\n{reasoning}\\n</think>\\n\\n<tool_call>\\n{"name": "f", "arguments": {...}}\\n</tool_call>
|
| 7 |
+
|
| 8 |
+
`--reasoning-parser ripple` returns reasoning and content without the separator newlines the
|
| 9 |
+
template adds back, so an assistant message sent back unchanged re-renders to the tokens the
|
| 10 |
+
model generated (SGLang's deepseek-v3 parser keeps them, doubling each newline on every turn).
|
| 11 |
+
Output counts as reasoning only when it opens with <think>.
|
| 12 |
+
|
| 13 |
+
`--tool-call-parser ripple` is hermes with the template's call layout, so strict tools and
|
| 14 |
+
tool_choice=required constrain decoding to `<tool_call>\\n{"name": "f", "arguments": ...}`
|
| 15 |
+
(hermes forces `<tool_call>{"name":"f"`, which the model was not trained on). Text before the
|
| 16 |
+
first call keeps its whitespace, minus the newline the template puts before <tool_call>.
|
| 17 |
+
|
| 18 |
+
Output the template cannot render (`</think>\\n<tool_call>` with one newline, text after a call)
|
| 19 |
+
re-renders in the template's canonical layout.
|
| 20 |
+
|
| 21 |
+
`register()` adds both parsers and checks every tokenizer SGLang loads against the checkpoint's
|
| 22 |
+
tokenizer.json (Transformers 5 can rebuild the pre-tokenizer from `tokenizer_class`).
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import json
|
| 26 |
+
import logging
|
| 27 |
+
|
| 28 |
+
THINK_START, THINK_END = "<think>", "</think>"
|
| 29 |
+
|
| 30 |
+
# Exercise pre-tokenizer differences: digit runs, contractions, whitespace runs, non-ASCII.
|
| 31 |
+
TOKENIZER_PROBES = (
|
| 32 |
+
"x = 1234567890 + 3.14159; y=-42",
|
| 33 |
+
"I'M here, you'RE there, it's 4:05pm. DON'T",
|
| 34 |
+
"a b c\n\n\n d\t\te \r\n f",
|
| 35 |
+
" def f(self):\n return {'k': [1, 2]}\n",
|
| 36 |
+
"café naïve 日本語テキスト 한국어 🙂👍 Ελληνικά",
|
| 37 |
+
"<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n<think>\nok\n</think>\nHello<|im_end|>",
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _drop_newline(text, lead):
|
| 42 |
+
if lead:
|
| 43 |
+
return text[1:] if text.startswith("\n") else text
|
| 44 |
+
return text[:-1] if text.endswith("\n") else text
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _partial_suffix(text, token):
|
| 48 |
+
"""Length of the longest suffix of text that is a strict prefix of token."""
|
| 49 |
+
for i in range(min(len(text), len(token) - 1), 0, -1):
|
| 50 |
+
if token.startswith(text[-i:]):
|
| 51 |
+
return i
|
| 52 |
+
return 0
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _reasoning_detector_class():
|
| 56 |
+
from sglang.srt.parser.reasoning_parser import (
|
| 57 |
+
BaseReasoningFormatDetector,
|
| 58 |
+
StreamingParseResult,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
class RippleReasoningDetector(BaseReasoningFormatDetector):
|
| 62 |
+
def __init__(self, stream_reasoning=True, force_reasoning=False, **kwargs):
|
| 63 |
+
# force_reasoning is ignored: the generation prompt does not open <think>.
|
| 64 |
+
super().__init__(
|
| 65 |
+
THINK_START,
|
| 66 |
+
THINK_END,
|
| 67 |
+
force_reasoning=False,
|
| 68 |
+
stream_reasoning=stream_reasoning,
|
| 69 |
+
reasoning_default="explicit_thinking", # as deepseek-v3
|
| 70 |
+
**kwargs,
|
| 71 |
+
)
|
| 72 |
+
self._state = "start" # start -> lead -> think -> after -> content
|
| 73 |
+
|
| 74 |
+
def _detect_and_parse_impl(self, text):
|
| 75 |
+
if self.continue_final_message:
|
| 76 |
+
return super()._detect_and_parse_impl(text)
|
| 77 |
+
if not text.startswith(THINK_START):
|
| 78 |
+
return StreamingParseResult(normal_text=text)
|
| 79 |
+
body = _drop_newline(text[len(THINK_START):], lead=True)
|
| 80 |
+
reasoning, end, content = body.partition(THINK_END)
|
| 81 |
+
if not end: # truncated inside reasoning
|
| 82 |
+
return StreamingParseResult(reasoning_text=reasoning)
|
| 83 |
+
return StreamingParseResult(
|
| 84 |
+
normal_text=_drop_newline(content, lead=True),
|
| 85 |
+
reasoning_text=_drop_newline(reasoning, lead=False),
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
def _parse_streaming_increment_impl(self, new_text):
|
| 89 |
+
if self.continue_final_message:
|
| 90 |
+
return super()._parse_streaming_increment_impl(new_text)
|
| 91 |
+
self._buffer += new_text
|
| 92 |
+
reasoning, normal = "", ""
|
| 93 |
+
while self._buffer:
|
| 94 |
+
buf = self._buffer
|
| 95 |
+
if self._state == "start":
|
| 96 |
+
if buf.startswith(THINK_START):
|
| 97 |
+
self._buffer, self._state, self._in_reasoning = buf[len(THINK_START):], "lead", True
|
| 98 |
+
elif THINK_START.startswith(buf):
|
| 99 |
+
break
|
| 100 |
+
else:
|
| 101 |
+
self._state = "content"
|
| 102 |
+
elif self._state == "lead":
|
| 103 |
+
self._buffer, self._state = _drop_newline(buf, lead=True), "think"
|
| 104 |
+
elif self._state == "think":
|
| 105 |
+
i = buf.find(THINK_END)
|
| 106 |
+
if i >= 0:
|
| 107 |
+
reasoning += _drop_newline(buf[:i], lead=False)
|
| 108 |
+
self._buffer, self._state, self._in_reasoning = buf[i + len(THINK_END):], "after", False
|
| 109 |
+
continue
|
| 110 |
+
if not self.stream_reasoning:
|
| 111 |
+
break
|
| 112 |
+
# Hold back a possible "\n</think>" prefix: its newline is the separator.
|
| 113 |
+
hold = max(_partial_suffix(buf, "\n" + THINK_END), _partial_suffix(buf, THINK_END))
|
| 114 |
+
reasoning += buf[:len(buf) - hold]
|
| 115 |
+
self._buffer = buf[len(buf) - hold:]
|
| 116 |
+
break
|
| 117 |
+
elif self._state == "after":
|
| 118 |
+
self._buffer, self._state = _drop_newline(buf, lead=True), "content"
|
| 119 |
+
else:
|
| 120 |
+
normal += buf
|
| 121 |
+
self._buffer = ""
|
| 122 |
+
return StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
|
| 123 |
+
|
| 124 |
+
def finish(self):
|
| 125 |
+
if self.continue_final_message:
|
| 126 |
+
return super().finish()
|
| 127 |
+
buf, self._buffer = self._buffer, ""
|
| 128 |
+
if self._state in ("lead", "think"): # truncated inside reasoning
|
| 129 |
+
if self._force_nonempty_content:
|
| 130 |
+
text, self._accumulated_reasoning = self._accumulated_reasoning + buf, ""
|
| 131 |
+
return StreamingParseResult(normal_text=text)
|
| 132 |
+
return StreamingParseResult(reasoning_text=buf)
|
| 133 |
+
return StreamingParseResult(normal_text=buf)
|
| 134 |
+
|
| 135 |
+
return RippleReasoningDetector
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _tool_detector_class():
|
| 139 |
+
from sglang.srt.function_call.core_types import StructureInfo, ToolCallItem
|
| 140 |
+
from sglang.srt.function_call.hermes_detector import HermesDetector
|
| 141 |
+
|
| 142 |
+
class RippleToolDetector(HermesDetector):
|
| 143 |
+
"""Text before the first call keeps its whitespace except the one newline the template
|
| 144 |
+
puts before <tool_call> (hermes strips it all)."""
|
| 145 |
+
|
| 146 |
+
def __init__(self):
|
| 147 |
+
super().__init__()
|
| 148 |
+
self._held_newline = ""
|
| 149 |
+
self._in_calls = False
|
| 150 |
+
# Hermes can leave whole calls buffered when one speculative delta spans
|
| 151 |
+
# multiple calls. Reconcile the full output with emitted deltas at finish.
|
| 152 |
+
self._seen = ""
|
| 153 |
+
self._emitted = {}
|
| 154 |
+
|
| 155 |
+
def _record(self, calls):
|
| 156 |
+
for call in calls:
|
| 157 |
+
emitted = self._emitted.setdefault(call.tool_index, ["", ""])
|
| 158 |
+
emitted[0] += call.name or ""
|
| 159 |
+
emitted[1] += call.parameters or ""
|
| 160 |
+
# SGLang's serving layer also flushes arguments after finish(). Keep
|
| 161 |
+
# its accounting current so it cannot send the same suffix twice.
|
| 162 |
+
while len(self.streamed_args_for_tool) <= call.tool_index:
|
| 163 |
+
self.streamed_args_for_tool.append("")
|
| 164 |
+
self.streamed_args_for_tool[call.tool_index] = emitted[1]
|
| 165 |
+
|
| 166 |
+
def detect_and_parse(self, text, tools):
|
| 167 |
+
res = super().detect_and_parse(text, tools)
|
| 168 |
+
idx = text.find(self.bot_token)
|
| 169 |
+
if idx != -1 and res.normal_text == text[:idx].strip(): # parsed (not the error path)
|
| 170 |
+
res.normal_text = _drop_newline(text[:idx], lead=False)
|
| 171 |
+
return res
|
| 172 |
+
|
| 173 |
+
def parse_streaming_increment(self, new_text, tools):
|
| 174 |
+
self._seen += new_text
|
| 175 |
+
res = super().parse_streaming_increment(new_text, tools)
|
| 176 |
+
self._record(res.calls)
|
| 177 |
+
if self._in_calls: # text after the first call is dropped, as in detect_and_parse
|
| 178 |
+
res.normal_text = ""
|
| 179 |
+
return res
|
| 180 |
+
call_started = bool(self._buffer.startswith(self.bot_token) or self.current_tool_id >= 0 or res.calls)
|
| 181 |
+
self._in_calls = call_started
|
| 182 |
+
if res.normal_text:
|
| 183 |
+
text, self._held_newline = self._held_newline + res.normal_text, ""
|
| 184 |
+
if call_started: # its trailing newline is the separator before <tool_call>
|
| 185 |
+
text = _drop_newline(text, lead=False)
|
| 186 |
+
elif text.endswith("\n"): # undecided until the next text arrives
|
| 187 |
+
text, self._held_newline = text[:-1], "\n"
|
| 188 |
+
res.normal_text = text
|
| 189 |
+
elif call_started:
|
| 190 |
+
self._held_newline = ""
|
| 191 |
+
return res
|
| 192 |
+
|
| 193 |
+
def finish(self, tools):
|
| 194 |
+
res = super().finish(tools)
|
| 195 |
+
self._record(res.calls)
|
| 196 |
+
if self.bot_token in self._seen:
|
| 197 |
+
for i, call in enumerate(self.detect_and_parse(self._seen, tools).calls):
|
| 198 |
+
name, args = self._emitted.get(i, ["", ""])
|
| 199 |
+
want = call.parameters or ""
|
| 200 |
+
if want.startswith(args):
|
| 201 |
+
suffix = want[len(args):]
|
| 202 |
+
if suffix or not name:
|
| 203 |
+
item = ToolCallItem(
|
| 204 |
+
tool_index=i, name=None if name else call.name, parameters=suffix,
|
| 205 |
+
)
|
| 206 |
+
res.calls.append(item)
|
| 207 |
+
self._record([item])
|
| 208 |
+
if self._in_calls:
|
| 209 |
+
res.normal_text = ""
|
| 210 |
+
elif self._held_newline:
|
| 211 |
+
res.normal_text, self._held_newline = self._held_newline + (res.normal_text or ""), ""
|
| 212 |
+
return res
|
| 213 |
+
|
| 214 |
+
def structure_info(self):
|
| 215 |
+
return lambda name: StructureInfo(
|
| 216 |
+
begin='<tool_call>\n{"name": ' + json.dumps(name, ensure_ascii=False) + ', "arguments": ',
|
| 217 |
+
end="}\n</tool_call>",
|
| 218 |
+
trigger="<tool_call>",
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
return RippleToolDetector
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def check_tokenizer(tokenizer, tokenizer_json):
|
| 225 |
+
"""Make `tokenizer` encode and decode like the raw tokenizer.json, or raise.
|
| 226 |
+
|
| 227 |
+
Restores the normalizer, pre-tokenizer and decoder from tokenizer.json when they differ."""
|
| 228 |
+
from tokenizers import Tokenizer
|
| 229 |
+
|
| 230 |
+
raw = Tokenizer.from_file(tokenizer_json)
|
| 231 |
+
|
| 232 |
+
def mismatches():
|
| 233 |
+
bad = []
|
| 234 |
+
for text in TOKENIZER_PROBES:
|
| 235 |
+
ids = raw.encode(text, add_special_tokens=False).ids
|
| 236 |
+
got = tokenizer.encode(text, add_special_tokens=False)
|
| 237 |
+
if got != ids or tokenizer.decode(ids) != raw.decode(ids, skip_special_tokens=False):
|
| 238 |
+
bad.append(text)
|
| 239 |
+
return bad
|
| 240 |
+
|
| 241 |
+
if not mismatches():
|
| 242 |
+
return False
|
| 243 |
+
backend = getattr(tokenizer, "_tokenizer", None)
|
| 244 |
+
if backend is not None:
|
| 245 |
+
backend.normalizer, backend.pre_tokenizer = raw.normalizer, raw.pre_tokenizer
|
| 246 |
+
backend.decoder = raw.decoder
|
| 247 |
+
bad = mismatches()
|
| 248 |
+
if bad:
|
| 249 |
+
raise RuntimeError(
|
| 250 |
+
f"{type(tokenizer).__name__} does not match {tokenizer_json} on {bad!r}; "
|
| 251 |
+
"export the tokenizer with tokenizer_class PreTrainedTokenizerFast"
|
| 252 |
+
)
|
| 253 |
+
return True
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def _tokenizer_json(name, revision=None):
|
| 257 |
+
from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
|
| 258 |
+
|
| 259 |
+
try:
|
| 260 |
+
return hf_tok._resolve_local_or_cached_file(name, "tokenizer.json", revision)
|
| 261 |
+
except (OSError, ValueError): # no tokenizer.json (e.g. a slow-only tokenizer)
|
| 262 |
+
return None
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def register():
|
| 266 |
+
"""Add the `ripple` parsers to SGLang and check tokenizers on load. Idempotent; call at
|
| 267 |
+
import time so SGLang's spawned processes register too."""
|
| 268 |
+
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
| 269 |
+
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
| 270 |
+
from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
|
| 271 |
+
|
| 272 |
+
ReasoningParser.DetectorMap.setdefault("ripple", _reasoning_detector_class())
|
| 273 |
+
FunctionCallParser.ToolCallParserEnum.setdefault("ripple", _tool_detector_class())
|
| 274 |
+
|
| 275 |
+
fixes = hf_tok._apply_post_load_fixes # fail loudly if SGLang renames it
|
| 276 |
+
if getattr(fixes, "_ripple", False):
|
| 277 |
+
return
|
| 278 |
+
|
| 279 |
+
def apply_post_load_fixes(tokenizer, name, revision=None):
|
| 280 |
+
tokenizer = fixes(tokenizer, name, revision)
|
| 281 |
+
path = _tokenizer_json(name, revision)
|
| 282 |
+
if path is not None and check_tokenizer(tokenizer, path):
|
| 283 |
+
logging.getLogger(__name__).warning(
|
| 284 |
+
"Restored %s's normalizer, pre-tokenizer and decoder from tokenizer.json", name)
|
| 285 |
+
return tokenizer
|
| 286 |
+
|
| 287 |
+
apply_post_load_fixes._ripple = True
|
| 288 |
+
hf_tok._apply_post_load_fixes = apply_post_load_fixes
|
inference/server/serve.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.12,<3.13"
|
| 3 |
+
# dependencies = [
|
| 4 |
+
# "sglang==0.5.19",
|
| 5 |
+
# "torch==2.13.0+cu130",
|
| 6 |
+
# ]
|
| 7 |
+
# [tool.uv]
|
| 8 |
+
# index-strategy = "unsafe-best-match"
|
| 9 |
+
# prerelease = "allow"
|
| 10 |
+
# [[tool.uv.index]]
|
| 11 |
+
# url = "https://download.pytorch.org/whl/cu130"
|
| 12 |
+
# ///
|
| 13 |
+
"""Serve Ripple + DFlash2 on two B200s. Accepts standard SGLang CLI flags."""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import shlex
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
sys.dont_write_bytecode = True
|
| 21 |
+
|
| 22 |
+
# SGLang sizes the draft's positional cache for the target context, even with
|
| 23 |
+
# windowed draft attention. Allow its 128Ki position limit to grow to 1Mi;
|
| 24 |
+
# --speculative-draft-window-size still limits draft attention to 16Ki tokens.
|
| 25 |
+
os.environ.setdefault("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", "1")
|
| 26 |
+
|
| 27 |
+
# Top-level registration also runs in SGLang's spawned worker processes.
|
| 28 |
+
from sglang.srt.models.registry import ModelRegistry
|
| 29 |
+
|
| 30 |
+
from model import RicursiveForCausalLM
|
| 31 |
+
import ripple_format
|
| 32 |
+
|
| 33 |
+
ModelRegistry.models["RicursiveForCausalLM"] = RicursiveForCausalLM
|
| 34 |
+
ripple_format.register()
|
| 35 |
+
|
| 36 |
+
DEFAULT_ARGS = shlex.split("""
|
| 37 |
+
--host 127.0.0.1 --served-model-name ripple --trust-remote-code
|
| 38 |
+
--dtype bfloat16 --kv-cache-dtype bfloat16 --tp-size 2
|
| 39 |
+
--attention-backend trtllm_mha --page-size 64
|
| 40 |
+
--speculative-algorithm DFLASH --speculative-draft-model-quantization unquant
|
| 41 |
+
--speculative-num-draft-tokens 16 --speculative-draft-window-size 16384
|
| 42 |
+
--speculative-draft-attention-backend triton
|
| 43 |
+
--context-length 1048576 --mem-fraction-static 0.80 --max-running-requests 16
|
| 44 |
+
--cuda-graph-backend-prefill disabled
|
| 45 |
+
--tool-call-parser ripple --reasoning-parser ripple
|
| 46 |
+
""")
|
| 47 |
+
DEFAULT_ARGS += [
|
| 48 |
+
"--chat-template",
|
| 49 |
+
str(Path(__file__).resolve().parents[2] / "chat_template.jinja"),
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def main():
|
| 54 |
+
from sglang.srt.entrypoints.http_server import launch_server
|
| 55 |
+
from sglang.srt.server_args import prepare_server_args
|
| 56 |
+
from sglang.srt.utils import kill_process_tree
|
| 57 |
+
|
| 58 |
+
args = prepare_server_args(DEFAULT_ARGS + sys.argv[1:])
|
| 59 |
+
if args.dtype != "bfloat16" or args.kv_cache_dtype != "bfloat16":
|
| 60 |
+
raise ValueError("Only BF16 weights and KV cache are supported")
|
| 61 |
+
try:
|
| 62 |
+
launch_server(args)
|
| 63 |
+
finally:
|
| 64 |
+
kill_process_tree(os.getpid(), include_parent=False)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
if __name__ == "__main__":
|
| 68 |
+
main()
|
model-00001-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0701bedc60edbb370e128cdff50478f6676cacd674080c16cca122a8d4b06c72
|
| 3 |
+
size 8420238166
|
model-00002-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f22cba23852fcfd5df22e54285952ec3803b7921461fc8e54629c586e1ab8a60
|
| 3 |
+
size 8399320620
|
model-00003-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed8cbb1946f546d32c96b569d30b700ca6936994025623d6c69c9e436b7caced
|
| 3 |
+
size 8388824258
|
model-00004-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:54281881efb8ab45301aa3d22e9501281e4c59bc69bd736ce08e8de5e09fae05
|
| 3 |
+
size 8388855346
|
model-00005-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a78d9cac0b076eede2fda612704b43f9d21d6f27276ef1924656327c992a1c5c
|
| 3 |
+
size 8304949852
|
model-00006-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cd281215be38ff363f13e6de47d20b062a4cfe288d7c2d98fc93c0a962d4e21a
|
| 3 |
+
size 8388824290
|
model-00007-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:384e1e3430c5972f33434269090c3dfe5b101649579950b9645fecc9739949be
|
| 3 |
+
size 8388824258
|
model-00008-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d011f18e13bc7cdea10f9066681797051d4ae28fd4edb96f84abca264e9934eb
|
| 3 |
+
size 8388855346
|
model-00009-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:921d6e15e08f717dc5be562dd1ceb56a096fd9d3707f1f1d742d15625917daa1
|
| 3 |
+
size 8304949852
|
model-00010-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:31f3417c654d8db5596f811db381b75b732f2e11925d54a8dbb7ec3d25e3a086
|
| 3 |
+
size 8388824290
|
model-00011-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6aa7575de621c2f74761a0ed4cf1d303395fbaec65f64e7c3dcd695230fedc14
|
| 3 |
+
size 8388824274
|
model-00012-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:882b853d96ed396346ab65bb4070bf489f7b7e2d3cb60457d6d785fe2e218272
|
| 3 |
+
size 8388855338
|
model-00013-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:75a1cd9a3688a8d2483232e45ebfc01fa7287accd52237c955ae11af3621e028
|
| 3 |
+
size 8304949868
|
model-00014-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:269b31e8abf1d73bb0dd158d30a4e0bacfaad1eaa7cbc442541cbce9443023bf
|
| 3 |
+
size 8388824274
|
model-00015-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:32b4c1bb53297f5e564bb327b2cd40318c7b672be04d94899c6c0eee37102c9b
|
| 3 |
+
size 8388824274
|
model-00016-of-00016.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0a2571453449e5fe940cba3adb623ffc1d9b3d00a32a5d5f0a979e82bc39eeb
|
| 3 |
+
size 3942775166
|
model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
modeling_ricursive.py
ADDED
|
@@ -0,0 +1,606 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HuggingFace modeling implementation for the Ricursive transformer architecture.
|
| 2 |
+
|
| 3 |
+
Designed for inference with vLLM (via --trust-remote-code). Key differences from
|
| 4 |
+
standard Llama/Mistral:
|
| 5 |
+
- PeriNorm blocks: 4 layer norms per block (pre/post attention + pre/post FFN)
|
| 6 |
+
- Gated attention: elementwise sigmoid gate on attention output
|
| 7 |
+
- Mixed local/global layers: local = sliding window + RoPE, global = full attention + no RoPE
|
| 8 |
+
- Per-head Q/K RMSNorm
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn as nn
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
from transformers import PreTrainedModel
|
| 17 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 18 |
+
from transformers.modeling_outputs import (
|
| 19 |
+
BaseModelOutputWithPast,
|
| 20 |
+
CausalLMOutputWithPast,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
from .configuration_ricursive import RicursiveConfig
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class RicursiveRMSNorm(nn.Module):
|
| 27 |
+
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 30 |
+
self.eps = eps
|
| 31 |
+
|
| 32 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 33 |
+
input_dtype = hidden_states.dtype
|
| 34 |
+
hidden_states = hidden_states.float()
|
| 35 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 36 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
|
| 37 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class RicursiveRotaryEmbedding(nn.Module):
|
| 41 |
+
def __init__(self, dim: int, theta: float = 10000.0):
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.dim = dim
|
| 44 |
+
self.theta = theta
|
| 45 |
+
# NOTE: inv_freq is computed on-the-fly in forward() rather than stored
|
| 46 |
+
# as a buffer, because non-persistent buffers get corrupted during
|
| 47 |
+
# from_pretrained() loading.
|
| 48 |
+
|
| 49 |
+
@torch.no_grad()
|
| 50 |
+
def forward(self, x: torch.Tensor, position_ids: torch.Tensor):
|
| 51 |
+
inv_freq = 1.0 / (
|
| 52 |
+
self.theta ** (torch.arange(0, self.dim, 2, device=x.device).float() / self.dim)
|
| 53 |
+
)
|
| 54 |
+
inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
| 55 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
| 56 |
+
|
| 57 |
+
device_type = x.device.type
|
| 58 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
| 59 |
+
freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
|
| 60 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 61 |
+
cos = emb.cos()
|
| 62 |
+
sin = emb.sin()
|
| 63 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def rotate_half(x: torch.Tensor) -> torch.Tensor:
|
| 67 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 68 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 69 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def apply_rotary_pos_emb(
|
| 73 |
+
q: torch.Tensor,
|
| 74 |
+
k: torch.Tensor,
|
| 75 |
+
cos: torch.Tensor,
|
| 76 |
+
sin: torch.Tensor,
|
| 77 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 78 |
+
cos = cos.unsqueeze(1) # (B, 1, T, dim)
|
| 79 |
+
sin = sin.unsqueeze(1) # (B, 1, T, dim)
|
| 80 |
+
# OLMo-core applies RoPE in full precision (float32) for numerical stability
|
| 81 |
+
orig_dtype = q.dtype
|
| 82 |
+
q_float, k_float = q.float(), k.float()
|
| 83 |
+
cos_float, sin_float = cos.float(), sin.float()
|
| 84 |
+
q_embed = (q_float * cos_float) + (rotate_half(q_float) * sin_float)
|
| 85 |
+
k_embed = (k_float * cos_float) + (rotate_half(k_float) * sin_float)
|
| 86 |
+
return q_embed.to(orig_dtype), k_embed.to(orig_dtype)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 90 |
+
if n_rep == 1:
|
| 91 |
+
return hidden_states
|
| 92 |
+
batch, num_kv_heads, slen, head_dim = hidden_states.shape
|
| 93 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(
|
| 94 |
+
batch, num_kv_heads, n_rep, slen, head_dim
|
| 95 |
+
)
|
| 96 |
+
return hidden_states.reshape(batch, num_kv_heads * n_rep, slen, head_dim)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class RicursiveAttention(nn.Module):
|
| 100 |
+
def __init__(self, config: RicursiveConfig, layer_idx: int):
|
| 101 |
+
super().__init__()
|
| 102 |
+
self.config = config
|
| 103 |
+
self.layer_idx = layer_idx
|
| 104 |
+
self.layer_type = config.layer_types[layer_idx]
|
| 105 |
+
|
| 106 |
+
self.hidden_size = config.hidden_size
|
| 107 |
+
self.num_heads = config.num_attention_heads
|
| 108 |
+
self.num_kv_heads = config.num_key_value_heads
|
| 109 |
+
self.head_dim = config.head_dim
|
| 110 |
+
self.num_kv_groups = self.num_heads // self.num_kv_heads
|
| 111 |
+
|
| 112 |
+
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
|
| 113 |
+
self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
|
| 114 |
+
self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
|
| 115 |
+
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
| 116 |
+
if config.use_valres:
|
| 117 |
+
self.value_res_lambda = nn.Parameter(torch.zeros(1))
|
| 118 |
+
|
| 119 |
+
# Per-head Q/K norms (RMSNorm on head_dim)
|
| 120 |
+
self.q_norm = RicursiveRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 121 |
+
self.k_norm = RicursiveRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 122 |
+
|
| 123 |
+
# Gated attention
|
| 124 |
+
if config.use_gated_attention:
|
| 125 |
+
self.g_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
|
| 126 |
+
|
| 127 |
+
# RoPE only for local (sliding_attention) layers
|
| 128 |
+
if self.layer_type == "sliding_attention":
|
| 129 |
+
self.rotary_emb = RicursiveRotaryEmbedding(self.head_dim, theta=config.rope_theta)
|
| 130 |
+
|
| 131 |
+
def _apply_sliding_window_mask(
|
| 132 |
+
self,
|
| 133 |
+
attention_mask: Optional[torch.Tensor],
|
| 134 |
+
q_len: int,
|
| 135 |
+
kv_len: int,
|
| 136 |
+
dtype: torch.dtype,
|
| 137 |
+
device: torch.device,
|
| 138 |
+
cache_position: Optional[torch.LongTensor],
|
| 139 |
+
) -> Optional[torch.Tensor]:
|
| 140 |
+
if self.layer_type != "sliding_attention":
|
| 141 |
+
return attention_mask
|
| 142 |
+
|
| 143 |
+
key_positions = torch.arange(kv_len, device=device)
|
| 144 |
+
if cache_position is not None:
|
| 145 |
+
query_positions = cache_position.to(device=device)
|
| 146 |
+
else:
|
| 147 |
+
query_positions = torch.arange(kv_len - q_len, kv_len, device=device)
|
| 148 |
+
|
| 149 |
+
window = self.config.sliding_window
|
| 150 |
+
allowed = (
|
| 151 |
+
(key_positions[None, :] <= query_positions[:, None])
|
| 152 |
+
& (key_positions[None, :] >= query_positions[:, None] - (window - 1))
|
| 153 |
+
)[None, None, :, :]
|
| 154 |
+
|
| 155 |
+
if attention_mask is None:
|
| 156 |
+
return allowed
|
| 157 |
+
if attention_mask.dtype == torch.bool:
|
| 158 |
+
return attention_mask & allowed
|
| 159 |
+
return attention_mask.masked_fill(~allowed, torch.finfo(dtype).min)
|
| 160 |
+
|
| 161 |
+
def forward(
|
| 162 |
+
self,
|
| 163 |
+
hidden_states: torch.Tensor,
|
| 164 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 165 |
+
position_ids: Optional[torch.Tensor] = None,
|
| 166 |
+
past_key_value: Optional[Cache] = None,
|
| 167 |
+
output_attentions: bool = False,
|
| 168 |
+
use_cache: bool = False,
|
| 169 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 170 |
+
value_residual: Optional[torch.Tensor] = None,
|
| 171 |
+
**kwargs,
|
| 172 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 173 |
+
bsz, q_len, _ = hidden_states.shape
|
| 174 |
+
|
| 175 |
+
# Project Q, K, V
|
| 176 |
+
q = self.q_proj(hidden_states)
|
| 177 |
+
k = self.k_proj(hidden_states)
|
| 178 |
+
value_states = hidden_states
|
| 179 |
+
if value_residual is not None:
|
| 180 |
+
if not self.config.use_valres:
|
| 181 |
+
raise RuntimeError("value_residual was supplied but config.use_valres is false")
|
| 182 |
+
blend = torch.sigmoid(self.value_res_lambda[0]).to(dtype=hidden_states.dtype)
|
| 183 |
+
value_states = torch.lerp(
|
| 184 |
+
value_residual.to(dtype=hidden_states.dtype), hidden_states, blend
|
| 185 |
+
)
|
| 186 |
+
v = self.v_proj(value_states)
|
| 187 |
+
|
| 188 |
+
# Reshape to (B, T, n_heads, head_dim) then transpose to (B, n_heads, T, head_dim)
|
| 189 |
+
q = q.view(bsz, q_len, self.num_heads, self.head_dim)
|
| 190 |
+
k = k.view(bsz, q_len, self.num_kv_heads, self.head_dim)
|
| 191 |
+
v = v.view(bsz, q_len, self.num_kv_heads, self.head_dim)
|
| 192 |
+
|
| 193 |
+
# Per-head Q/K norm (applied before transpose, on head_dim)
|
| 194 |
+
q = self.q_norm(q)
|
| 195 |
+
k = self.k_norm(k)
|
| 196 |
+
|
| 197 |
+
# Transpose to (B, n_heads, T, head_dim) for attention
|
| 198 |
+
q = q.transpose(1, 2)
|
| 199 |
+
k = k.transpose(1, 2)
|
| 200 |
+
v = v.transpose(1, 2)
|
| 201 |
+
|
| 202 |
+
# RoPE only for local layers
|
| 203 |
+
if self.layer_type == "sliding_attention":
|
| 204 |
+
cos, sin = self.rotary_emb(v, position_ids)
|
| 205 |
+
q, k = apply_rotary_pos_emb(q, k, cos, sin)
|
| 206 |
+
|
| 207 |
+
# KV cache
|
| 208 |
+
if past_key_value is not None:
|
| 209 |
+
k, v = past_key_value.update(k, v, self.layer_idx)
|
| 210 |
+
|
| 211 |
+
# GQA: expand KV heads
|
| 212 |
+
k = repeat_kv(k, self.num_kv_groups)
|
| 213 |
+
v = repeat_kv(v, self.num_kv_groups)
|
| 214 |
+
|
| 215 |
+
attention_mask = self._apply_sliding_window_mask(
|
| 216 |
+
attention_mask=attention_mask,
|
| 217 |
+
q_len=q.shape[2],
|
| 218 |
+
kv_len=k.shape[2],
|
| 219 |
+
dtype=q.dtype,
|
| 220 |
+
device=q.device,
|
| 221 |
+
cache_position=cache_position,
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
# Attention (use is_causal when no explicit mask for correct causal behavior)
|
| 225 |
+
# Cast to plain bool to avoid SymBool issues with torch.compile
|
| 226 |
+
is_causal = bool(attention_mask is None and q.shape[2] > 1)
|
| 227 |
+
attn_output = F.scaled_dot_product_attention(
|
| 228 |
+
q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
# (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
|
| 232 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 233 |
+
|
| 234 |
+
# Gated attention: output * sigmoid(g_proj(input))
|
| 235 |
+
if self.config.use_gated_attention:
|
| 236 |
+
gate = self.g_proj(hidden_states)
|
| 237 |
+
if self.config.gate_full_precision:
|
| 238 |
+
gate = gate.float()
|
| 239 |
+
gate = torch.sigmoid(gate).to(attn_output.dtype)
|
| 240 |
+
attn_output = attn_output.view(bsz, q_len, -1) * gate
|
| 241 |
+
else:
|
| 242 |
+
attn_output = attn_output.view(bsz, q_len, -1)
|
| 243 |
+
|
| 244 |
+
# Output projection
|
| 245 |
+
attn_output = self.o_proj(attn_output)
|
| 246 |
+
|
| 247 |
+
return attn_output, None, past_key_value
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
class RicursiveMLP(nn.Module):
|
| 251 |
+
def __init__(self, config: RicursiveConfig):
|
| 252 |
+
super().__init__()
|
| 253 |
+
self.clamp_threshold = config.ffn_clamp_threshold
|
| 254 |
+
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 255 |
+
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
|
| 256 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 257 |
+
|
| 258 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 259 |
+
gate = self.gate_proj(x)
|
| 260 |
+
up = self.up_proj(x)
|
| 261 |
+
if self.clamp_threshold:
|
| 262 |
+
gate = gate.clamp(min=None, max=self.clamp_threshold)
|
| 263 |
+
up = up.clamp(min=-self.clamp_threshold, max=self.clamp_threshold)
|
| 264 |
+
return self.down_proj(F.silu(gate) * up)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
class RicursiveDecoderLayer(nn.Module):
|
| 268 |
+
"""PeriNorm decoder layer with 4 layer norms."""
|
| 269 |
+
|
| 270 |
+
def __init__(self, config: RicursiveConfig, layer_idx: int):
|
| 271 |
+
super().__init__()
|
| 272 |
+
self.self_attn = RicursiveAttention(config, layer_idx)
|
| 273 |
+
self.mlp = RicursiveMLP(config)
|
| 274 |
+
|
| 275 |
+
# PeriNorm: 4 norms
|
| 276 |
+
self.input_layernorm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 277 |
+
self.post_attention_layernorm = RicursiveRMSNorm(
|
| 278 |
+
config.hidden_size, eps=config.rms_norm_eps
|
| 279 |
+
)
|
| 280 |
+
self.pre_feedforward_layernorm = RicursiveRMSNorm(
|
| 281 |
+
config.hidden_size, eps=config.rms_norm_eps
|
| 282 |
+
)
|
| 283 |
+
self.post_feedforward_layernorm = RicursiveRMSNorm(
|
| 284 |
+
config.hidden_size, eps=config.rms_norm_eps
|
| 285 |
+
)
|
| 286 |
+
|
| 287 |
+
def forward(
|
| 288 |
+
self,
|
| 289 |
+
hidden_states: torch.Tensor,
|
| 290 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 291 |
+
position_ids: Optional[torch.Tensor] = None,
|
| 292 |
+
past_key_value: Optional[Cache] = None,
|
| 293 |
+
output_attentions: bool = False,
|
| 294 |
+
use_cache: bool = False,
|
| 295 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 296 |
+
value_residual: Optional[torch.Tensor] = None,
|
| 297 |
+
**kwargs,
|
| 298 |
+
) -> Tuple[torch.Tensor, ...]:
|
| 299 |
+
# Attention sub-block (PeriNorm: pre-norm -> attn -> post-norm -> residual)
|
| 300 |
+
normed = self.input_layernorm(hidden_states)
|
| 301 |
+
attn_output, attn_weights, present_key_value = self.self_attn(
|
| 302 |
+
normed,
|
| 303 |
+
attention_mask=attention_mask,
|
| 304 |
+
position_ids=position_ids,
|
| 305 |
+
past_key_value=past_key_value,
|
| 306 |
+
output_attentions=output_attentions,
|
| 307 |
+
use_cache=use_cache,
|
| 308 |
+
cache_position=cache_position,
|
| 309 |
+
value_residual=value_residual,
|
| 310 |
+
**kwargs,
|
| 311 |
+
)
|
| 312 |
+
hidden_states = hidden_states + self.post_attention_layernorm(attn_output)
|
| 313 |
+
|
| 314 |
+
# FFN sub-block (PeriNorm: pre-norm -> ffn -> post-norm -> residual)
|
| 315 |
+
normed = self.pre_feedforward_layernorm(hidden_states)
|
| 316 |
+
ff_output = self.mlp(normed)
|
| 317 |
+
hidden_states = hidden_states + self.post_feedforward_layernorm(ff_output)
|
| 318 |
+
|
| 319 |
+
outputs = (hidden_states,)
|
| 320 |
+
if output_attentions:
|
| 321 |
+
outputs += (attn_weights,)
|
| 322 |
+
if use_cache:
|
| 323 |
+
outputs += (present_key_value,)
|
| 324 |
+
return outputs
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
class RicursivePreTrainedModel(PreTrainedModel):
|
| 328 |
+
config_class = RicursiveConfig
|
| 329 |
+
base_model_prefix = "model"
|
| 330 |
+
supports_gradient_checkpointing = True
|
| 331 |
+
_no_split_modules = ["RicursiveDecoderLayer"]
|
| 332 |
+
_supports_sdpa = True
|
| 333 |
+
_supports_cache_class = True
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
class RicursiveModel(RicursivePreTrainedModel):
|
| 337 |
+
def __init__(self, config: RicursiveConfig):
|
| 338 |
+
super().__init__(config)
|
| 339 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 340 |
+
self.layers = nn.ModuleList(
|
| 341 |
+
[RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
|
| 342 |
+
)
|
| 343 |
+
self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 344 |
+
self.post_init()
|
| 345 |
+
|
| 346 |
+
def forward(
|
| 347 |
+
self,
|
| 348 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 349 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 350 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 351 |
+
past_key_values: Optional[Cache] = None,
|
| 352 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 353 |
+
use_cache: Optional[bool] = None,
|
| 354 |
+
output_attentions: Optional[bool] = None,
|
| 355 |
+
output_hidden_states: Optional[bool] = None,
|
| 356 |
+
return_dict: Optional[bool] = None,
|
| 357 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 358 |
+
**kwargs,
|
| 359 |
+
) -> Union[Tuple, BaseModelOutputWithPast]:
|
| 360 |
+
output_attentions = (
|
| 361 |
+
output_attentions if output_attentions is not None else self.config.output_attentions
|
| 362 |
+
)
|
| 363 |
+
output_hidden_states = (
|
| 364 |
+
output_hidden_states
|
| 365 |
+
if output_hidden_states is not None
|
| 366 |
+
else self.config.output_hidden_states
|
| 367 |
+
)
|
| 368 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 369 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 370 |
+
|
| 371 |
+
if inputs_embeds is None:
|
| 372 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
| 373 |
+
|
| 374 |
+
# embed_scale: non-folded checkpoints carry the scale (e.g. sqrt(hidden_size)) in the
|
| 375 |
+
# config and apply it here at runtime; folded checkpoints set embed_scale=None and bake
|
| 376 |
+
# the scale into embed_tokens.weight, so this is a no-op. This mirrors the SGLang native
|
| 377 |
+
# port (sglang_ricursive) so served and trained forwards agree. It must run BEFORE the
|
| 378 |
+
# value-residual seed below, whose source is the (scaled) layer-0 embeddings.
|
| 379 |
+
embed_scale = getattr(self.config, "embed_scale", None)
|
| 380 |
+
if embed_scale is not None:
|
| 381 |
+
inputs_embeds = inputs_embeds * embed_scale
|
| 382 |
+
|
| 383 |
+
if use_cache and past_key_values is None:
|
| 384 |
+
past_key_values = DynamicCache()
|
| 385 |
+
|
| 386 |
+
if cache_position is None:
|
| 387 |
+
past_seen_tokens = (
|
| 388 |
+
past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 389 |
+
)
|
| 390 |
+
cache_position = torch.arange(
|
| 391 |
+
past_seen_tokens,
|
| 392 |
+
past_seen_tokens + inputs_embeds.shape[1],
|
| 393 |
+
device=inputs_embeds.device,
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
if position_ids is None:
|
| 397 |
+
position_ids = cache_position.unsqueeze(0)
|
| 398 |
+
|
| 399 |
+
# Build causal mask
|
| 400 |
+
causal_mask = self._update_causal_mask(
|
| 401 |
+
attention_mask, inputs_embeds, cache_position, past_key_values
|
| 402 |
+
)
|
| 403 |
+
|
| 404 |
+
hidden_states = inputs_embeds
|
| 405 |
+
value_residual = hidden_states if self.config.use_valres else None
|
| 406 |
+
all_hidden_states = () if output_hidden_states else None
|
| 407 |
+
all_self_attns = () if output_attentions else None
|
| 408 |
+
|
| 409 |
+
for layer in self.layers:
|
| 410 |
+
if output_hidden_states:
|
| 411 |
+
all_hidden_states += (hidden_states,)
|
| 412 |
+
|
| 413 |
+
layer_outputs = layer(
|
| 414 |
+
hidden_states,
|
| 415 |
+
attention_mask=causal_mask,
|
| 416 |
+
position_ids=position_ids,
|
| 417 |
+
past_key_value=past_key_values,
|
| 418 |
+
output_attentions=output_attentions,
|
| 419 |
+
use_cache=use_cache,
|
| 420 |
+
cache_position=cache_position,
|
| 421 |
+
value_residual=value_residual,
|
| 422 |
+
**kwargs,
|
| 423 |
+
)
|
| 424 |
+
hidden_states = layer_outputs[0]
|
| 425 |
+
|
| 426 |
+
if output_attentions:
|
| 427 |
+
all_self_attns += (layer_outputs[1],)
|
| 428 |
+
|
| 429 |
+
hidden_states = self.norm(hidden_states)
|
| 430 |
+
|
| 431 |
+
if output_hidden_states:
|
| 432 |
+
all_hidden_states += (hidden_states,)
|
| 433 |
+
|
| 434 |
+
next_cache = past_key_values if use_cache else None
|
| 435 |
+
|
| 436 |
+
if not return_dict:
|
| 437 |
+
return tuple(
|
| 438 |
+
v
|
| 439 |
+
for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
|
| 440 |
+
if v is not None
|
| 441 |
+
)
|
| 442 |
+
return BaseModelOutputWithPast(
|
| 443 |
+
last_hidden_state=hidden_states,
|
| 444 |
+
past_key_values=next_cache,
|
| 445 |
+
hidden_states=all_hidden_states,
|
| 446 |
+
attentions=all_self_attns,
|
| 447 |
+
)
|
| 448 |
+
|
| 449 |
+
def _update_causal_mask(
|
| 450 |
+
self,
|
| 451 |
+
attention_mask: Optional[torch.Tensor],
|
| 452 |
+
input_tensor: torch.Tensor,
|
| 453 |
+
cache_position: torch.Tensor,
|
| 454 |
+
past_key_values: Optional[Cache],
|
| 455 |
+
) -> Optional[torch.Tensor]:
|
| 456 |
+
# When there's no explicit attention_mask (no padding) and we're doing
|
| 457 |
+
# standard prefill or cached decoding, return None and let SDPA use
|
| 458 |
+
# its efficient is_causal path. This also avoids numerical issues
|
| 459 |
+
# from float min mask values with large activations.
|
| 460 |
+
if attention_mask is None:
|
| 461 |
+
return None
|
| 462 |
+
if torch.all(attention_mask == 1):
|
| 463 |
+
return None
|
| 464 |
+
|
| 465 |
+
# Only build an explicit mask when we have a padding mask
|
| 466 |
+
dtype, device = input_tensor.dtype, input_tensor.device
|
| 467 |
+
sequence_length = input_tensor.shape[1]
|
| 468 |
+
|
| 469 |
+
if past_key_values is not None:
|
| 470 |
+
target_length = past_key_values.get_max_cache_shape()
|
| 471 |
+
if target_length is None or target_length < 0:
|
| 472 |
+
target_length = (
|
| 473 |
+
cache_position[-1].item() + 1 if cache_position is not None else sequence_length
|
| 474 |
+
)
|
| 475 |
+
else:
|
| 476 |
+
target_length = sequence_length
|
| 477 |
+
|
| 478 |
+
causal_mask = torch.full(
|
| 479 |
+
(sequence_length, target_length),
|
| 480 |
+
fill_value=torch.finfo(dtype).min,
|
| 481 |
+
dtype=dtype,
|
| 482 |
+
device=device,
|
| 483 |
+
)
|
| 484 |
+
if sequence_length != 1:
|
| 485 |
+
causal_mask = torch.triu(causal_mask, diagonal=target_length - sequence_length + 1)
|
| 486 |
+
|
| 487 |
+
causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1)
|
| 488 |
+
|
| 489 |
+
if attention_mask.dim() == 2:
|
| 490 |
+
mask_length = attention_mask.shape[-1]
|
| 491 |
+
padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(
|
| 492 |
+
causal_mask.dtype
|
| 493 |
+
).masked_fill(attention_mask[:, None, None, :] == 0, torch.finfo(dtype).min)
|
| 494 |
+
causal_mask = (
|
| 495 |
+
torch.cat([padding_mask, causal_mask[:, :, :, mask_length:]], dim=-1)
|
| 496 |
+
if mask_length < target_length
|
| 497 |
+
else padding_mask
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
return causal_mask
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
class RicursiveForCausalLM(RicursivePreTrainedModel):
|
| 504 |
+
_tied_weights_keys = []
|
| 505 |
+
|
| 506 |
+
def __init__(self, config: RicursiveConfig):
|
| 507 |
+
super().__init__(config)
|
| 508 |
+
self.model = RicursiveModel(config)
|
| 509 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 510 |
+
self.post_init()
|
| 511 |
+
|
| 512 |
+
def get_input_embeddings(self):
|
| 513 |
+
return self.model.embed_tokens
|
| 514 |
+
|
| 515 |
+
def set_input_embeddings(self, value):
|
| 516 |
+
self.model.embed_tokens = value
|
| 517 |
+
|
| 518 |
+
def get_output_embeddings(self):
|
| 519 |
+
return self.lm_head
|
| 520 |
+
|
| 521 |
+
def set_output_embeddings(self, new_embeddings):
|
| 522 |
+
self.lm_head = new_embeddings
|
| 523 |
+
|
| 524 |
+
def forward(
|
| 525 |
+
self,
|
| 526 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 527 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 528 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 529 |
+
past_key_values: Optional[Cache] = None,
|
| 530 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 531 |
+
labels: Optional[torch.LongTensor] = None,
|
| 532 |
+
use_cache: Optional[bool] = None,
|
| 533 |
+
output_attentions: Optional[bool] = None,
|
| 534 |
+
output_hidden_states: Optional[bool] = None,
|
| 535 |
+
return_dict: Optional[bool] = None,
|
| 536 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 537 |
+
**kwargs,
|
| 538 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 539 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 540 |
+
|
| 541 |
+
outputs = self.model(
|
| 542 |
+
input_ids=input_ids,
|
| 543 |
+
attention_mask=attention_mask,
|
| 544 |
+
position_ids=position_ids,
|
| 545 |
+
past_key_values=past_key_values,
|
| 546 |
+
inputs_embeds=inputs_embeds,
|
| 547 |
+
use_cache=use_cache,
|
| 548 |
+
output_attentions=output_attentions,
|
| 549 |
+
output_hidden_states=output_hidden_states,
|
| 550 |
+
return_dict=return_dict,
|
| 551 |
+
cache_position=cache_position,
|
| 552 |
+
**kwargs,
|
| 553 |
+
)
|
| 554 |
+
hidden_states = outputs[0]
|
| 555 |
+
logits = self.lm_head(hidden_states)
|
| 556 |
+
|
| 557 |
+
loss = None
|
| 558 |
+
if labels is not None:
|
| 559 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 560 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 561 |
+
loss = F.cross_entropy(
|
| 562 |
+
shift_logits.view(-1, shift_logits.size(-1)),
|
| 563 |
+
shift_labels.view(-1),
|
| 564 |
+
)
|
| 565 |
+
|
| 566 |
+
if not return_dict:
|
| 567 |
+
output = (logits,) + outputs[1:]
|
| 568 |
+
return (loss,) + output if loss is not None else output
|
| 569 |
+
|
| 570 |
+
return CausalLMOutputWithPast(
|
| 571 |
+
loss=loss,
|
| 572 |
+
logits=logits,
|
| 573 |
+
past_key_values=outputs.past_key_values,
|
| 574 |
+
hidden_states=outputs.hidden_states,
|
| 575 |
+
attentions=outputs.attentions,
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
def prepare_inputs_for_generation(
|
| 579 |
+
self,
|
| 580 |
+
input_ids: torch.LongTensor,
|
| 581 |
+
past_key_values: Optional[Cache] = None,
|
| 582 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 583 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 584 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 585 |
+
use_cache: Optional[bool] = None,
|
| 586 |
+
**kwargs,
|
| 587 |
+
):
|
| 588 |
+
if past_key_values is not None:
|
| 589 |
+
if input_ids.shape[1] != 1:
|
| 590 |
+
input_ids = input_ids[:, cache_position]
|
| 591 |
+
|
| 592 |
+
if inputs_embeds is not None and cache_position[0] == 0:
|
| 593 |
+
model_inputs = {"inputs_embeds": inputs_embeds}
|
| 594 |
+
else:
|
| 595 |
+
model_inputs = {"input_ids": input_ids}
|
| 596 |
+
|
| 597 |
+
model_inputs.update(
|
| 598 |
+
{
|
| 599 |
+
"position_ids": cache_position.unsqueeze(0) if cache_position is not None else None,
|
| 600 |
+
"past_key_values": past_key_values,
|
| 601 |
+
"use_cache": use_cache,
|
| 602 |
+
"attention_mask": attention_mask,
|
| 603 |
+
"cache_position": cache_position,
|
| 604 |
+
}
|
| 605 |
+
)
|
| 606 |
+
return model_inputs
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"bos_token": {"content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}, "eos_token": "<|im_end|>", "pad_token": {"content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}, "unk_token": {"content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"added_tokens_decoder": {
|
| 4 |
+
"100256": {
|
| 5 |
+
"content": "<|extra_id_0|>",
|
| 6 |
+
"lstrip": false,
|
| 7 |
+
"normalized": false,
|
| 8 |
+
"rstrip": false,
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"special": false
|
| 11 |
+
},
|
| 12 |
+
"100257": {
|
| 13 |
+
"content": "<|endoftext|>",
|
| 14 |
+
"lstrip": false,
|
| 15 |
+
"normalized": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"single_word": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
"100258": {
|
| 21 |
+
"content": "<|fim_prefix|>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": false,
|
| 25 |
+
"single_word": false,
|
| 26 |
+
"special": true
|
| 27 |
+
},
|
| 28 |
+
"100259": {
|
| 29 |
+
"content": "<|fim_middle|>",
|
| 30 |
+
"lstrip": false,
|
| 31 |
+
"normalized": false,
|
| 32 |
+
"rstrip": false,
|
| 33 |
+
"single_word": false,
|
| 34 |
+
"special": true
|
| 35 |
+
},
|
| 36 |
+
"100260": {
|
| 37 |
+
"content": "<|fim_suffix|>",
|
| 38 |
+
"lstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"rstrip": false,
|
| 41 |
+
"single_word": false,
|
| 42 |
+
"special": true
|
| 43 |
+
},
|
| 44 |
+
"100261": {
|
| 45 |
+
"content": "|||PHONE_NUMBER|||",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false,
|
| 50 |
+
"special": false
|
| 51 |
+
},
|
| 52 |
+
"100262": {
|
| 53 |
+
"content": "|||EMAIL_ADDRESS|||",
|
| 54 |
+
"lstrip": false,
|
| 55 |
+
"normalized": false,
|
| 56 |
+
"rstrip": false,
|
| 57 |
+
"single_word": false,
|
| 58 |
+
"special": false
|
| 59 |
+
},
|
| 60 |
+
"100263": {
|
| 61 |
+
"content": "|||IP_ADDRESS|||",
|
| 62 |
+
"lstrip": false,
|
| 63 |
+
"normalized": false,
|
| 64 |
+
"rstrip": false,
|
| 65 |
+
"single_word": false,
|
| 66 |
+
"special": false
|
| 67 |
+
},
|
| 68 |
+
"100264": {
|
| 69 |
+
"content": "<|im_start|>",
|
| 70 |
+
"lstrip": false,
|
| 71 |
+
"normalized": false,
|
| 72 |
+
"rstrip": false,
|
| 73 |
+
"single_word": false,
|
| 74 |
+
"special": true
|
| 75 |
+
},
|
| 76 |
+
"100265": {
|
| 77 |
+
"content": "<|im_end|>",
|
| 78 |
+
"lstrip": false,
|
| 79 |
+
"normalized": false,
|
| 80 |
+
"rstrip": false,
|
| 81 |
+
"single_word": false,
|
| 82 |
+
"special": true
|
| 83 |
+
},
|
| 84 |
+
"100266": {
|
| 85 |
+
"content": "<|extra_id_1|>",
|
| 86 |
+
"lstrip": false,
|
| 87 |
+
"normalized": false,
|
| 88 |
+
"rstrip": false,
|
| 89 |
+
"single_word": false,
|
| 90 |
+
"special": false
|
| 91 |
+
},
|
| 92 |
+
"100267": {
|
| 93 |
+
"content": "<|extra_id_2|>",
|
| 94 |
+
"lstrip": false,
|
| 95 |
+
"normalized": false,
|
| 96 |
+
"rstrip": false,
|
| 97 |
+
"single_word": false,
|
| 98 |
+
"special": false
|
| 99 |
+
},
|
| 100 |
+
"100268": {
|
| 101 |
+
"content": "<|extra_id_3|>",
|
| 102 |
+
"lstrip": false,
|
| 103 |
+
"normalized": false,
|
| 104 |
+
"rstrip": false,
|
| 105 |
+
"single_word": false,
|
| 106 |
+
"special": false
|
| 107 |
+
},
|
| 108 |
+
"100269": {
|
| 109 |
+
"content": "<|extra_id_4|>",
|
| 110 |
+
"lstrip": false,
|
| 111 |
+
"normalized": false,
|
| 112 |
+
"rstrip": false,
|
| 113 |
+
"single_word": false,
|
| 114 |
+
"special": false
|
| 115 |
+
},
|
| 116 |
+
"100270": {
|
| 117 |
+
"content": "<|extra_id_5|>",
|
| 118 |
+
"lstrip": false,
|
| 119 |
+
"normalized": false,
|
| 120 |
+
"rstrip": false,
|
| 121 |
+
"single_word": false,
|
| 122 |
+
"special": false
|
| 123 |
+
},
|
| 124 |
+
"100271": {
|
| 125 |
+
"content": "<|extra_id_6|>",
|
| 126 |
+
"lstrip": false,
|
| 127 |
+
"normalized": false,
|
| 128 |
+
"rstrip": false,
|
| 129 |
+
"single_word": false,
|
| 130 |
+
"special": false
|
| 131 |
+
},
|
| 132 |
+
"100272": {
|
| 133 |
+
"content": "<|extra_id_7|>",
|
| 134 |
+
"lstrip": false,
|
| 135 |
+
"normalized": false,
|
| 136 |
+
"rstrip": false,
|
| 137 |
+
"single_word": false,
|
| 138 |
+
"special": false
|
| 139 |
+
},
|
| 140 |
+
"100273": {
|
| 141 |
+
"content": "<|extra_id_8|>",
|
| 142 |
+
"lstrip": false,
|
| 143 |
+
"normalized": false,
|
| 144 |
+
"rstrip": false,
|
| 145 |
+
"single_word": false,
|
| 146 |
+
"special": false
|
| 147 |
+
},
|
| 148 |
+
"100274": {
|
| 149 |
+
"content": "<|extra_id_9|>",
|
| 150 |
+
"lstrip": false,
|
| 151 |
+
"normalized": false,
|
| 152 |
+
"rstrip": false,
|
| 153 |
+
"single_word": false,
|
| 154 |
+
"special": false
|
| 155 |
+
},
|
| 156 |
+
"100275": {
|
| 157 |
+
"content": "<|extra_id_10|>",
|
| 158 |
+
"lstrip": false,
|
| 159 |
+
"normalized": false,
|
| 160 |
+
"rstrip": false,
|
| 161 |
+
"single_word": false,
|
| 162 |
+
"special": false
|
| 163 |
+
},
|
| 164 |
+
"100276": {
|
| 165 |
+
"content": "<|endofprompt|>",
|
| 166 |
+
"lstrip": false,
|
| 167 |
+
"normalized": false,
|
| 168 |
+
"rstrip": false,
|
| 169 |
+
"single_word": false,
|
| 170 |
+
"special": true
|
| 171 |
+
},
|
| 172 |
+
"100277": {
|
| 173 |
+
"content": "<|pad|>",
|
| 174 |
+
"lstrip": false,
|
| 175 |
+
"normalized": false,
|
| 176 |
+
"rstrip": false,
|
| 177 |
+
"single_word": false,
|
| 178 |
+
"special": true
|
| 179 |
+
}
|
| 180 |
+
},
|
| 181 |
+
"bos_token": "<|endoftext|>",
|
| 182 |
+
"chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}",
|
| 183 |
+
"clean_up_tokenization_spaces": false,
|
| 184 |
+
"eos_token": "<|im_end|>",
|
| 185 |
+
"extra_special_tokens": {},
|
| 186 |
+
"model_max_length": 131072,
|
| 187 |
+
"pad_token": "<|pad|>",
|
| 188 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 189 |
+
"unk_token": "<|endoftext|>"
|
| 190 |
+
}
|