xsingh commited on
Commit
df03673
·
0 Parent(s):

Duplicate from Adaop/SGI-Preview

Browse files
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
README.md ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ ---
4
+ # Ripple-65B Preview
5
+
6
+ <div align="center">
7
+ <picture>
8
+ <source media="(prefers-color-scheme: dark)" srcset="https://framerusercontent.com/images/rSaJGoPP1Saln5kpScZNGCw.png" width="30%">
9
+ <source media="(prefers-color-scheme: light)" srcset="https://framerusercontent.com/images/3rFANSmoZuSBhjRwxZolyEU5guo.png" width="30%">
10
+ <img alt="Ricursive Logo" src="https://framerusercontent.com/images/3rFANSmoZuSBhjRwxZolyEU5guo.png" width="30%">
11
+ </picture>
12
+ </div>
13
+
14
+ <p align="center">
15
+ <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview">BF16</a> |
16
+ <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview-NVFP4">NVFP4</a> |
17
+ <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Base-Preview">Base Model</a> |
18
+ <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview-DFlash2">DFlash2</a> |
19
+ <a href="https://www.ricursive.com/ripple">Launch Blog</a> |
20
+ License: <a href="https://huggingface.co/ricursiveAI/Ripple-65B-Preview/blob/main/LICENSE">Apache 2.0</a>
21
+ </p>
22
+
23
+ Ricursive Intelligence introduces **Ripple-65B Preview**, a 65B parameter dense Transformer model pretrained on 12 trillion tokens with support for up to 1M context length.
24
+ Ripple-65B Preview delivers strong results on agentic coding, scoring **80.52 on TerminalBench 2.1** and
25
+ surpassing significantly-larger US open-weight models such as Reflection Beam 501B (**80.1**), Meta Muse Spark Open 816B (**76.4**),
26
+ Thinking Machines Inkling 975B (**63.8**), and NVIDIA Nemotron 3 Ultra 550B (**56.4**).
27
+
28
+ Our Preview release includes an instruction-tuned model in BF16 and NVFP4 numerics, and a BF16 base model. **This repo contains the BF16 instruction-tuned model.**
29
+
30
+ Starting from an initial Olmo 3 baseline, we iteratively improved all aspects of the training recipe including pretraining scaling laws, model architecture, data, and posttraining stack.
31
+ Our final Ripple v1 model ladder improves Compute Efficiency (CE) on code by **16x** relative to the Olmo 3 ladder, with the advantage further increasing with scale up to
32
+ **55x** for the full hero run.
33
+
34
+ Ripple-65B was codesigned to be trained and served efficiently on NVIDIA hardware.
35
+ For our first model, a key objective was to make design decisions which reduce pretraining and posttraining
36
+ infrastructure complexity while still enabling strong reasoning and agentic performance.
37
+ Therefore, we chose to train a modernized dense Transformer model rather than use Mixture-of-Experts (MoE) like other labs.
38
+ This decision quickly paid off, and we were able to train a strong model from scratch with a small team that scaled from 1 to 5 people during development,
39
+ while maintaining solid MFU (40%) throughout pretraining and posttraining with simple sharding, no custom kernels, and no quantization (BF16).
40
+
41
+ Ripple-65B uses a **thin-and-deep** model aspect ratio, with a narrow d_model (5120) and 80 total layers.
42
+ We switch from the Olmo 3 llama-like architecture to a Gemma 3N-like architecture to improve training stability at scale.
43
+ To improve reasoning performance, we **overprovision the number of attention heads** while increasing the SWA-to-global layer ratio
44
+ and reducing SWA window size to compensate for the increased cost.
45
+ In addition, we added FFN activation clipping, gated attention, global NoPE, and value residuals.
46
+
47
+ The end result is a model that is especially capable at reasoning and agentic tasks given its model size, matching or exceeding other
48
+ US open-weight models with more than 10x the number of total parameters.
49
+ At only 65B total parameters, the model can fit on consumer hardware such as the NVIDIA DGX Spark.
50
+ To encourage local model development, we provide an NVFP4 checkpoint and colab to help you get started.
51
+ The model is also fast at token generation when running on datacenter GPUs, reaching 250 tok/s on 2xB200 batch=8 prior to quantization or custom kernels.
52
+
53
+ **Pre-training.** Ripple-65B Preview is pretrained from scratch on our 12T text-only predominantly-English and code corpus at 8K sequence length.
54
+ Ripple-65B Base Preview is a **pretrain-only checkpoint** without any midtraining or synthetic QA data.
55
+
56
+ **Post-training.** We use a standard post-training recipe consisting of midtraining, long context, SFT, long context SFT, and mixed domain RL.
57
+ Our Preview release has gone through relatively light posttraining, with a focus on agentic coding.
58
+
59
+ We show a brief overview of each stage below:
60
+
61
+ <div align="center">
62
+
63
+ | Stage | Tokens | Step | Seqlen | Batch Size |
64
+ | ----- | ------ | ------ | ---- | ----- |
65
+ | Pretrain | 12T | 877,500 | 8K | 1408 -> 3072 |
66
+ | Midtrain | 200B | 8,000 | 64K | 384 |
67
+ | Long Context | 6.5B | 389 | 2M | 8 |
68
+ | SFT | 183B | 7,274 | 128K | 192 |
69
+ | LC SFT | 13.5B | 802 | 2M | 8 |
70
+ | RL | N/A | 700 | 128K | 2048 |
71
+
72
+ </div>
73
+
74
+ ## Instruct Model Benchmarks
75
+
76
+ We compare Ripple-65B Preview against currently-available US open-weight frontier models.
77
+ All models are tested at full BF16 precision with maximum effort levels.
78
+
79
+ Each lab typically only provides a limited subset of available benchmarks.
80
+ Our table below shows official model card scores where available, and we use our own in-house eval flow for missing evals (italized).
81
+ For SWE-bench Pro v2.0 Hard and DeepSWE 1.1, we use Terminus 2 for Ripple-65B Preview.
82
+ In other cases where the harness is not specified, we use OpenCode.
83
+ A higher number is better for all benchmarks.
84
+
85
+ Replicating benchmark scores has been challenging, particularly for modern agentic evals, due to the number of factors which can cause
86
+ eval scores to shift.
87
+ For example, we found that Terminal-Bench 2.1 scores were universally lower (with the exception of Muse Spark Open)
88
+ when using the official Terminus 2 harness at 128K sequence length with summarization.
89
+ Therefore, our results represent a best-effort attempt at a fair comparison.
90
+
91
+ <div align="center">
92
+
93
+ | Benchmark | Ripple-65B<br>Preview | Nemotron 3<br>Ultra 550B | Inkling<br>975B | Muse Spark<br>Open 816B | Reflection<br>Beam 501B |
94
+ |---|---|---|---|---|---|
95
+ | **Model** | | | | | |
96
+ | Architecture | Dense | MoE | MoE | MoE | MoE |
97
+ | Total params | 65B | 550B | **975B** | 816B | 501B |
98
+ | Active params | **65B** | 55B | 41B | 42B | 23B |
99
+ | Pre-training tokens | 12T | 20T | **45T** | N/A | 23.8T |
100
+ | **Knowledge** | | | | | |
101
+ | MMLU | 89.6 | *92.0* | *91.3* | ***92.6*** | *92.4* |
102
+ | MMLU-Pro | 84.2 | 86.8 | *87.3* | ***87.9*** | *76.5* |
103
+ | GPQA Diamond | 85.2 | 87.0 | 87.2 | 89.0 | **90.6** |
104
+ | **Math** | | | | | |
105
+ | AIME 2024 | 97.1 | *94.2* | *97.5* | ***97.9*** | *96.7* |
106
+ | AIME 2025 | 98.3 | *94.2* | *91.7* | ***100.0*** | *99.6* |
107
+ | AIME 2026 | 95.0 | *95.4* | 97.1 | ***98.3*** | 97.7 |
108
+ | HMMT Feb 2026 | 90.5 | *80.3* | 86.3 | ***93.9*** | ***93.9*** |
109
+ | **Coding & Agentic** | | | | | |
110
+ | Terminal-Bench 2.1 (self-reported) | **80.5** | 56.4 | 63.8 | 76.4 | 80.1 |
111
+ | Terminal-Bench 2.1 (Terminus 2) | **80.5** | *55.1* | *56.2* | *78.7* | *70.8* |
112
+ | SWE-bench Verified | 76.5 | 71.9 | 77.6 | *78.7* | **81.6** |
113
+ | SWE-bench Pro v1.0 | 61.42 | *46.9* | 54.3 | *58.2* | **66.1** |
114
+ | SWE-bench Pro v2.0 Hard | **79.1** | *30.7* | *2.0* | *52.3* | 76.5 |
115
+ | DeepSWE 1.1 | 44.3 | *0.2* | *1.8* | **60.0** | 43.3 |
116
+ | SciCode (Inspect AI) | 39.4 | 44.6 | 46.1 | **58.5** | 49.2 |
117
+ | τ³-bench banking | 31.27 | 22.6 | 23.7 | *14.4* | **36.1** |
118
+ | AutomationBench (Public) | 36.4 | *8.4* | *14.6* | *42.4* | 39.3 |
119
+ | MCP Atlas | 71.67 | *62.2* | 76.0 | **89.2** | 80.8 |
120
+ | **Safety** | | | | | |
121
+ | FORTRESS Adversarial | 83.8 | *79.3* | 78.0 | **94.2** | *92.1* |
122
+ | FORTRESS Benign | 95.5 | *89.1* | **95.9** | *82.6* | *86.4* |
123
+ | StrongREJECT | 97.5 | *98.2* | **98.6** | *94.7* | *95.8* |
124
+ | **Instruction Following** | | | | | |
125
+ | IFBench | 79.2 | **81.7** | 79.8 | *78.6* | 79.6 |
126
+ | **Long Context** | | | | | |
127
+ | AA-LCR | 75.0 | 65.4 | *78.3* | ***80.7*** | 78.2 |
128
+
129
+ </div>
130
+
131
+
132
+ ## Base Model Benchmarks
133
+
134
+ We show traditional pretraining evals comparing Ripple-65B Base Preview against a selection of open-weight base models. Although we would have liked to
135
+ compare against the same set of models throughout the model card, many companies are now choosing to only open-source their instruction-tuned models.
136
+
137
+ With the rise of the importance of posttraining, many modern LLM recipes now blend posttraining data earlier into the training process, including during pretraining.
138
+ For our first model, we kept our pretraining data clean with minimal synthetic data, focusing on high quality natural data rather than aggressively targeting evals.
139
+ We instead added significantly more synthetic data during midtraining, which allowed us to quickly iterate on our synthetic data recipe without blocking pretraining.
140
+
141
+ Ripple-65B Base Preview is a pretrain-only checkpoint. The base model overall beats Poolside Laguna M.1,
142
+ and is comparable with DeepSeek V3.1 and V4 Flash on reasoning, math, and code.
143
+ Nemotron 3 Ultra dominates on traditional pretraining evals, likely due to their heavy use of [task-seeded synthetic data](https://huggingface.co/blog/nvidia/task-seeded-sdg).
144
+ However, its pretraining eval performance did not translate to modern agentic tasks, likely due to their synthetic data being out of distribution relative to these tasks.
145
+
146
+ We report official model results where available.
147
+ For our own model and for missing evals, we use our internal fork of [Olmes](https://github.com/allenai/olmes).
148
+
149
+ <div align="center">
150
+
151
+ | Benchmark (Metric) | # Shots | Ripple-65B<br>Preview | Laguna<br>M.1 225B | DeepSeek<br>V3.1 671B | DeepSeek<br>V4 Flash 284B | Nemotron 3<br>Ultra 550B |
152
+ |---|---|---|---|---|---|---|
153
+ | **Model** | | | | | | |
154
+ | Architecture | — | Dense | MoE | MoE | MoE | MoE |
155
+ | Total params | — | 65B | 225B | **671B** | 284B | 550B |
156
+ | Active params | — | **65B** | 23B | 37B | 13B | 55B |
157
+ | Training tokens (T) | — | 12T | 30T+ | 15.6T | **32T+** | 20T |
158
+ | **World Knowledge** | | | | | | |
159
+ | MMLU (MC acc) | 5-shot | *83.7* | *76.6* | *88.3* | 88.7 | **89.1** |
160
+ | MMLU-Pro (CoT EM) | 5-shot | *61.5* | *52.2* | *57.8* | 68.3 | **79.1** |
161
+ | AGIEval-En (acc) | 3-5-shot | *74.4* | *66.2* | *81.9* | **83.9** | 78.7 |
162
+ | GPQA main (CoT EM) | 5-shot | *37.7* | *31.7* | *30.8* | *44.0* | ***49.6*** |
163
+ | GPQA-Diamond (CoT EM) | 5-shot | *38.4* | *27.8* | *37.4* | *44.4* | ***53.0*** |
164
+ | C-Eval (MC acc) | 5-shot | *69.6* | *59.3* | *90.7* | **92.1** | *82.2* |
165
+ | SuperGPQA (MC acc) | 5-shot | *34.2* | *24.5* | *43.1* | 46.5 | ***52.9*** |
166
+ | SimpleQA-Verified (EM) | 25-shot | *10.1* | *10.8* | *17.3* | **30.1** | *27.1* |
167
+ | **Language & Reasoning** | | | | | | |
168
+ | BBH (CoT EM) | 3-shot | *85.6* | *79.6* | *87.2* | 86.9 | ***88.0*** |
169
+ | BBEH (EM) | 1-shot | *17.9* | *14.2* | *18.4* | **25.4** | *21.2* |
170
+ | DROP (F1) | 1-shot | *77.8* | *66.8* | *81.2* | **88.6** | *87.6* |
171
+ | HellaSwag (acc_norm) | 0-shot | *84.6* | *86.2* | *86.8* | 85.7 | ***89.1*** |
172
+ | HellaSwag (acc_norm) | 10-shot | *87.2* | *88.1* | *89.4* | *88.8* | **90.5** |
173
+ | ARC-Challenge (MC acc) | 25-shot | *95.7* | *90.8* | *95.1* | *95.8* | **97.4** |
174
+ | OpenBookQA (acc_norm) | 0-shot | *47.4* | *48.6* | ***50.4*** | *48.0* | 48.6 |
175
+ | PIQA (acc_norm) | 0-shot | *83.6* | *84.8* | ***85.2*** | *84.7* | 83.8 |
176
+ | WinoGrande (acc) | 5-shot | ***83.4*** | *83.2* | *83.2* | *81.4* | 79.3 |
177
+ | **Code & Math** | | | | | | |
178
+ | HumanEval (pass@1, n=20, T=0.2) | 0-shot | *70.9* | *83.4* | *69.8* | 69.5 | **83.8** |
179
+ | MBPP Sanitized (pass@1, n=32, T=0.2) | 3-shot | *81.5* | *81.1* | *75.2* | *74.6* | **86.0** |
180
+ | BigCodeBench (pass@1, n=5, T=0.8) | 3-shot | *49.2* | *50.5* | ***59.9*** | 56.8 | *55.6* |
181
+ | GSM8K (CoT EM) | 8-shot | *88.5* | *80.4* | *79.6* | **90.8** | 88.1 |
182
+ | GSM1K (CoT EM) | 8-shot | *88.4* | *79.7* | *81.8* | *83.2* | ***90.2*** |
183
+ | MATH (flex EM) | 4-shot | *59.1* | *52.5* | *59.7* | 57.4 | **82.0** |
184
+ | MGSM (CoT EM, 11-lang avg) | 8-shot | *78.2* | *65.4* | *81.7* | 85.7 | **87.7** |
185
+
186
+
187
+ </div>
188
+
189
+ ## Citation
190
+
191
+ ```bibtex
192
+ @misc{ricursive2026ripple65b,
193
+ title={Ripple-65B: Pushing the Frontier of Local Intelligence},
194
+ author={Ricursive Intelligence},
195
+ year={2026},
196
+ }
197
+ ```
198
+
199
+ ## Contact
200
+
201
+ For questions and comments, please contact us at [info@ricursive.com](mailto:info@ricursive.com). To learn more, please visit our [website](https://www.ricursive.com).
chat_template.jinja ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0]['role'] == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- for message in messages %}
18
+ {%- set content = message.content or '' %}
19
+ {%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
20
+ {%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
21
+ {%- endif %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if content %}
27
+ {{- '\n' + content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": ' }}
34
+ {{- tool_call.name | tojson }}
35
+ {{- ', "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
45
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
46
+ {{- '<|im_end|>\n' }}
47
+ {%- endif %}
48
+ {%- endif %}
49
+ {%- endfor %}
50
+ {%- if add_generation_prompt %}
51
+ {{- '<|im_start|>assistant\n' }}
52
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "RicursiveForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_ricursive.RicursiveConfig",
7
+ "AutoModel": "modeling_ricursive.RicursiveModel",
8
+ "AutoModelForCausalLM": "modeling_ricursive.RicursiveForCausalLM"
9
+ },
10
+ "bos_token_id": 100257,
11
+ "embed_scale": 71.55417527999327,
12
+ "eos_token_id": [
13
+ 100265,
14
+ 100257
15
+ ],
16
+ "ffn_clamp_threshold": 20.0,
17
+ "gate_full_precision": true,
18
+ "head_dim": 128,
19
+ "hidden_act": "silu",
20
+ "hidden_size": 5120,
21
+ "hybrid_layer_pattern": [
22
+ 1,
23
+ 1,
24
+ 1,
25
+ 1,
26
+ 0,
27
+ 1,
28
+ 1,
29
+ 1,
30
+ 1,
31
+ 0,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 0,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 0,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 0,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 0,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 0,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 0,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 0,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 0,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 0,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 0,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 0,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 0,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 0,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 0
102
+ ],
103
+ "intermediate_size": 40960,
104
+ "is_hybrid_swa": true,
105
+ "layer_types": [
106
+ "sliding_attention",
107
+ "sliding_attention",
108
+ "sliding_attention",
109
+ "sliding_attention",
110
+ "full_attention",
111
+ "sliding_attention",
112
+ "sliding_attention",
113
+ "sliding_attention",
114
+ "sliding_attention",
115
+ "full_attention",
116
+ "sliding_attention",
117
+ "sliding_attention",
118
+ "sliding_attention",
119
+ "sliding_attention",
120
+ "full_attention",
121
+ "sliding_attention",
122
+ "sliding_attention",
123
+ "sliding_attention",
124
+ "sliding_attention",
125
+ "full_attention",
126
+ "sliding_attention",
127
+ "sliding_attention",
128
+ "sliding_attention",
129
+ "sliding_attention",
130
+ "full_attention",
131
+ "sliding_attention",
132
+ "sliding_attention",
133
+ "sliding_attention",
134
+ "sliding_attention",
135
+ "full_attention",
136
+ "sliding_attention",
137
+ "sliding_attention",
138
+ "sliding_attention",
139
+ "sliding_attention",
140
+ "full_attention",
141
+ "sliding_attention",
142
+ "sliding_attention",
143
+ "sliding_attention",
144
+ "sliding_attention",
145
+ "full_attention",
146
+ "sliding_attention",
147
+ "sliding_attention",
148
+ "sliding_attention",
149
+ "sliding_attention",
150
+ "full_attention",
151
+ "sliding_attention",
152
+ "sliding_attention",
153
+ "sliding_attention",
154
+ "sliding_attention",
155
+ "full_attention",
156
+ "sliding_attention",
157
+ "sliding_attention",
158
+ "sliding_attention",
159
+ "sliding_attention",
160
+ "full_attention",
161
+ "sliding_attention",
162
+ "sliding_attention",
163
+ "sliding_attention",
164
+ "sliding_attention",
165
+ "full_attention",
166
+ "sliding_attention",
167
+ "sliding_attention",
168
+ "sliding_attention",
169
+ "sliding_attention",
170
+ "full_attention",
171
+ "sliding_attention",
172
+ "sliding_attention",
173
+ "sliding_attention",
174
+ "sliding_attention",
175
+ "full_attention",
176
+ "sliding_attention",
177
+ "sliding_attention",
178
+ "sliding_attention",
179
+ "sliding_attention",
180
+ "full_attention",
181
+ "sliding_attention",
182
+ "sliding_attention",
183
+ "sliding_attention",
184
+ "sliding_attention",
185
+ "full_attention"
186
+ ],
187
+ "max_position_embeddings": 2097152,
188
+ "model_type": "ricursive",
189
+ "num_attention_heads": 80,
190
+ "num_hidden_layers": 80,
191
+ "num_key_value_heads": 8,
192
+ "pad_token_id": 100277,
193
+ "rms_norm_eps": 1e-06,
194
+ "rope_theta": 10000.0,
195
+ "sliding_window": 1024,
196
+ "tie_word_embeddings": false,
197
+ "transformers_version": "4.57.6",
198
+ "use_cache": true,
199
+ "use_gated_attention": true,
200
+ "use_valres": true,
201
+ "vocab_size": 100352
202
+ }
configuration_ricursive.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HuggingFace PretrainedConfig for the Ricursive transformer architecture."""
2
+
3
+ from transformers import PretrainedConfig
4
+
5
+
6
+ class RicursiveConfig(PretrainedConfig):
7
+ """
8
+ Configuration for the Ricursive transformer.
9
+
10
+ Key architectural features:
11
+ - PeriNorm blocks (4 layer norms per block instead of 2)
12
+ - Gated attention with elementwise sigmoid gate
13
+ - Mixed local/global layers: local layers use sliding window + RoPE,
14
+ global layers (every 5th) use full attention with no RoPE
15
+ - Per-head Q/K RMSNorm
16
+ """
17
+
18
+ model_type = "ricursive"
19
+
20
+ def __init__(
21
+ self,
22
+ vocab_size=100352,
23
+ hidden_size=3072,
24
+ intermediate_size=24576,
25
+ num_hidden_layers=50,
26
+ num_attention_heads=48,
27
+ num_key_value_heads=8,
28
+ head_dim=128,
29
+ max_position_embeddings=2048,
30
+ hidden_act="silu",
31
+ rms_norm_eps=1e-6,
32
+ rope_theta=10000.0,
33
+ sliding_window=1024,
34
+ layer_types=None,
35
+ use_gated_attention=True,
36
+ gate_full_precision=True,
37
+ use_valres=False,
38
+ ffn_clamp_threshold=None,
39
+ embed_scale=None,
40
+ tie_word_embeddings=False,
41
+ **kwargs,
42
+ ):
43
+ self.vocab_size = vocab_size
44
+ self.hidden_size = hidden_size
45
+ self.intermediate_size = intermediate_size
46
+ self.num_hidden_layers = num_hidden_layers
47
+ self.num_attention_heads = num_attention_heads
48
+ self.num_key_value_heads = num_key_value_heads
49
+ self.head_dim = head_dim
50
+ self.max_position_embeddings = max_position_embeddings
51
+ self.hidden_act = hidden_act
52
+ self.rms_norm_eps = rms_norm_eps
53
+ self.rope_theta = rope_theta
54
+ self.sliding_window = sliding_window
55
+ self.use_gated_attention = use_gated_attention
56
+ self.gate_full_precision = gate_full_precision
57
+ self.use_valres = use_valres
58
+ self.ffn_clamp_threshold = ffn_clamp_threshold
59
+ # Hybrid-SWA metadata is derived below. Serialized copies must not override the
60
+ # architecture encoded by layer_types.
61
+ kwargs.pop("is_hybrid_swa", None)
62
+ kwargs.pop("hybrid_layer_pattern", None)
63
+ # Embedding scale convention (see modeling_ricursive.RicursiveModel.forward):
64
+ # - non-folded checkpoints set embed_scale (e.g. sqrt(hidden_size)) and the model
65
+ # scales the embeddings at runtime;
66
+ # - folded checkpoints bake the scale into embed_tokens.weight and leave this None.
67
+ # Either way the SGLang native port (sglang_ricursive) reads this same key, so the
68
+ # served and trained forwards agree.
69
+ self.embed_scale = embed_scale
70
+
71
+ # Per-layer type: "sliding_attention" (window + RoPE) or "full_attention" (no RoPE)
72
+ # These names follow the HuggingFace PretrainedConfig layer_types validation.
73
+ if layer_types is not None:
74
+ self.layer_types = layer_types
75
+ else:
76
+ # Default: every 5th layer is full_attention (indices 4, 9, 14, ...)
77
+ self.layer_types = [
78
+ "full_attention" if i % 5 == 4 else "sliding_attention"
79
+ for i in range(num_hidden_layers)
80
+ ]
81
+ if len(self.layer_types) != num_hidden_layers:
82
+ raise ValueError(
83
+ f"layer_types has {len(self.layer_types)} entries, expected {num_hidden_layers}"
84
+ )
85
+ invalid_layer_types = set(self.layer_types) - {
86
+ "sliding_attention",
87
+ "full_attention",
88
+ }
89
+ if invalid_layer_types:
90
+ raise ValueError(f"unsupported Ricursive layer_types: {sorted(invalid_layer_types)}")
91
+
92
+ # SGLang's compact hybrid KV allocator uses 1 for sliding attention and 0 for full
93
+ # attention. Keep these fields fully derived so they cannot drift from model semantics.
94
+ self.hybrid_layer_pattern = [
95
+ 1 if layer_type == "sliding_attention" else 0 for layer_type in self.layer_types
96
+ ]
97
+ self.is_hybrid_swa = 0 in self.hybrid_layer_pattern and 1 in self.hybrid_layer_pattern
98
+
99
+ # Standard HF config fields expected by modeling code
100
+ self.use_cache = kwargs.pop("use_cache", True)
101
+
102
+ self.base_model_tp_plan = {
103
+ "layers.*.self_attn.q_proj": "colwise",
104
+ "layers.*.self_attn.k_proj": "colwise",
105
+ "layers.*.self_attn.v_proj": "colwise",
106
+ "layers.*.self_attn.o_proj": "rowwise",
107
+ "layers.*.self_attn.g_proj": "colwise",
108
+ "layers.*.mlp.gate_proj": "colwise",
109
+ "layers.*.mlp.up_proj": "colwise",
110
+ "layers.*.mlp.down_proj": "rowwise",
111
+ }
112
+ self.auto_map = {
113
+ "AutoConfig": "configuration_ricursive.RicursiveConfig",
114
+ "AutoModel": "modeling_ricursive.RicursiveModel",
115
+ "AutoModelForCausalLM": "modeling_ricursive.RicursiveForCausalLM",
116
+ }
117
+
118
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
119
+
120
+ # Must be set AFTER super().__init__ — HF's PretrainedConfig.__init__
121
+ # pops "architectures" from kwargs (default None) and assigns it to
122
+ # self.architectures, which would otherwise overwrite our value.
123
+ self.architectures = ["RicursiveForCausalLM"]
inference/server/README.md ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Ripple on B200
2
+
3
+ BF16 Ripple-65B + native DFlash2 on two NVIDIA B200 GPUs, using **SGLang 0.5.19**.
4
+ Four files here: this README, `model.py`, `ripple_format.py`, and `serve.py`. Both
5
+ inference folders share `chat_template.jinja` at the repository root; keep that layout.
6
+ Uses the tools-aware ChatML template from our evals.
7
+
8
+ Requires Linux, CUDA Toolkit 13, a C++ compiler, and [uv](https://docs.astral.sh/uv/).
9
+ Use complete, unquantized BF16 target and matching `DFlash2DraftModel` checkpoints.
10
+ Run the commands below from `inference/server`.
11
+
12
+ ```bash
13
+ export CUDA_HOME=/usr/local/cuda-13.0
14
+ export PATH="$CUDA_HOME/bin:$PATH"
15
+
16
+ CUDA_VISIBLE_DEVICES=0,1 uv run --script serve.py \
17
+ --model-path /path/to/target \
18
+ --speculative-draft-model-path /path/to/draft
19
+ ```
20
+
21
+ Dependencies install automatically. The server listens on `127.0.0.1:30000`.
22
+ It uses the checkpoint's tokenizer and the repository-root chat template.
23
+
24
+ ```bash
25
+ curl http://127.0.0.1:30000/v1/chat/completions \
26
+ -H 'Content-Type: application/json' \
27
+ -d '{"model":"ripple","messages":[{"role":"user","content":"Hello"}],"max_tokens":128}'
28
+ ```
29
+
30
+ Defaults: TP=2, BF16 weights and KV cache, TRT-LLM target attention, Triton draft
31
+ attention, 16 draft tokens, and a **1Mi-token target context** (`--context-length 1048576`,
32
+ including prompt and generated tokens). Standard SGLang CLI flags apply.
33
+ The DFlash2 draft's last layer is global attention without RoPE. It was trained on
34
+ 16Ki-token rows, so the draft keeps a **16Ki-token window** (`--speculative-draft-window-size 16384`);
35
+ smaller windows are refused.
36
+ The launcher defaults `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1` so stock SGLang can
37
+ extend the draft's positional cache beyond its 128Ki checkpoint limit. This does not
38
+ increase the draft attention window. The target checkpoint supports up to 2Mi positions.
39
+ For four GPUs, run a second copy with `CUDA_VISIBLE_DEVICES=2,3` and `--port 30001`.
40
+ This gives two independent replicas, each using two GPUs. Send eight concurrent
41
+ requests to each port for sixteen total.
42
+
43
+ The launcher enables Ripple's parsers from `ripple_format.py` (`--reasoning-parser ripple`
44
+ and `--tool-call-parser ripple`). Responses separate `message.reasoning_content`,
45
+ final-answer `message.content`, and structured `message.tool_calls`. Streaming uses the
46
+ corresponding `delta` fields. Allow enough output tokens for both reasoning and the answer.
47
+
48
+ Pass OpenAI-style `tools` and `tool_choice: "auto"` to request calls:
49
+
50
+ ```json
51
+ {
52
+ "model": "ripple",
53
+ "messages": [{"role": "user", "content": "Use add_numbers to calculate 23 + 19."}],
54
+ "tools": [{"type": "function", "function": {
55
+ "name": "add_numbers", "description": "Add two integers.", "strict": true,
56
+ "parameters": {"type": "object", "properties": {
57
+ "a": {"type": "integer"}, "b": {"type": "integer"}
58
+ }, "required": ["a", "b"], "additionalProperties": false}
59
+ }}],
60
+ "tool_choice": "auto",
61
+ "temperature": 0,
62
+ "max_tokens": 1024
63
+ }
64
+ ```
65
+
66
+ Your application executes the requested function. Append the returned assistant
67
+ message unchanged, then a `tool` message containing its `tool_call_id` and a string
68
+ `content` with the result, and send the history again. The template re-renders an
69
+ unchanged message to exactly the tokens the model generated. Previous
70
+ `reasoning_content` is retained by default (`chat_template_kwargs.clear_thinking=false`).
71
+ The template does not implement a thinking on/off switch or reasoning-effort levels.
72
+
73
+ For `tool_choice: "required"`, use `function.strict=true` and
74
+ `parameters.additionalProperties=false`, as above. Non-strict required calls
75
+ produced extra argument fields in validation with this SGLang version.
inference/server/model.py ADDED
@@ -0,0 +1,512 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ricursive transformer implemented with SGLang layers.
2
+ BF16 weights with native DFlash2 feature capture.
3
+ """
4
+
5
+ from typing import Iterable, Optional, Tuple
6
+
7
+ import torch
8
+ from sglang.srt.distributed import get_pp_group, get_tensor_model_parallel_world_size
9
+ from sglang.srt.layers.activation import SiluAndMul
10
+ from sglang.srt.layers.layernorm import RMSNorm
11
+ from sglang.srt.layers.linear import (
12
+ ColumnParallelLinear,
13
+ MergedColumnParallelLinear,
14
+ RowParallelLinear,
15
+ )
16
+ from sglang.srt.layers.logits_processor import LogitsProcessor
17
+ from sglang.srt.layers.radix_attention import RadixAttention
18
+ from sglang.srt.layers.rotary_embedding import get_rope
19
+ from sglang.srt.layers.vocab_parallel_embedding import (
20
+ ParallelLMHead,
21
+ VocabParallelEmbedding,
22
+ )
23
+ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
24
+ from sglang.srt.model_loader.weight_utils import default_weight_loader
25
+ from sglang.srt.utils import add_prefix, make_layers
26
+ from torch import nn
27
+ from transformers import PretrainedConfig
28
+
29
+
30
+ def rotary_cache_length(config) -> int:
31
+ """Include DFlash lookahead beyond the admitted context boundary."""
32
+ from sglang.srt.server_args import get_global_server_args
33
+
34
+ args = get_global_server_args()
35
+ native = int(args.context_length or config.max_position_embeddings)
36
+ return native + (
37
+ int(args.speculative_num_draft_tokens) if args.speculative_algorithm == "DFLASH" else 0
38
+ )
39
+
40
+
41
+ class RicursiveMLP(nn.Module):
42
+ def __init__(
43
+ self,
44
+ hidden_size: int,
45
+ intermediate_size: int,
46
+ activation_clamp: Optional[float] = None,
47
+ prefix: str = "",
48
+ ) -> None:
49
+ super().__init__()
50
+ self.activation_clamp = activation_clamp
51
+ self.gate_up_proj = MergedColumnParallelLinear(
52
+ hidden_size,
53
+ [intermediate_size] * 2,
54
+ bias=False,
55
+ prefix=add_prefix("gate_up_proj", prefix),
56
+ )
57
+ self.down_proj = RowParallelLinear(
58
+ intermediate_size,
59
+ hidden_size,
60
+ bias=False,
61
+ prefix=add_prefix("down_proj", prefix),
62
+ )
63
+ self.act_fn = SiluAndMul()
64
+
65
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
66
+ gate_up, _ = self.gate_up_proj(x)
67
+ if self.activation_clamp is not None:
68
+ gate, up = gate_up.chunk(2, dim=-1)
69
+ gate = gate.clamp(max=self.activation_clamp)
70
+ up = up.clamp(min=-self.activation_clamp, max=self.activation_clamp)
71
+ gate_up = torch.cat((gate, up), dim=-1)
72
+ x = self.act_fn(gate_up)
73
+ x, _ = self.down_proj(x)
74
+ return x
75
+
76
+
77
+ class RicursiveAttention(nn.Module):
78
+ def __init__(
79
+ self,
80
+ config: PretrainedConfig,
81
+ layer_id: int = 0,
82
+ prefix: str = "",
83
+ ) -> None:
84
+ super().__init__()
85
+ self.config = config
86
+ hidden_size = config.hidden_size
87
+ tp_size = get_tensor_model_parallel_world_size()
88
+ self.total_num_heads = config.num_attention_heads
89
+ self.num_heads = self.total_num_heads // tp_size
90
+ self.total_num_kv_heads = config.num_key_value_heads
91
+ if self.total_num_heads % tp_size or self.total_num_kv_heads % tp_size:
92
+ raise ValueError(
93
+ "Ricursive separate Q/K/V projections require heads divisible by attention TP"
94
+ )
95
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
96
+ self.head_dim = config.head_dim
97
+ self.q_size = self.num_heads * self.head_dim
98
+ self.kv_size = self.num_kv_heads * self.head_dim
99
+ self.scaling = self.head_dim**-0.5
100
+
101
+ self.q_proj = ColumnParallelLinear(
102
+ hidden_size,
103
+ self.total_num_heads * self.head_dim,
104
+ bias=False,
105
+ prefix=add_prefix("q_proj", prefix),
106
+ )
107
+ self.k_proj = ColumnParallelLinear(
108
+ hidden_size,
109
+ self.total_num_kv_heads * self.head_dim,
110
+ bias=False,
111
+ prefix=add_prefix("k_proj", prefix),
112
+ )
113
+ self.v_proj = ColumnParallelLinear(
114
+ hidden_size,
115
+ self.total_num_kv_heads * self.head_dim,
116
+ bias=False,
117
+ prefix=add_prefix("v_proj", prefix),
118
+ )
119
+ self.o_proj = RowParallelLinear(
120
+ self.total_num_heads * self.head_dim,
121
+ hidden_size,
122
+ bias=False,
123
+ prefix=add_prefix("o_proj", prefix),
124
+ )
125
+
126
+ if getattr(config, "use_gated_attention", True):
127
+ self.g_proj = ColumnParallelLinear(
128
+ hidden_size,
129
+ self.total_num_heads * self.head_dim,
130
+ bias=False,
131
+ prefix=add_prefix("g_proj", prefix),
132
+ )
133
+ else:
134
+ self.g_proj = None
135
+
136
+ self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
137
+ self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
138
+
139
+ self.use_valres = bool(
140
+ getattr(config, "use_valres", getattr(config, "use_value_residuals", False))
141
+ )
142
+ if self.use_valres:
143
+ self.value_res_lambda = nn.Parameter(torch.zeros(1))
144
+
145
+ layer_type = config.layer_types[layer_id]
146
+ self.is_sliding = layer_type == "sliding_attention"
147
+ sliding_window = (config.sliding_window - 1) if self.is_sliding else -1
148
+
149
+ self.rotary_emb = None
150
+ if self.is_sliding:
151
+ self.rotary_emb = get_rope(
152
+ self.head_dim,
153
+ rotary_dim=self.head_dim,
154
+ max_position=rotary_cache_length(config),
155
+ base=config.rope_theta,
156
+ is_neox_style=True,
157
+ )
158
+
159
+ self.attn = RadixAttention(
160
+ self.num_heads,
161
+ self.head_dim,
162
+ self.scaling,
163
+ num_kv_heads=self.num_kv_heads,
164
+ layer_id=layer_id,
165
+ sliding_window_size=sliding_window,
166
+ prefix=add_prefix("attn", prefix),
167
+ )
168
+
169
+ def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
170
+ q = self.q_norm.forward_native(q.reshape(-1, self.head_dim)).reshape(-1, self.q_size)
171
+ k = self.k_norm.forward_native(k.reshape(-1, self.head_dim)).reshape(-1, self.kv_size)
172
+ return q, k
173
+
174
+ def forward(
175
+ self,
176
+ positions: torch.Tensor,
177
+ hidden_states: torch.Tensor,
178
+ forward_batch: ForwardBatch,
179
+ value_residual: Optional[torch.Tensor] = None,
180
+ ) -> torch.Tensor:
181
+ value_input = hidden_states
182
+ if value_residual is not None and self.use_valres:
183
+ blend = torch.sigmoid(self.value_res_lambda[0]).to(dtype=hidden_states.dtype)
184
+ value_input = torch.lerp(
185
+ value_residual.to(dtype=hidden_states.dtype), hidden_states, blend
186
+ )
187
+
188
+ q, _ = self.q_proj(hidden_states)
189
+ k, _ = self.k_proj(hidden_states)
190
+ v, _ = self.v_proj(value_input)
191
+
192
+ q, k = self._apply_qk_norm(q, k)
193
+
194
+ if self.rotary_emb is not None:
195
+ q, k = self.rotary_emb(positions, q, k)
196
+
197
+ attn_output = self.attn(q, k, v, forward_batch)
198
+
199
+ if self.g_proj is not None:
200
+ gate, _ = self.g_proj(hidden_states)
201
+ gate = torch.sigmoid(gate.float()).to(attn_output.dtype)
202
+ attn_output = attn_output * gate
203
+
204
+ output, _ = self.o_proj(attn_output)
205
+ return output
206
+
207
+
208
+ class RicursiveDecoderLayer(nn.Module):
209
+ """PeriNorm decoder layer: 4 RMSNorms, post-norm inside the residual on both sublayers."""
210
+
211
+ def __init__(
212
+ self,
213
+ config: PretrainedConfig,
214
+ layer_id: int = 0,
215
+ prefix: str = "",
216
+ ) -> None:
217
+ super().__init__()
218
+ self.self_attn = RicursiveAttention(
219
+ config, layer_id, prefix=add_prefix("self_attn", prefix)
220
+ )
221
+ self.mlp = RicursiveMLP(
222
+ hidden_size=config.hidden_size,
223
+ intermediate_size=config.intermediate_size,
224
+ activation_clamp=getattr(
225
+ config, "ffn_activation_clamp", getattr(config, "ffn_clamp_threshold", None)
226
+ ),
227
+ prefix=add_prefix("mlp", prefix),
228
+ )
229
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
230
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
231
+ self.pre_feedforward_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
232
+ self.post_feedforward_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
233
+
234
+ def forward(
235
+ self,
236
+ positions: torch.Tensor,
237
+ hidden_states: torch.Tensor,
238
+ forward_batch: ForwardBatch,
239
+ value_residual: Optional[torch.Tensor] = None,
240
+ ) -> torch.Tensor:
241
+ residual = hidden_states
242
+ hidden_states = self.input_layernorm(hidden_states)
243
+ hidden_states = self.self_attn(
244
+ positions, hidden_states, forward_batch, value_residual=value_residual
245
+ )
246
+ hidden_states = residual + self.post_attention_layernorm(hidden_states)
247
+
248
+ residual = hidden_states
249
+ hidden_states = self.pre_feedforward_layernorm(hidden_states)
250
+ hidden_states = self.mlp(hidden_states)
251
+ hidden_states = residual + self.post_feedforward_layernorm(hidden_states)
252
+ return hidden_states
253
+
254
+
255
+ class RicursiveModel(nn.Module):
256
+ def __init__(
257
+ self,
258
+ config: PretrainedConfig,
259
+ prefix: str = "",
260
+ ) -> None:
261
+ super().__init__()
262
+ self.config = config
263
+ self.embed_tokens = VocabParallelEmbedding(
264
+ config.vocab_size,
265
+ config.hidden_size,
266
+ prefix=add_prefix("embed_tokens", prefix),
267
+ )
268
+ self.layers = make_layers(
269
+ config.num_hidden_layers,
270
+ lambda idx, prefix: RicursiveDecoderLayer(config=config, layer_id=idx, prefix=prefix),
271
+ prefix=add_prefix("layers", prefix),
272
+ )
273
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
274
+ self.embed_scale = getattr(config, "embed_scale", None)
275
+ self.use_valres = bool(getattr(config, "use_valres", False))
276
+ self.dflash_capture_layers: tuple[int, ...] = ()
277
+
278
+ def forward(
279
+ self,
280
+ input_ids: torch.Tensor,
281
+ positions: torch.Tensor,
282
+ forward_batch: ForwardBatch,
283
+ input_embeds: Optional[torch.Tensor] = None,
284
+ ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:
285
+ if input_embeds is None:
286
+ hidden_states = self.embed_tokens(input_ids)
287
+ else:
288
+ hidden_states = input_embeds
289
+ if self.embed_scale is not None:
290
+ hidden_states = hidden_states * self.embed_scale
291
+ value_residual = hidden_states if self.use_valres else None
292
+
293
+ captured = {}
294
+ for index, layer in enumerate(self.layers):
295
+ hidden_states = layer(
296
+ positions, hidden_states, forward_batch, value_residual=value_residual
297
+ )
298
+ if index in self.dflash_capture_layers:
299
+ captured[index] = hidden_states
300
+
301
+ hidden_states = self.norm(hidden_states)
302
+ if self.dflash_capture_layers:
303
+ aux_hidden_states = [captured[index] for index in self.dflash_capture_layers]
304
+ return hidden_states, aux_hidden_states
305
+ return hidden_states
306
+
307
+
308
+ class RicursiveForCausalLM(nn.Module):
309
+ def __init__(
310
+ self,
311
+ config: PretrainedConfig,
312
+ quant_config=None,
313
+ prefix: str = "",
314
+ ) -> None:
315
+ super().__init__()
316
+ if quant_config is not None or getattr(config, "quantization_config", None):
317
+ raise ValueError("Only unquantized BF16 checkpoints are supported")
318
+ if torch.get_default_dtype() != torch.bfloat16:
319
+ raise ValueError("Only BF16 is supported; use --dtype bfloat16")
320
+ self.config = config
321
+ self.pp_group = get_pp_group()
322
+ self.model = RicursiveModel(config, prefix=add_prefix("model", prefix))
323
+ if config.tie_word_embeddings:
324
+ self.lm_head = self.model.embed_tokens
325
+ else:
326
+ self.lm_head = ParallelLMHead(
327
+ config.vocab_size,
328
+ config.hidden_size,
329
+ org_num_embeddings=config.vocab_size,
330
+ prefix=add_prefix("lm_head", prefix),
331
+ )
332
+ self.logits_processor = LogitsProcessor(config)
333
+
334
+ def get_input_embeddings(self) -> VocabParallelEmbedding:
335
+ """Expose the target embedding table shared by the DFlash worker."""
336
+ return self.model.embed_tokens
337
+
338
+ def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None:
339
+ """Capture post-block, pre-final-norm states in the drafter's configured order.
340
+
341
+ :param layer_ids: Zero-based target block indices from the draft configuration.
342
+ :raises ValueError: If indices are empty, duplicated, or outside the target model.
343
+ """
344
+ if (
345
+ not layer_ids
346
+ or any(type(index) is not int for index in layer_ids)
347
+ or len(set(layer_ids)) != len(layer_ids)
348
+ or any(index < 0 or index >= len(self.model.layers) for index in layer_ids)
349
+ ):
350
+ raise ValueError("DFlash requires unique, valid target layer indices")
351
+ self.model.dflash_capture_layers = tuple(layer_ids)
352
+
353
+ def get_attention_sliding_window_size(self) -> Optional[int]:
354
+ """Return the window used by Ricursive local-attention layers.
355
+
356
+ SGLang's native ``--disable-hybrid-swa-memory`` flag controls compact KV allocation.
357
+ Per-layer attention semantics are architectural and must not depend on an environment
358
+ variable.
359
+ """
360
+ layer_types = getattr(self.config, "layer_types", ()) or ()
361
+ if "sliding_attention" not in layer_types:
362
+ return None
363
+ sliding_window = getattr(self.config, "sliding_window", None)
364
+ if sliding_window is None:
365
+ return None
366
+ return sliding_window - 1
367
+
368
+ @torch.no_grad()
369
+ def forward(
370
+ self,
371
+ input_ids: torch.Tensor,
372
+ positions: torch.Tensor,
373
+ forward_batch: ForwardBatch,
374
+ input_embeds: Optional[torch.Tensor] = None,
375
+ ) -> torch.Tensor:
376
+ hidden_states = self.model(
377
+ input_ids=input_ids,
378
+ positions=positions,
379
+ forward_batch=forward_batch,
380
+ input_embeds=input_embeds,
381
+ )
382
+ if self.model.dflash_capture_layers:
383
+ hidden_states, aux_hidden_states = hidden_states
384
+ return self.logits_processor(
385
+ input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
386
+ )
387
+ return self.logits_processor(input_ids, hidden_states, self.lm_head, forward_batch)
388
+
389
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
390
+ """Map the checkpoint's separate gate/up tensors into SGLang's fused MLP."""
391
+ params = dict(self.named_parameters(remove_duplicate=False))
392
+ loaded, unused = set(), []
393
+ for name, weight in weights:
394
+ if "rotary_emb.inv_freq" in name:
395
+ continue
396
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
397
+ continue
398
+ for source, shard in (("gate_proj", 0), ("up_proj", 1)):
399
+ if source in name:
400
+ mapped = name.replace(source, "gate_up_proj")
401
+ if mapped in params:
402
+ params[mapped].weight_loader(params[mapped], weight, shard)
403
+ loaded.add(mapped)
404
+ else:
405
+ unused.append(name)
406
+ break
407
+ else:
408
+ if name in params:
409
+ param = params[name]
410
+ getattr(param, "weight_loader", default_weight_loader)(param, weight)
411
+ loaded.add(name)
412
+ else:
413
+ unused.append(name)
414
+ tied = {"lm_head.weight"} if self.config.tie_word_embeddings else set()
415
+ _raise_on_load_gaps("Ricursive target", unused, set(params) - loaded - tied)
416
+
417
+
418
+ class _NoRope(nn.Module):
419
+ """Identity RoPE for DFlash2 draft layers listed in config.no_rope_layers."""
420
+
421
+ # SGLang's fused draft context-KV path applies one RoPE table to every layer; it
422
+ # only accepts neox-style RoPE, so this marker turns it off for drafts with NoPE layers.
423
+ is_neox_style = False
424
+
425
+ def forward(self, positions, q, k):
426
+ return q, k
427
+
428
+
429
+ # DFlash2 drafts are trained on 16K-token rows (12,288 tokens of context + 4,096 predicted),
430
+ # so a global draft layer never attends further back than this.
431
+ DRAFT_TRAINING_CONTEXT = 16384
432
+
433
+
434
+ def _patch_draft_attention():
435
+ """Fit SGLang 0.5.19's DFlash draft attention to DFlash2 drafts with global layers.
436
+
437
+ It applies RoPE in every layer, ignoring no_rope_layers, and its draft window
438
+ (--speculative-draft-window-size) truncates every layer, including full attention,
439
+ so the window must cover the draft's training context.
440
+ """
441
+ from sglang.srt.models.dflash import DFlashAttention
442
+ from sglang.srt.server_args import get_global_server_args
443
+
444
+ init = DFlashAttention.__init__
445
+
446
+ def __init__(self, config, layer_id, *args, **kwargs):
447
+ init(self, config, layer_id, *args, **kwargs)
448
+ if layer_id in (getattr(config, "no_rope_layers", None) or ()):
449
+ self.rotary_emb = _NoRope()
450
+ self.use_table_qk_norm_rope = False
451
+ layer_types = getattr(config, "layer_types", None) or ()
452
+ window = get_global_server_args().speculative_draft_window_size
453
+ if (
454
+ layer_id < len(layer_types)
455
+ and layer_types[layer_id] == "full_attention"
456
+ and window is not None
457
+ and window < DRAFT_TRAINING_CONTEXT
458
+ ):
459
+ raise ValueError(
460
+ f"Draft layer {layer_id} attends globally over its {DRAFT_TRAINING_CONTEXT}-token "
461
+ f"training rows; use --speculative-draft-window-size >= {DRAFT_TRAINING_CONTEXT}"
462
+ )
463
+
464
+ DFlashAttention.__init__ = __init__
465
+
466
+
467
+ _patch_draft_attention()
468
+
469
+
470
+ def _raise_on_load_gaps(model: str, unused, unloaded) -> None:
471
+ """Refuse checkpoint tensors nothing consumed and parameters nothing loaded."""
472
+ if unused or unloaded:
473
+ raise ValueError(f"{model}: unused tensors {sorted(unused)[:20]}, "
474
+ f"unloaded parameters {sorted(unloaded)[:20]}")
475
+
476
+
477
+ def _strict_draft_loading():
478
+ """SGLang 0.5.19's DFlash loader skips tensors it cannot place: refuse such drafts."""
479
+ from sglang.srt.models.dflash import DFlashDraftModel
480
+
481
+ load, shared = DFlashDraftModel.load_weights, ("inv_freq", "embed_tokens.weight", "lm_head.weight")
482
+
483
+ def load_weights(self, weights):
484
+ if (getattr(self.config, "dflash_config", None) or {}).get("block_norm", "pre") != "pre":
485
+ raise ValueError("Only pre-norm DFlash drafts are supported")
486
+ params, calls, unused = dict(self.named_parameters()), [], []
487
+ own = {n: p.weight_loader for n, p in params.items() if "weight_loader" in p.__dict__}
488
+ for n, p in params.items(): # record which parameters receive a tensor
489
+ f = own.get(n, default_weight_loader)
490
+ p.weight_loader = lambda q, *a, _f=f, _n=n: (calls.append(_n), _f(q, *a))[1]
491
+
492
+ def watch(stream): # a tensor that reached no weight_loader was skipped
493
+ for name, weight in stream:
494
+ before = len(calls)
495
+ yield name, weight
496
+ if len(calls) == before and not name.endswith(shared):
497
+ unused.append(name)
498
+
499
+ try:
500
+ load(self, watch(weights))
501
+ finally:
502
+ for n, p in params.items():
503
+ del p.weight_loader
504
+ if n in own:
505
+ p.weight_loader = own[n]
506
+ unloaded = {n for n in params if not n.endswith(("k_scale", "v_scale"))} - set(calls)
507
+ _raise_on_load_gaps("DFlash draft", unused, unloaded)
508
+
509
+ DFlashDraftModel.load_weights = load_weights
510
+
511
+
512
+ _strict_draft_loading()
inference/server/ripple_format.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ripple's chat format for SGLang: reasoning parser, tool-call parser and tokenizer check.
2
+
3
+ The model generates the layout chat_template.jinja renders:
4
+
5
+ <think>\\n{reasoning}\\n</think>\\n{content}
6
+ <think>\\n{reasoning}\\n</think>\\n\\n<tool_call>\\n{"name": "f", "arguments": {...}}\\n</tool_call>
7
+
8
+ `--reasoning-parser ripple` returns reasoning and content without the separator newlines the
9
+ template adds back, so an assistant message sent back unchanged re-renders to the tokens the
10
+ model generated (SGLang's deepseek-v3 parser keeps them, doubling each newline on every turn).
11
+ Output counts as reasoning only when it opens with <think>.
12
+
13
+ `--tool-call-parser ripple` is hermes with the template's call layout, so strict tools and
14
+ tool_choice=required constrain decoding to `<tool_call>\\n{"name": "f", "arguments": ...}`
15
+ (hermes forces `<tool_call>{"name":"f"`, which the model was not trained on). Text before the
16
+ first call keeps its whitespace, minus the newline the template puts before <tool_call>.
17
+
18
+ Output the template cannot render (`</think>\\n<tool_call>` with one newline, text after a call)
19
+ re-renders in the template's canonical layout.
20
+
21
+ `register()` adds both parsers and checks every tokenizer SGLang loads against the checkpoint's
22
+ tokenizer.json (Transformers 5 can rebuild the pre-tokenizer from `tokenizer_class`).
23
+ """
24
+
25
+ import json
26
+ import logging
27
+
28
+ THINK_START, THINK_END = "<think>", "</think>"
29
+
30
+ # Exercise pre-tokenizer differences: digit runs, contractions, whitespace runs, non-ASCII.
31
+ TOKENIZER_PROBES = (
32
+ "x = 1234567890 + 3.14159; y=-42",
33
+ "I'M here, you'RE there, it's 4:05pm. DON'T",
34
+ "a b c\n\n\n d\t\te \r\n f",
35
+ " def f(self):\n return {'k': [1, 2]}\n",
36
+ "café naïve 日本語テキスト 한국어 🙂👍 Ελληνικά",
37
+ "<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n<think>\nok\n</think>\nHello<|im_end|>",
38
+ )
39
+
40
+
41
+ def _drop_newline(text, lead):
42
+ if lead:
43
+ return text[1:] if text.startswith("\n") else text
44
+ return text[:-1] if text.endswith("\n") else text
45
+
46
+
47
+ def _partial_suffix(text, token):
48
+ """Length of the longest suffix of text that is a strict prefix of token."""
49
+ for i in range(min(len(text), len(token) - 1), 0, -1):
50
+ if token.startswith(text[-i:]):
51
+ return i
52
+ return 0
53
+
54
+
55
+ def _reasoning_detector_class():
56
+ from sglang.srt.parser.reasoning_parser import (
57
+ BaseReasoningFormatDetector,
58
+ StreamingParseResult,
59
+ )
60
+
61
+ class RippleReasoningDetector(BaseReasoningFormatDetector):
62
+ def __init__(self, stream_reasoning=True, force_reasoning=False, **kwargs):
63
+ # force_reasoning is ignored: the generation prompt does not open <think>.
64
+ super().__init__(
65
+ THINK_START,
66
+ THINK_END,
67
+ force_reasoning=False,
68
+ stream_reasoning=stream_reasoning,
69
+ reasoning_default="explicit_thinking", # as deepseek-v3
70
+ **kwargs,
71
+ )
72
+ self._state = "start" # start -> lead -> think -> after -> content
73
+
74
+ def _detect_and_parse_impl(self, text):
75
+ if self.continue_final_message:
76
+ return super()._detect_and_parse_impl(text)
77
+ if not text.startswith(THINK_START):
78
+ return StreamingParseResult(normal_text=text)
79
+ body = _drop_newline(text[len(THINK_START):], lead=True)
80
+ reasoning, end, content = body.partition(THINK_END)
81
+ if not end: # truncated inside reasoning
82
+ return StreamingParseResult(reasoning_text=reasoning)
83
+ return StreamingParseResult(
84
+ normal_text=_drop_newline(content, lead=True),
85
+ reasoning_text=_drop_newline(reasoning, lead=False),
86
+ )
87
+
88
+ def _parse_streaming_increment_impl(self, new_text):
89
+ if self.continue_final_message:
90
+ return super()._parse_streaming_increment_impl(new_text)
91
+ self._buffer += new_text
92
+ reasoning, normal = "", ""
93
+ while self._buffer:
94
+ buf = self._buffer
95
+ if self._state == "start":
96
+ if buf.startswith(THINK_START):
97
+ self._buffer, self._state, self._in_reasoning = buf[len(THINK_START):], "lead", True
98
+ elif THINK_START.startswith(buf):
99
+ break
100
+ else:
101
+ self._state = "content"
102
+ elif self._state == "lead":
103
+ self._buffer, self._state = _drop_newline(buf, lead=True), "think"
104
+ elif self._state == "think":
105
+ i = buf.find(THINK_END)
106
+ if i >= 0:
107
+ reasoning += _drop_newline(buf[:i], lead=False)
108
+ self._buffer, self._state, self._in_reasoning = buf[i + len(THINK_END):], "after", False
109
+ continue
110
+ if not self.stream_reasoning:
111
+ break
112
+ # Hold back a possible "\n</think>" prefix: its newline is the separator.
113
+ hold = max(_partial_suffix(buf, "\n" + THINK_END), _partial_suffix(buf, THINK_END))
114
+ reasoning += buf[:len(buf) - hold]
115
+ self._buffer = buf[len(buf) - hold:]
116
+ break
117
+ elif self._state == "after":
118
+ self._buffer, self._state = _drop_newline(buf, lead=True), "content"
119
+ else:
120
+ normal += buf
121
+ self._buffer = ""
122
+ return StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
123
+
124
+ def finish(self):
125
+ if self.continue_final_message:
126
+ return super().finish()
127
+ buf, self._buffer = self._buffer, ""
128
+ if self._state in ("lead", "think"): # truncated inside reasoning
129
+ if self._force_nonempty_content:
130
+ text, self._accumulated_reasoning = self._accumulated_reasoning + buf, ""
131
+ return StreamingParseResult(normal_text=text)
132
+ return StreamingParseResult(reasoning_text=buf)
133
+ return StreamingParseResult(normal_text=buf)
134
+
135
+ return RippleReasoningDetector
136
+
137
+
138
+ def _tool_detector_class():
139
+ from sglang.srt.function_call.core_types import StructureInfo, ToolCallItem
140
+ from sglang.srt.function_call.hermes_detector import HermesDetector
141
+
142
+ class RippleToolDetector(HermesDetector):
143
+ """Text before the first call keeps its whitespace except the one newline the template
144
+ puts before <tool_call> (hermes strips it all)."""
145
+
146
+ def __init__(self):
147
+ super().__init__()
148
+ self._held_newline = ""
149
+ self._in_calls = False
150
+ # Hermes can leave whole calls buffered when one speculative delta spans
151
+ # multiple calls. Reconcile the full output with emitted deltas at finish.
152
+ self._seen = ""
153
+ self._emitted = {}
154
+
155
+ def _record(self, calls):
156
+ for call in calls:
157
+ emitted = self._emitted.setdefault(call.tool_index, ["", ""])
158
+ emitted[0] += call.name or ""
159
+ emitted[1] += call.parameters or ""
160
+ # SGLang's serving layer also flushes arguments after finish(). Keep
161
+ # its accounting current so it cannot send the same suffix twice.
162
+ while len(self.streamed_args_for_tool) <= call.tool_index:
163
+ self.streamed_args_for_tool.append("")
164
+ self.streamed_args_for_tool[call.tool_index] = emitted[1]
165
+
166
+ def detect_and_parse(self, text, tools):
167
+ res = super().detect_and_parse(text, tools)
168
+ idx = text.find(self.bot_token)
169
+ if idx != -1 and res.normal_text == text[:idx].strip(): # parsed (not the error path)
170
+ res.normal_text = _drop_newline(text[:idx], lead=False)
171
+ return res
172
+
173
+ def parse_streaming_increment(self, new_text, tools):
174
+ self._seen += new_text
175
+ res = super().parse_streaming_increment(new_text, tools)
176
+ self._record(res.calls)
177
+ if self._in_calls: # text after the first call is dropped, as in detect_and_parse
178
+ res.normal_text = ""
179
+ return res
180
+ call_started = bool(self._buffer.startswith(self.bot_token) or self.current_tool_id >= 0 or res.calls)
181
+ self._in_calls = call_started
182
+ if res.normal_text:
183
+ text, self._held_newline = self._held_newline + res.normal_text, ""
184
+ if call_started: # its trailing newline is the separator before <tool_call>
185
+ text = _drop_newline(text, lead=False)
186
+ elif text.endswith("\n"): # undecided until the next text arrives
187
+ text, self._held_newline = text[:-1], "\n"
188
+ res.normal_text = text
189
+ elif call_started:
190
+ self._held_newline = ""
191
+ return res
192
+
193
+ def finish(self, tools):
194
+ res = super().finish(tools)
195
+ self._record(res.calls)
196
+ if self.bot_token in self._seen:
197
+ for i, call in enumerate(self.detect_and_parse(self._seen, tools).calls):
198
+ name, args = self._emitted.get(i, ["", ""])
199
+ want = call.parameters or ""
200
+ if want.startswith(args):
201
+ suffix = want[len(args):]
202
+ if suffix or not name:
203
+ item = ToolCallItem(
204
+ tool_index=i, name=None if name else call.name, parameters=suffix,
205
+ )
206
+ res.calls.append(item)
207
+ self._record([item])
208
+ if self._in_calls:
209
+ res.normal_text = ""
210
+ elif self._held_newline:
211
+ res.normal_text, self._held_newline = self._held_newline + (res.normal_text or ""), ""
212
+ return res
213
+
214
+ def structure_info(self):
215
+ return lambda name: StructureInfo(
216
+ begin='<tool_call>\n{"name": ' + json.dumps(name, ensure_ascii=False) + ', "arguments": ',
217
+ end="}\n</tool_call>",
218
+ trigger="<tool_call>",
219
+ )
220
+
221
+ return RippleToolDetector
222
+
223
+
224
+ def check_tokenizer(tokenizer, tokenizer_json):
225
+ """Make `tokenizer` encode and decode like the raw tokenizer.json, or raise.
226
+
227
+ Restores the normalizer, pre-tokenizer and decoder from tokenizer.json when they differ."""
228
+ from tokenizers import Tokenizer
229
+
230
+ raw = Tokenizer.from_file(tokenizer_json)
231
+
232
+ def mismatches():
233
+ bad = []
234
+ for text in TOKENIZER_PROBES:
235
+ ids = raw.encode(text, add_special_tokens=False).ids
236
+ got = tokenizer.encode(text, add_special_tokens=False)
237
+ if got != ids or tokenizer.decode(ids) != raw.decode(ids, skip_special_tokens=False):
238
+ bad.append(text)
239
+ return bad
240
+
241
+ if not mismatches():
242
+ return False
243
+ backend = getattr(tokenizer, "_tokenizer", None)
244
+ if backend is not None:
245
+ backend.normalizer, backend.pre_tokenizer = raw.normalizer, raw.pre_tokenizer
246
+ backend.decoder = raw.decoder
247
+ bad = mismatches()
248
+ if bad:
249
+ raise RuntimeError(
250
+ f"{type(tokenizer).__name__} does not match {tokenizer_json} on {bad!r}; "
251
+ "export the tokenizer with tokenizer_class PreTrainedTokenizerFast"
252
+ )
253
+ return True
254
+
255
+
256
+ def _tokenizer_json(name, revision=None):
257
+ from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
258
+
259
+ try:
260
+ return hf_tok._resolve_local_or_cached_file(name, "tokenizer.json", revision)
261
+ except (OSError, ValueError): # no tokenizer.json (e.g. a slow-only tokenizer)
262
+ return None
263
+
264
+
265
+ def register():
266
+ """Add the `ripple` parsers to SGLang and check tokenizers on load. Idempotent; call at
267
+ import time so SGLang's spawned processes register too."""
268
+ from sglang.srt.function_call.function_call_parser import FunctionCallParser
269
+ from sglang.srt.parser.reasoning_parser import ReasoningParser
270
+ from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
271
+
272
+ ReasoningParser.DetectorMap.setdefault("ripple", _reasoning_detector_class())
273
+ FunctionCallParser.ToolCallParserEnum.setdefault("ripple", _tool_detector_class())
274
+
275
+ fixes = hf_tok._apply_post_load_fixes # fail loudly if SGLang renames it
276
+ if getattr(fixes, "_ripple", False):
277
+ return
278
+
279
+ def apply_post_load_fixes(tokenizer, name, revision=None):
280
+ tokenizer = fixes(tokenizer, name, revision)
281
+ path = _tokenizer_json(name, revision)
282
+ if path is not None and check_tokenizer(tokenizer, path):
283
+ logging.getLogger(__name__).warning(
284
+ "Restored %s's normalizer, pre-tokenizer and decoder from tokenizer.json", name)
285
+ return tokenizer
286
+
287
+ apply_post_load_fixes._ripple = True
288
+ hf_tok._apply_post_load_fixes = apply_post_load_fixes
inference/server/serve.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.12,<3.13"
3
+ # dependencies = [
4
+ # "sglang==0.5.19",
5
+ # "torch==2.13.0+cu130",
6
+ # ]
7
+ # [tool.uv]
8
+ # index-strategy = "unsafe-best-match"
9
+ # prerelease = "allow"
10
+ # [[tool.uv.index]]
11
+ # url = "https://download.pytorch.org/whl/cu130"
12
+ # ///
13
+ """Serve Ripple + DFlash2 on two B200s. Accepts standard SGLang CLI flags."""
14
+
15
+ import os
16
+ import shlex
17
+ import sys
18
+ from pathlib import Path
19
+
20
+ sys.dont_write_bytecode = True
21
+
22
+ # SGLang sizes the draft's positional cache for the target context, even with
23
+ # windowed draft attention. Allow its 128Ki position limit to grow to 1Mi;
24
+ # --speculative-draft-window-size still limits draft attention to 16Ki tokens.
25
+ os.environ.setdefault("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", "1")
26
+
27
+ # Top-level registration also runs in SGLang's spawned worker processes.
28
+ from sglang.srt.models.registry import ModelRegistry
29
+
30
+ from model import RicursiveForCausalLM
31
+ import ripple_format
32
+
33
+ ModelRegistry.models["RicursiveForCausalLM"] = RicursiveForCausalLM
34
+ ripple_format.register()
35
+
36
+ DEFAULT_ARGS = shlex.split("""
37
+ --host 127.0.0.1 --served-model-name ripple --trust-remote-code
38
+ --dtype bfloat16 --kv-cache-dtype bfloat16 --tp-size 2
39
+ --attention-backend trtllm_mha --page-size 64
40
+ --speculative-algorithm DFLASH --speculative-draft-model-quantization unquant
41
+ --speculative-num-draft-tokens 16 --speculative-draft-window-size 16384
42
+ --speculative-draft-attention-backend triton
43
+ --context-length 1048576 --mem-fraction-static 0.80 --max-running-requests 16
44
+ --cuda-graph-backend-prefill disabled
45
+ --tool-call-parser ripple --reasoning-parser ripple
46
+ """)
47
+ DEFAULT_ARGS += [
48
+ "--chat-template",
49
+ str(Path(__file__).resolve().parents[2] / "chat_template.jinja"),
50
+ ]
51
+
52
+
53
+ def main():
54
+ from sglang.srt.entrypoints.http_server import launch_server
55
+ from sglang.srt.server_args import prepare_server_args
56
+ from sglang.srt.utils import kill_process_tree
57
+
58
+ args = prepare_server_args(DEFAULT_ARGS + sys.argv[1:])
59
+ if args.dtype != "bfloat16" or args.kv_cache_dtype != "bfloat16":
60
+ raise ValueError("Only BF16 weights and KV cache are supported")
61
+ try:
62
+ launch_server(args)
63
+ finally:
64
+ kill_process_tree(os.getpid(), include_parent=False)
65
+
66
+
67
+ if __name__ == "__main__":
68
+ main()
model-00001-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0701bedc60edbb370e128cdff50478f6676cacd674080c16cca122a8d4b06c72
3
+ size 8420238166
model-00002-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f22cba23852fcfd5df22e54285952ec3803b7921461fc8e54629c586e1ab8a60
3
+ size 8399320620
model-00003-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed8cbb1946f546d32c96b569d30b700ca6936994025623d6c69c9e436b7caced
3
+ size 8388824258
model-00004-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54281881efb8ab45301aa3d22e9501281e4c59bc69bd736ce08e8de5e09fae05
3
+ size 8388855346
model-00005-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a78d9cac0b076eede2fda612704b43f9d21d6f27276ef1924656327c992a1c5c
3
+ size 8304949852
model-00006-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd281215be38ff363f13e6de47d20b062a4cfe288d7c2d98fc93c0a962d4e21a
3
+ size 8388824290
model-00007-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:384e1e3430c5972f33434269090c3dfe5b101649579950b9645fecc9739949be
3
+ size 8388824258
model-00008-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d011f18e13bc7cdea10f9066681797051d4ae28fd4edb96f84abca264e9934eb
3
+ size 8388855346
model-00009-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:921d6e15e08f717dc5be562dd1ceb56a096fd9d3707f1f1d742d15625917daa1
3
+ size 8304949852
model-00010-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31f3417c654d8db5596f811db381b75b732f2e11925d54a8dbb7ec3d25e3a086
3
+ size 8388824290
model-00011-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6aa7575de621c2f74761a0ed4cf1d303395fbaec65f64e7c3dcd695230fedc14
3
+ size 8388824274
model-00012-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:882b853d96ed396346ab65bb4070bf489f7b7e2d3cb60457d6d785fe2e218272
3
+ size 8388855338
model-00013-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75a1cd9a3688a8d2483232e45ebfc01fa7287accd52237c955ae11af3621e028
3
+ size 8304949868
model-00014-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:269b31e8abf1d73bb0dd158d30a4e0bacfaad1eaa7cbc442541cbce9443023bf
3
+ size 8388824274
model-00015-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32b4c1bb53297f5e564bb327b2cd40318c7b672be04d94899c6c0eee37102c9b
3
+ size 8388824274
model-00016-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0a2571453449e5fe940cba3adb623ffc1d9b3d00a32a5d5f0a979e82bc39eeb
3
+ size 3942775166
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
modeling_ricursive.py ADDED
@@ -0,0 +1,606 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HuggingFace modeling implementation for the Ricursive transformer architecture.
2
+
3
+ Designed for inference with vLLM (via --trust-remote-code). Key differences from
4
+ standard Llama/Mistral:
5
+ - PeriNorm blocks: 4 layer norms per block (pre/post attention + pre/post FFN)
6
+ - Gated attention: elementwise sigmoid gate on attention output
7
+ - Mixed local/global layers: local = sliding window + RoPE, global = full attention + no RoPE
8
+ - Per-head Q/K RMSNorm
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ import torch.nn as nn
15
+ import torch.nn.functional as F
16
+ from transformers import PreTrainedModel
17
+ from transformers.cache_utils import Cache, DynamicCache
18
+ from transformers.modeling_outputs import (
19
+ BaseModelOutputWithPast,
20
+ CausalLMOutputWithPast,
21
+ )
22
+
23
+ from .configuration_ricursive import RicursiveConfig
24
+
25
+
26
+ class RicursiveRMSNorm(nn.Module):
27
+ def __init__(self, hidden_size: int, eps: float = 1e-6):
28
+ super().__init__()
29
+ self.weight = nn.Parameter(torch.ones(hidden_size))
30
+ self.eps = eps
31
+
32
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
33
+ input_dtype = hidden_states.dtype
34
+ hidden_states = hidden_states.float()
35
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
36
+ hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
37
+ return self.weight * hidden_states.to(input_dtype)
38
+
39
+
40
+ class RicursiveRotaryEmbedding(nn.Module):
41
+ def __init__(self, dim: int, theta: float = 10000.0):
42
+ super().__init__()
43
+ self.dim = dim
44
+ self.theta = theta
45
+ # NOTE: inv_freq is computed on-the-fly in forward() rather than stored
46
+ # as a buffer, because non-persistent buffers get corrupted during
47
+ # from_pretrained() loading.
48
+
49
+ @torch.no_grad()
50
+ def forward(self, x: torch.Tensor, position_ids: torch.Tensor):
51
+ inv_freq = 1.0 / (
52
+ self.theta ** (torch.arange(0, self.dim, 2, device=x.device).float() / self.dim)
53
+ )
54
+ inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
55
+ position_ids_expanded = position_ids[:, None, :].float()
56
+
57
+ device_type = x.device.type
58
+ with torch.autocast(device_type=device_type, enabled=False):
59
+ freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
60
+ emb = torch.cat((freqs, freqs), dim=-1)
61
+ cos = emb.cos()
62
+ sin = emb.sin()
63
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
64
+
65
+
66
+ def rotate_half(x: torch.Tensor) -> torch.Tensor:
67
+ x1 = x[..., : x.shape[-1] // 2]
68
+ x2 = x[..., x.shape[-1] // 2 :]
69
+ return torch.cat((-x2, x1), dim=-1)
70
+
71
+
72
+ def apply_rotary_pos_emb(
73
+ q: torch.Tensor,
74
+ k: torch.Tensor,
75
+ cos: torch.Tensor,
76
+ sin: torch.Tensor,
77
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
78
+ cos = cos.unsqueeze(1) # (B, 1, T, dim)
79
+ sin = sin.unsqueeze(1) # (B, 1, T, dim)
80
+ # OLMo-core applies RoPE in full precision (float32) for numerical stability
81
+ orig_dtype = q.dtype
82
+ q_float, k_float = q.float(), k.float()
83
+ cos_float, sin_float = cos.float(), sin.float()
84
+ q_embed = (q_float * cos_float) + (rotate_half(q_float) * sin_float)
85
+ k_embed = (k_float * cos_float) + (rotate_half(k_float) * sin_float)
86
+ return q_embed.to(orig_dtype), k_embed.to(orig_dtype)
87
+
88
+
89
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
90
+ if n_rep == 1:
91
+ return hidden_states
92
+ batch, num_kv_heads, slen, head_dim = hidden_states.shape
93
+ hidden_states = hidden_states[:, :, None, :, :].expand(
94
+ batch, num_kv_heads, n_rep, slen, head_dim
95
+ )
96
+ return hidden_states.reshape(batch, num_kv_heads * n_rep, slen, head_dim)
97
+
98
+
99
+ class RicursiveAttention(nn.Module):
100
+ def __init__(self, config: RicursiveConfig, layer_idx: int):
101
+ super().__init__()
102
+ self.config = config
103
+ self.layer_idx = layer_idx
104
+ self.layer_type = config.layer_types[layer_idx]
105
+
106
+ self.hidden_size = config.hidden_size
107
+ self.num_heads = config.num_attention_heads
108
+ self.num_kv_heads = config.num_key_value_heads
109
+ self.head_dim = config.head_dim
110
+ self.num_kv_groups = self.num_heads // self.num_kv_heads
111
+
112
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
113
+ self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
114
+ self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
115
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
116
+ if config.use_valres:
117
+ self.value_res_lambda = nn.Parameter(torch.zeros(1))
118
+
119
+ # Per-head Q/K norms (RMSNorm on head_dim)
120
+ self.q_norm = RicursiveRMSNorm(self.head_dim, eps=config.rms_norm_eps)
121
+ self.k_norm = RicursiveRMSNorm(self.head_dim, eps=config.rms_norm_eps)
122
+
123
+ # Gated attention
124
+ if config.use_gated_attention:
125
+ self.g_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
126
+
127
+ # RoPE only for local (sliding_attention) layers
128
+ if self.layer_type == "sliding_attention":
129
+ self.rotary_emb = RicursiveRotaryEmbedding(self.head_dim, theta=config.rope_theta)
130
+
131
+ def _apply_sliding_window_mask(
132
+ self,
133
+ attention_mask: Optional[torch.Tensor],
134
+ q_len: int,
135
+ kv_len: int,
136
+ dtype: torch.dtype,
137
+ device: torch.device,
138
+ cache_position: Optional[torch.LongTensor],
139
+ ) -> Optional[torch.Tensor]:
140
+ if self.layer_type != "sliding_attention":
141
+ return attention_mask
142
+
143
+ key_positions = torch.arange(kv_len, device=device)
144
+ if cache_position is not None:
145
+ query_positions = cache_position.to(device=device)
146
+ else:
147
+ query_positions = torch.arange(kv_len - q_len, kv_len, device=device)
148
+
149
+ window = self.config.sliding_window
150
+ allowed = (
151
+ (key_positions[None, :] <= query_positions[:, None])
152
+ & (key_positions[None, :] >= query_positions[:, None] - (window - 1))
153
+ )[None, None, :, :]
154
+
155
+ if attention_mask is None:
156
+ return allowed
157
+ if attention_mask.dtype == torch.bool:
158
+ return attention_mask & allowed
159
+ return attention_mask.masked_fill(~allowed, torch.finfo(dtype).min)
160
+
161
+ def forward(
162
+ self,
163
+ hidden_states: torch.Tensor,
164
+ attention_mask: Optional[torch.Tensor] = None,
165
+ position_ids: Optional[torch.Tensor] = None,
166
+ past_key_value: Optional[Cache] = None,
167
+ output_attentions: bool = False,
168
+ use_cache: bool = False,
169
+ cache_position: Optional[torch.LongTensor] = None,
170
+ value_residual: Optional[torch.Tensor] = None,
171
+ **kwargs,
172
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
173
+ bsz, q_len, _ = hidden_states.shape
174
+
175
+ # Project Q, K, V
176
+ q = self.q_proj(hidden_states)
177
+ k = self.k_proj(hidden_states)
178
+ value_states = hidden_states
179
+ if value_residual is not None:
180
+ if not self.config.use_valres:
181
+ raise RuntimeError("value_residual was supplied but config.use_valres is false")
182
+ blend = torch.sigmoid(self.value_res_lambda[0]).to(dtype=hidden_states.dtype)
183
+ value_states = torch.lerp(
184
+ value_residual.to(dtype=hidden_states.dtype), hidden_states, blend
185
+ )
186
+ v = self.v_proj(value_states)
187
+
188
+ # Reshape to (B, T, n_heads, head_dim) then transpose to (B, n_heads, T, head_dim)
189
+ q = q.view(bsz, q_len, self.num_heads, self.head_dim)
190
+ k = k.view(bsz, q_len, self.num_kv_heads, self.head_dim)
191
+ v = v.view(bsz, q_len, self.num_kv_heads, self.head_dim)
192
+
193
+ # Per-head Q/K norm (applied before transpose, on head_dim)
194
+ q = self.q_norm(q)
195
+ k = self.k_norm(k)
196
+
197
+ # Transpose to (B, n_heads, T, head_dim) for attention
198
+ q = q.transpose(1, 2)
199
+ k = k.transpose(1, 2)
200
+ v = v.transpose(1, 2)
201
+
202
+ # RoPE only for local layers
203
+ if self.layer_type == "sliding_attention":
204
+ cos, sin = self.rotary_emb(v, position_ids)
205
+ q, k = apply_rotary_pos_emb(q, k, cos, sin)
206
+
207
+ # KV cache
208
+ if past_key_value is not None:
209
+ k, v = past_key_value.update(k, v, self.layer_idx)
210
+
211
+ # GQA: expand KV heads
212
+ k = repeat_kv(k, self.num_kv_groups)
213
+ v = repeat_kv(v, self.num_kv_groups)
214
+
215
+ attention_mask = self._apply_sliding_window_mask(
216
+ attention_mask=attention_mask,
217
+ q_len=q.shape[2],
218
+ kv_len=k.shape[2],
219
+ dtype=q.dtype,
220
+ device=q.device,
221
+ cache_position=cache_position,
222
+ )
223
+
224
+ # Attention (use is_causal when no explicit mask for correct causal behavior)
225
+ # Cast to plain bool to avoid SymBool issues with torch.compile
226
+ is_causal = bool(attention_mask is None and q.shape[2] > 1)
227
+ attn_output = F.scaled_dot_product_attention(
228
+ q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=is_causal
229
+ )
230
+
231
+ # (B, n_heads, T, head_dim) -> (B, T, n_heads * head_dim)
232
+ attn_output = attn_output.transpose(1, 2).contiguous()
233
+
234
+ # Gated attention: output * sigmoid(g_proj(input))
235
+ if self.config.use_gated_attention:
236
+ gate = self.g_proj(hidden_states)
237
+ if self.config.gate_full_precision:
238
+ gate = gate.float()
239
+ gate = torch.sigmoid(gate).to(attn_output.dtype)
240
+ attn_output = attn_output.view(bsz, q_len, -1) * gate
241
+ else:
242
+ attn_output = attn_output.view(bsz, q_len, -1)
243
+
244
+ # Output projection
245
+ attn_output = self.o_proj(attn_output)
246
+
247
+ return attn_output, None, past_key_value
248
+
249
+
250
+ class RicursiveMLP(nn.Module):
251
+ def __init__(self, config: RicursiveConfig):
252
+ super().__init__()
253
+ self.clamp_threshold = config.ffn_clamp_threshold
254
+ self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
255
+ self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
256
+ self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
257
+
258
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
259
+ gate = self.gate_proj(x)
260
+ up = self.up_proj(x)
261
+ if self.clamp_threshold:
262
+ gate = gate.clamp(min=None, max=self.clamp_threshold)
263
+ up = up.clamp(min=-self.clamp_threshold, max=self.clamp_threshold)
264
+ return self.down_proj(F.silu(gate) * up)
265
+
266
+
267
+ class RicursiveDecoderLayer(nn.Module):
268
+ """PeriNorm decoder layer with 4 layer norms."""
269
+
270
+ def __init__(self, config: RicursiveConfig, layer_idx: int):
271
+ super().__init__()
272
+ self.self_attn = RicursiveAttention(config, layer_idx)
273
+ self.mlp = RicursiveMLP(config)
274
+
275
+ # PeriNorm: 4 norms
276
+ self.input_layernorm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
277
+ self.post_attention_layernorm = RicursiveRMSNorm(
278
+ config.hidden_size, eps=config.rms_norm_eps
279
+ )
280
+ self.pre_feedforward_layernorm = RicursiveRMSNorm(
281
+ config.hidden_size, eps=config.rms_norm_eps
282
+ )
283
+ self.post_feedforward_layernorm = RicursiveRMSNorm(
284
+ config.hidden_size, eps=config.rms_norm_eps
285
+ )
286
+
287
+ def forward(
288
+ self,
289
+ hidden_states: torch.Tensor,
290
+ attention_mask: Optional[torch.Tensor] = None,
291
+ position_ids: Optional[torch.Tensor] = None,
292
+ past_key_value: Optional[Cache] = None,
293
+ output_attentions: bool = False,
294
+ use_cache: bool = False,
295
+ cache_position: Optional[torch.LongTensor] = None,
296
+ value_residual: Optional[torch.Tensor] = None,
297
+ **kwargs,
298
+ ) -> Tuple[torch.Tensor, ...]:
299
+ # Attention sub-block (PeriNorm: pre-norm -> attn -> post-norm -> residual)
300
+ normed = self.input_layernorm(hidden_states)
301
+ attn_output, attn_weights, present_key_value = self.self_attn(
302
+ normed,
303
+ attention_mask=attention_mask,
304
+ position_ids=position_ids,
305
+ past_key_value=past_key_value,
306
+ output_attentions=output_attentions,
307
+ use_cache=use_cache,
308
+ cache_position=cache_position,
309
+ value_residual=value_residual,
310
+ **kwargs,
311
+ )
312
+ hidden_states = hidden_states + self.post_attention_layernorm(attn_output)
313
+
314
+ # FFN sub-block (PeriNorm: pre-norm -> ffn -> post-norm -> residual)
315
+ normed = self.pre_feedforward_layernorm(hidden_states)
316
+ ff_output = self.mlp(normed)
317
+ hidden_states = hidden_states + self.post_feedforward_layernorm(ff_output)
318
+
319
+ outputs = (hidden_states,)
320
+ if output_attentions:
321
+ outputs += (attn_weights,)
322
+ if use_cache:
323
+ outputs += (present_key_value,)
324
+ return outputs
325
+
326
+
327
+ class RicursivePreTrainedModel(PreTrainedModel):
328
+ config_class = RicursiveConfig
329
+ base_model_prefix = "model"
330
+ supports_gradient_checkpointing = True
331
+ _no_split_modules = ["RicursiveDecoderLayer"]
332
+ _supports_sdpa = True
333
+ _supports_cache_class = True
334
+
335
+
336
+ class RicursiveModel(RicursivePreTrainedModel):
337
+ def __init__(self, config: RicursiveConfig):
338
+ super().__init__(config)
339
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
340
+ self.layers = nn.ModuleList(
341
+ [RicursiveDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
342
+ )
343
+ self.norm = RicursiveRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
344
+ self.post_init()
345
+
346
+ def forward(
347
+ self,
348
+ input_ids: Optional[torch.LongTensor] = None,
349
+ attention_mask: Optional[torch.Tensor] = None,
350
+ position_ids: Optional[torch.LongTensor] = None,
351
+ past_key_values: Optional[Cache] = None,
352
+ inputs_embeds: Optional[torch.FloatTensor] = None,
353
+ use_cache: Optional[bool] = None,
354
+ output_attentions: Optional[bool] = None,
355
+ output_hidden_states: Optional[bool] = None,
356
+ return_dict: Optional[bool] = None,
357
+ cache_position: Optional[torch.LongTensor] = None,
358
+ **kwargs,
359
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
360
+ output_attentions = (
361
+ output_attentions if output_attentions is not None else self.config.output_attentions
362
+ )
363
+ output_hidden_states = (
364
+ output_hidden_states
365
+ if output_hidden_states is not None
366
+ else self.config.output_hidden_states
367
+ )
368
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
369
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
370
+
371
+ if inputs_embeds is None:
372
+ inputs_embeds = self.embed_tokens(input_ids)
373
+
374
+ # embed_scale: non-folded checkpoints carry the scale (e.g. sqrt(hidden_size)) in the
375
+ # config and apply it here at runtime; folded checkpoints set embed_scale=None and bake
376
+ # the scale into embed_tokens.weight, so this is a no-op. This mirrors the SGLang native
377
+ # port (sglang_ricursive) so served and trained forwards agree. It must run BEFORE the
378
+ # value-residual seed below, whose source is the (scaled) layer-0 embeddings.
379
+ embed_scale = getattr(self.config, "embed_scale", None)
380
+ if embed_scale is not None:
381
+ inputs_embeds = inputs_embeds * embed_scale
382
+
383
+ if use_cache and past_key_values is None:
384
+ past_key_values = DynamicCache()
385
+
386
+ if cache_position is None:
387
+ past_seen_tokens = (
388
+ past_key_values.get_seq_length() if past_key_values is not None else 0
389
+ )
390
+ cache_position = torch.arange(
391
+ past_seen_tokens,
392
+ past_seen_tokens + inputs_embeds.shape[1],
393
+ device=inputs_embeds.device,
394
+ )
395
+
396
+ if position_ids is None:
397
+ position_ids = cache_position.unsqueeze(0)
398
+
399
+ # Build causal mask
400
+ causal_mask = self._update_causal_mask(
401
+ attention_mask, inputs_embeds, cache_position, past_key_values
402
+ )
403
+
404
+ hidden_states = inputs_embeds
405
+ value_residual = hidden_states if self.config.use_valres else None
406
+ all_hidden_states = () if output_hidden_states else None
407
+ all_self_attns = () if output_attentions else None
408
+
409
+ for layer in self.layers:
410
+ if output_hidden_states:
411
+ all_hidden_states += (hidden_states,)
412
+
413
+ layer_outputs = layer(
414
+ hidden_states,
415
+ attention_mask=causal_mask,
416
+ position_ids=position_ids,
417
+ past_key_value=past_key_values,
418
+ output_attentions=output_attentions,
419
+ use_cache=use_cache,
420
+ cache_position=cache_position,
421
+ value_residual=value_residual,
422
+ **kwargs,
423
+ )
424
+ hidden_states = layer_outputs[0]
425
+
426
+ if output_attentions:
427
+ all_self_attns += (layer_outputs[1],)
428
+
429
+ hidden_states = self.norm(hidden_states)
430
+
431
+ if output_hidden_states:
432
+ all_hidden_states += (hidden_states,)
433
+
434
+ next_cache = past_key_values if use_cache else None
435
+
436
+ if not return_dict:
437
+ return tuple(
438
+ v
439
+ for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
440
+ if v is not None
441
+ )
442
+ return BaseModelOutputWithPast(
443
+ last_hidden_state=hidden_states,
444
+ past_key_values=next_cache,
445
+ hidden_states=all_hidden_states,
446
+ attentions=all_self_attns,
447
+ )
448
+
449
+ def _update_causal_mask(
450
+ self,
451
+ attention_mask: Optional[torch.Tensor],
452
+ input_tensor: torch.Tensor,
453
+ cache_position: torch.Tensor,
454
+ past_key_values: Optional[Cache],
455
+ ) -> Optional[torch.Tensor]:
456
+ # When there's no explicit attention_mask (no padding) and we're doing
457
+ # standard prefill or cached decoding, return None and let SDPA use
458
+ # its efficient is_causal path. This also avoids numerical issues
459
+ # from float min mask values with large activations.
460
+ if attention_mask is None:
461
+ return None
462
+ if torch.all(attention_mask == 1):
463
+ return None
464
+
465
+ # Only build an explicit mask when we have a padding mask
466
+ dtype, device = input_tensor.dtype, input_tensor.device
467
+ sequence_length = input_tensor.shape[1]
468
+
469
+ if past_key_values is not None:
470
+ target_length = past_key_values.get_max_cache_shape()
471
+ if target_length is None or target_length < 0:
472
+ target_length = (
473
+ cache_position[-1].item() + 1 if cache_position is not None else sequence_length
474
+ )
475
+ else:
476
+ target_length = sequence_length
477
+
478
+ causal_mask = torch.full(
479
+ (sequence_length, target_length),
480
+ fill_value=torch.finfo(dtype).min,
481
+ dtype=dtype,
482
+ device=device,
483
+ )
484
+ if sequence_length != 1:
485
+ causal_mask = torch.triu(causal_mask, diagonal=target_length - sequence_length + 1)
486
+
487
+ causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1)
488
+
489
+ if attention_mask.dim() == 2:
490
+ mask_length = attention_mask.shape[-1]
491
+ padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(
492
+ causal_mask.dtype
493
+ ).masked_fill(attention_mask[:, None, None, :] == 0, torch.finfo(dtype).min)
494
+ causal_mask = (
495
+ torch.cat([padding_mask, causal_mask[:, :, :, mask_length:]], dim=-1)
496
+ if mask_length < target_length
497
+ else padding_mask
498
+ )
499
+
500
+ return causal_mask
501
+
502
+
503
+ class RicursiveForCausalLM(RicursivePreTrainedModel):
504
+ _tied_weights_keys = []
505
+
506
+ def __init__(self, config: RicursiveConfig):
507
+ super().__init__(config)
508
+ self.model = RicursiveModel(config)
509
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
510
+ self.post_init()
511
+
512
+ def get_input_embeddings(self):
513
+ return self.model.embed_tokens
514
+
515
+ def set_input_embeddings(self, value):
516
+ self.model.embed_tokens = value
517
+
518
+ def get_output_embeddings(self):
519
+ return self.lm_head
520
+
521
+ def set_output_embeddings(self, new_embeddings):
522
+ self.lm_head = new_embeddings
523
+
524
+ def forward(
525
+ self,
526
+ input_ids: Optional[torch.LongTensor] = None,
527
+ attention_mask: Optional[torch.Tensor] = None,
528
+ position_ids: Optional[torch.LongTensor] = None,
529
+ past_key_values: Optional[Cache] = None,
530
+ inputs_embeds: Optional[torch.FloatTensor] = None,
531
+ labels: Optional[torch.LongTensor] = None,
532
+ use_cache: Optional[bool] = None,
533
+ output_attentions: Optional[bool] = None,
534
+ output_hidden_states: Optional[bool] = None,
535
+ return_dict: Optional[bool] = None,
536
+ cache_position: Optional[torch.LongTensor] = None,
537
+ **kwargs,
538
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
539
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
540
+
541
+ outputs = self.model(
542
+ input_ids=input_ids,
543
+ attention_mask=attention_mask,
544
+ position_ids=position_ids,
545
+ past_key_values=past_key_values,
546
+ inputs_embeds=inputs_embeds,
547
+ use_cache=use_cache,
548
+ output_attentions=output_attentions,
549
+ output_hidden_states=output_hidden_states,
550
+ return_dict=return_dict,
551
+ cache_position=cache_position,
552
+ **kwargs,
553
+ )
554
+ hidden_states = outputs[0]
555
+ logits = self.lm_head(hidden_states)
556
+
557
+ loss = None
558
+ if labels is not None:
559
+ shift_logits = logits[..., :-1, :].contiguous()
560
+ shift_labels = labels[..., 1:].contiguous()
561
+ loss = F.cross_entropy(
562
+ shift_logits.view(-1, shift_logits.size(-1)),
563
+ shift_labels.view(-1),
564
+ )
565
+
566
+ if not return_dict:
567
+ output = (logits,) + outputs[1:]
568
+ return (loss,) + output if loss is not None else output
569
+
570
+ return CausalLMOutputWithPast(
571
+ loss=loss,
572
+ logits=logits,
573
+ past_key_values=outputs.past_key_values,
574
+ hidden_states=outputs.hidden_states,
575
+ attentions=outputs.attentions,
576
+ )
577
+
578
+ def prepare_inputs_for_generation(
579
+ self,
580
+ input_ids: torch.LongTensor,
581
+ past_key_values: Optional[Cache] = None,
582
+ attention_mask: Optional[torch.Tensor] = None,
583
+ inputs_embeds: Optional[torch.FloatTensor] = None,
584
+ cache_position: Optional[torch.LongTensor] = None,
585
+ use_cache: Optional[bool] = None,
586
+ **kwargs,
587
+ ):
588
+ if past_key_values is not None:
589
+ if input_ids.shape[1] != 1:
590
+ input_ids = input_ids[:, cache_position]
591
+
592
+ if inputs_embeds is not None and cache_position[0] == 0:
593
+ model_inputs = {"inputs_embeds": inputs_embeds}
594
+ else:
595
+ model_inputs = {"input_ids": input_ids}
596
+
597
+ model_inputs.update(
598
+ {
599
+ "position_ids": cache_position.unsqueeze(0) if cache_position is not None else None,
600
+ "past_key_values": past_key_values,
601
+ "use_cache": use_cache,
602
+ "attention_mask": attention_mask,
603
+ "cache_position": cache_position,
604
+ }
605
+ )
606
+ return model_inputs
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": {"content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}, "eos_token": "<|im_end|>", "pad_token": {"content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}, "unk_token": {"content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false}}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "100256": {
5
+ "content": "<|extra_id_0|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": false
11
+ },
12
+ "100257": {
13
+ "content": "<|endoftext|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "100258": {
21
+ "content": "<|fim_prefix|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "100259": {
29
+ "content": "<|fim_middle|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "100260": {
37
+ "content": "<|fim_suffix|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "100261": {
45
+ "content": "|||PHONE_NUMBER|||",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": false
51
+ },
52
+ "100262": {
53
+ "content": "|||EMAIL_ADDRESS|||",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": false
59
+ },
60
+ "100263": {
61
+ "content": "|||IP_ADDRESS|||",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": false
67
+ },
68
+ "100264": {
69
+ "content": "<|im_start|>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "100265": {
77
+ "content": "<|im_end|>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "100266": {
85
+ "content": "<|extra_id_1|>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": false
91
+ },
92
+ "100267": {
93
+ "content": "<|extra_id_2|>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": false
99
+ },
100
+ "100268": {
101
+ "content": "<|extra_id_3|>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": false
107
+ },
108
+ "100269": {
109
+ "content": "<|extra_id_4|>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": false
115
+ },
116
+ "100270": {
117
+ "content": "<|extra_id_5|>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": false
123
+ },
124
+ "100271": {
125
+ "content": "<|extra_id_6|>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": false
131
+ },
132
+ "100272": {
133
+ "content": "<|extra_id_7|>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": false
139
+ },
140
+ "100273": {
141
+ "content": "<|extra_id_8|>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": false
147
+ },
148
+ "100274": {
149
+ "content": "<|extra_id_9|>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": false
155
+ },
156
+ "100275": {
157
+ "content": "<|extra_id_10|>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": false
163
+ },
164
+ "100276": {
165
+ "content": "<|endofprompt|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "100277": {
173
+ "content": "<|pad|>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ }
180
+ },
181
+ "bos_token": "<|endoftext|>",
182
+ "chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}",
183
+ "clean_up_tokenization_spaces": false,
184
+ "eos_token": "<|im_end|>",
185
+ "extra_special_tokens": {},
186
+ "model_max_length": 131072,
187
+ "pad_token": "<|pad|>",
188
+ "tokenizer_class": "PreTrainedTokenizerFast",
189
+ "unk_token": "<|endoftext|>"
190
+ }