Hugging Face's logo

ricursiveAI
/
Ripple-65B-Preview-NVFP4

Safetensors
ricursive
custom_code
8-bit precision
modelopt
Model card Files
xet
 Community
1
william-ricursive commited on
Commit
9a8e832
·
verified ·
1 Parent(s): 80402f6

Use conservative NVFP4/BF16 checkpoint and refresh Spark notebook

Browse files

Replace the target with NVFP4 MLP / BF16 attention weights. Update local inference from ripple-inference PR #16 (0f85bbf8d8), retain the shared format parser and template, and remove the demo video. Document conservative quantization in the model card.

Files changed (21) hide show
  1. README.md +3 -1
  2. chat_template.jinja +52 -4
  3. config.json +0 -0
  4. generation_config.json +11 -0
  5. hf_quant_config.json +93 -2490
  6. inference/local/chat_template.jinja +0 -52
  7. inference/local/model.py +99 -2
  8. inference/local/ripple.ipynb +17 -24
  9. inference/local/spark.py +41 -51
  10. inference/server/ripple_format.py +258 -0
  11. dgx-demo.mp4 → model-00001-of-00006.safetensors +2 -2
  12. model-00003-of-00005.safetensors → model-00002-of-00006.safetensors +2 -2
  13. model-00001-of-00005.safetensors → model-00003-of-00006.safetensors +2 -2
  14. model-00004-of-00005.safetensors +0 -3
  15. model-00002-of-00005.safetensors → model-00004-of-00006.safetensors +2 -2
  16. model-00005-of-00005.safetensors +0 -3
  17. model-00005-of-00006.safetensors +3 -0
  18. model-00006-of-00006.safetensors +3 -0
  19. model.safetensors.index.json +0 -0
  20. tokenizer.json +26 -1
  21. tokenizer_config.json +4 -180
README.md CHANGED
@@ -24,7 +24,9 @@ Ripple-65B Preview delivers strong results on agentic coding, scoring **80.52 on
24
  surpassing significantly-larger US open-weight models such as Reflection Beam 501B (**80.1**), Meta Muse Spark Open 816B (**76.4**),
25
  Thinking Machines Inkling 975B (**63.8**), and NVIDIA Nemotron 3 Ultra 550B (**56.4**).
26
 
27
- Our Preview release includes an instruction-tuned model in BF16 and NVFP4 numerics, and a BF16 base model. **This repo contains the mixed NVFP4/FP8 instruction-tuned checkpoint.**
 
 
28
 
29
  The benchmark results below describe the BF16 models and are not measurements of this quantized checkpoint.
30
 
 
24
  surpassing significantly-larger US open-weight models such as Reflection Beam 501B (**80.1**), Meta Muse Spark Open 816B (**76.4**),
25
  Thinking Machines Inkling 975B (**63.8**), and NVIDIA Nemotron 3 Ultra 550B (**56.4**).
26
 
27
+ Our Preview release includes an instruction-tuned model in BF16 and NVFP4 numerics, and a BF16 base model. **This repo contains the NVFP4-MLP/BF16-attention instruction-tuned checkpoint.**
28
+
29
+ This is a conservative quantization of the BF16 checkpoint, using NVFP4 for the MLP layers while keeping attention in BF16.
30
 
31
  The benchmark results below describe the BF16 models and are not measurements of this quantized checkpoint.
32
 
chat_template.jinja CHANGED
@@ -1,4 +1,52 @@
1
- {%- for message in messages %}{{- '<|im_start|>' + message['role'] + '
2
- ' + message['content'] + '<|im_end|>' + '
3
- ' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant
4
- ' }}{%- endif %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0]['role'] == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- for message in messages %}
18
+ {%- set content = message.content or '' %}
19
+ {%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
20
+ {%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
21
+ {%- endif %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if content %}
27
+ {{- '\n' + content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": ' }}
34
+ {{- tool_call.name | tojson }}
35
+ {{- ', "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
45
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
46
+ {{- '<|im_end|>\n' }}
47
+ {%- endif %}
48
+ {%- endif %}
49
+ {%- endfor %}
50
+ {%- if add_generation_prompt %}
51
+ {{- '<|im_start|>assistant\n' }}
52
+ {%- endif %}
config.json CHANGED
The diff for this file is too large to render. See raw diff
 
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 100257,
4
+ "eos_token_id": [
5
+ 100265,
6
+ 100257
7
+ ],
8
+ "pad_token_id": 100277,
9
+ "transformers_version": "5.12.1",
10
+ "use_cache": true
11
+ }
hf_quant_config.json CHANGED
@@ -1,2492 +1,95 @@
1
  {
2
- "producer": {
3
- "name": "modelopt",
4
- "version": "0.46.0"
5
- },
6
- "quantization": {
7
- "kv_cache_quant_algo": null,
8
- "quant_algo": "MIXED_PRECISION",
9
- "quantized_layers": {
10
- "model.layers.0.mlp.down_proj": {
11
- "group_size": 16,
12
- "quant_algo": "NVFP4"
13
- },
14
- "model.layers.0.mlp.gate_proj": {
15
- "group_size": 16,
16
- "quant_algo": "NVFP4"
17
- },
18
- "model.layers.0.mlp.gate_up_proj": {
19
- "group_size": 16,
20
- "quant_algo": "NVFP4"
21
- },
22
- "model.layers.0.mlp.up_proj": {
23
- "group_size": 16,
24
- "quant_algo": "NVFP4"
25
- },
26
- "model.layers.0.self_attn.g_proj": {
27
- "quant_algo": "FP8"
28
- },
29
- "model.layers.0.self_attn.k_proj": {
30
- "quant_algo": "FP8"
31
- },
32
- "model.layers.0.self_attn.o_proj": {
33
- "quant_algo": "FP8"
34
- },
35
- "model.layers.0.self_attn.q_proj": {
36
- "quant_algo": "FP8"
37
- },
38
- "model.layers.0.self_attn.v_proj": {
39
- "quant_algo": "FP8"
40
- },
41
- "model.layers.1.mlp.down_proj": {
42
- "group_size": 16,
43
- "quant_algo": "NVFP4"
44
- },
45
- "model.layers.1.mlp.gate_proj": {
46
- "group_size": 16,
47
- "quant_algo": "NVFP4"
48
- },
49
- "model.layers.1.mlp.gate_up_proj": {
50
- "group_size": 16,
51
- "quant_algo": "NVFP4"
52
- },
53
- "model.layers.1.mlp.up_proj": {
54
- "group_size": 16,
55
- "quant_algo": "NVFP4"
56
- },
57
- "model.layers.1.self_attn.g_proj": {
58
- "quant_algo": "FP8"
59
- },
60
- "model.layers.1.self_attn.k_proj": {
61
- "quant_algo": "FP8"
62
- },
63
- "model.layers.1.self_attn.o_proj": {
64
- "quant_algo": "FP8"
65
- },
66
- "model.layers.1.self_attn.q_proj": {
67
- "quant_algo": "FP8"
68
- },
69
- "model.layers.1.self_attn.v_proj": {
70
- "quant_algo": "FP8"
71
- },
72
- "model.layers.10.mlp.down_proj": {
73
- "group_size": 16,
74
- "quant_algo": "NVFP4"
75
- },
76
- "model.layers.10.mlp.gate_proj": {
77
- "group_size": 16,
78
- "quant_algo": "NVFP4"
79
- },
80
- "model.layers.10.mlp.gate_up_proj": {
81
- "group_size": 16,
82
- "quant_algo": "NVFP4"
83
- },
84
- "model.layers.10.mlp.up_proj": {
85
- "group_size": 16,
86
- "quant_algo": "NVFP4"
87
- },
88
- "model.layers.10.self_attn.g_proj": {
89
- "quant_algo": "FP8"
90
- },
91
- "model.layers.10.self_attn.k_proj": {
92
- "quant_algo": "FP8"
93
- },
94
- "model.layers.10.self_attn.o_proj": {
95
- "quant_algo": "FP8"
96
- },
97
- "model.layers.10.self_attn.q_proj": {
98
- "quant_algo": "FP8"
99
- },
100
- "model.layers.10.self_attn.v_proj": {
101
- "quant_algo": "FP8"
102
- },
103
- "model.layers.11.mlp.down_proj": {
104
- "group_size": 16,
105
- "quant_algo": "NVFP4"
106
- },
107
- "model.layers.11.mlp.gate_proj": {
108
- "group_size": 16,
109
- "quant_algo": "NVFP4"
110
- },
111
- "model.layers.11.mlp.gate_up_proj": {
112
- "group_size": 16,
113
- "quant_algo": "NVFP4"
114
- },
115
- "model.layers.11.mlp.up_proj": {
116
- "group_size": 16,
117
- "quant_algo": "NVFP4"
118
- },
119
- "model.layers.11.self_attn.g_proj": {
120
- "quant_algo": "FP8"
121
- },
122
- "model.layers.11.self_attn.k_proj": {
123
- "quant_algo": "FP8"
124
- },
125
- "model.layers.11.self_attn.o_proj": {
126
- "quant_algo": "FP8"
127
- },
128
- "model.layers.11.self_attn.q_proj": {
129
- "quant_algo": "FP8"
130
- },
131
- "model.layers.11.self_attn.v_proj": {
132
- "quant_algo": "FP8"
133
- },
134
- "model.layers.12.mlp.down_proj": {
135
- "group_size": 16,
136
- "quant_algo": "NVFP4"
137
- },
138
- "model.layers.12.mlp.gate_proj": {
139
- "group_size": 16,
140
- "quant_algo": "NVFP4"
141
- },
142
- "model.layers.12.mlp.gate_up_proj": {
143
- "group_size": 16,
144
- "quant_algo": "NVFP4"
145
- },
146
- "model.layers.12.mlp.up_proj": {
147
- "group_size": 16,
148
- "quant_algo": "NVFP4"
149
- },
150
- "model.layers.12.self_attn.g_proj": {
151
- "quant_algo": "FP8"
152
- },
153
- "model.layers.12.self_attn.k_proj": {
154
- "quant_algo": "FP8"
155
- },
156
- "model.layers.12.self_attn.o_proj": {
157
- "quant_algo": "FP8"
158
- },
159
- "model.layers.12.self_attn.q_proj": {
160
- "quant_algo": "FP8"
161
- },
162
- "model.layers.12.self_attn.v_proj": {
163
- "quant_algo": "FP8"
164
- },
165
- "model.layers.13.mlp.down_proj": {
166
- "group_size": 16,
167
- "quant_algo": "NVFP4"
168
- },
169
- "model.layers.13.mlp.gate_proj": {
170
- "group_size": 16,
171
- "quant_algo": "NVFP4"
172
- },
173
- "model.layers.13.mlp.gate_up_proj": {
174
- "group_size": 16,
175
- "quant_algo": "NVFP4"
176
- },
177
- "model.layers.13.mlp.up_proj": {
178
- "group_size": 16,
179
- "quant_algo": "NVFP4"
180
- },
181
- "model.layers.13.self_attn.g_proj": {
182
- "quant_algo": "FP8"
183
- },
184
- "model.layers.13.self_attn.k_proj": {
185
- "quant_algo": "FP8"
186
- },
187
- "model.layers.13.self_attn.o_proj": {
188
- "quant_algo": "FP8"
189
- },
190
- "model.layers.13.self_attn.q_proj": {
191
- "quant_algo": "FP8"
192
- },
193
- "model.layers.13.self_attn.v_proj": {
194
- "quant_algo": "FP8"
195
- },
196
- "model.layers.14.mlp.down_proj": {
197
- "group_size": 16,
198
- "quant_algo": "NVFP4"
199
- },
200
- "model.layers.14.mlp.gate_proj": {
201
- "group_size": 16,
202
- "quant_algo": "NVFP4"
203
- },
204
- "model.layers.14.mlp.gate_up_proj": {
205
- "group_size": 16,
206
- "quant_algo": "NVFP4"
207
- },
208
- "model.layers.14.mlp.up_proj": {
209
- "group_size": 16,
210
- "quant_algo": "NVFP4"
211
- },
212
- "model.layers.14.self_attn.g_proj": {
213
- "quant_algo": "FP8"
214
- },
215
- "model.layers.14.self_attn.k_proj": {
216
- "quant_algo": "FP8"
217
- },
218
- "model.layers.14.self_attn.o_proj": {
219
- "quant_algo": "FP8"
220
- },
221
- "model.layers.14.self_attn.q_proj": {
222
- "quant_algo": "FP8"
223
- },
224
- "model.layers.14.self_attn.v_proj": {
225
- "quant_algo": "FP8"
226
- },
227
- "model.layers.15.mlp.down_proj": {
228
- "group_size": 16,
229
- "quant_algo": "NVFP4"
230
- },
231
- "model.layers.15.mlp.gate_proj": {
232
- "group_size": 16,
233
- "quant_algo": "NVFP4"
234
- },
235
- "model.layers.15.mlp.gate_up_proj": {
236
- "group_size": 16,
237
- "quant_algo": "NVFP4"
238
- },
239
- "model.layers.15.mlp.up_proj": {
240
- "group_size": 16,
241
- "quant_algo": "NVFP4"
242
- },
243
- "model.layers.15.self_attn.g_proj": {
244
- "quant_algo": "FP8"
245
- },
246
- "model.layers.15.self_attn.k_proj": {
247
- "quant_algo": "FP8"
248
- },
249
- "model.layers.15.self_attn.o_proj": {
250
- "quant_algo": "FP8"
251
- },
252
- "model.layers.15.self_attn.q_proj": {
253
- "quant_algo": "FP8"
254
- },
255
- "model.layers.15.self_attn.v_proj": {
256
- "quant_algo": "FP8"
257
- },
258
- "model.layers.16.mlp.down_proj": {
259
- "group_size": 16,
260
- "quant_algo": "NVFP4"
261
- },
262
- "model.layers.16.mlp.gate_proj": {
263
- "group_size": 16,
264
- "quant_algo": "NVFP4"
265
- },
266
- "model.layers.16.mlp.gate_up_proj": {
267
- "group_size": 16,
268
- "quant_algo": "NVFP4"
269
- },
270
- "model.layers.16.mlp.up_proj": {
271
- "group_size": 16,
272
- "quant_algo": "NVFP4"
273
- },
274
- "model.layers.16.self_attn.g_proj": {
275
- "quant_algo": "FP8"
276
- },
277
- "model.layers.16.self_attn.k_proj": {
278
- "quant_algo": "FP8"
279
- },
280
- "model.layers.16.self_attn.o_proj": {
281
- "quant_algo": "FP8"
282
- },
283
- "model.layers.16.self_attn.q_proj": {
284
- "quant_algo": "FP8"
285
- },
286
- "model.layers.16.self_attn.v_proj": {
287
- "quant_algo": "FP8"
288
- },
289
- "model.layers.17.mlp.down_proj": {
290
- "group_size": 16,
291
- "quant_algo": "NVFP4"
292
- },
293
- "model.layers.17.mlp.gate_proj": {
294
- "group_size": 16,
295
- "quant_algo": "NVFP4"
296
- },
297
- "model.layers.17.mlp.gate_up_proj": {
298
- "group_size": 16,
299
- "quant_algo": "NVFP4"
300
- },
301
- "model.layers.17.mlp.up_proj": {
302
- "group_size": 16,
303
- "quant_algo": "NVFP4"
304
- },
305
- "model.layers.17.self_attn.g_proj": {
306
- "quant_algo": "FP8"
307
- },
308
- "model.layers.17.self_attn.k_proj": {
309
- "quant_algo": "FP8"
310
- },
311
- "model.layers.17.self_attn.o_proj": {
312
- "quant_algo": "FP8"
313
- },
314
- "model.layers.17.self_attn.q_proj": {
315
- "quant_algo": "FP8"
316
- },
317
- "model.layers.17.self_attn.v_proj": {
318
- "quant_algo": "FP8"
319
- },
320
- "model.layers.18.mlp.down_proj": {
321
- "group_size": 16,
322
- "quant_algo": "NVFP4"
323
- },
324
- "model.layers.18.mlp.gate_proj": {
325
- "group_size": 16,
326
- "quant_algo": "NVFP4"
327
- },
328
- "model.layers.18.mlp.gate_up_proj": {
329
- "group_size": 16,
330
- "quant_algo": "NVFP4"
331
- },
332
- "model.layers.18.mlp.up_proj": {
333
- "group_size": 16,
334
- "quant_algo": "NVFP4"
335
- },
336
- "model.layers.18.self_attn.g_proj": {
337
- "quant_algo": "FP8"
338
- },
339
- "model.layers.18.self_attn.k_proj": {
340
- "quant_algo": "FP8"
341
- },
342
- "model.layers.18.self_attn.o_proj": {
343
- "quant_algo": "FP8"
344
- },
345
- "model.layers.18.self_attn.q_proj": {
346
- "quant_algo": "FP8"
347
- },
348
- "model.layers.18.self_attn.v_proj": {
349
- "quant_algo": "FP8"
350
- },
351
- "model.layers.19.mlp.down_proj": {
352
- "group_size": 16,
353
- "quant_algo": "NVFP4"
354
- },
355
- "model.layers.19.mlp.gate_proj": {
356
- "group_size": 16,
357
- "quant_algo": "NVFP4"
358
- },
359
- "model.layers.19.mlp.gate_up_proj": {
360
- "group_size": 16,
361
- "quant_algo": "NVFP4"
362
- },
363
- "model.layers.19.mlp.up_proj": {
364
- "group_size": 16,
365
- "quant_algo": "NVFP4"
366
- },
367
- "model.layers.19.self_attn.g_proj": {
368
- "quant_algo": "FP8"
369
- },
370
- "model.layers.19.self_attn.k_proj": {
371
- "quant_algo": "FP8"
372
- },
373
- "model.layers.19.self_attn.o_proj": {
374
- "quant_algo": "FP8"
375
- },
376
- "model.layers.19.self_attn.q_proj": {
377
- "quant_algo": "FP8"
378
- },
379
- "model.layers.19.self_attn.v_proj": {
380
- "quant_algo": "FP8"
381
- },
382
- "model.layers.2.mlp.down_proj": {
383
- "group_size": 16,
384
- "quant_algo": "NVFP4"
385
- },
386
- "model.layers.2.mlp.gate_proj": {
387
- "group_size": 16,
388
- "quant_algo": "NVFP4"
389
- },
390
- "model.layers.2.mlp.gate_up_proj": {
391
- "group_size": 16,
392
- "quant_algo": "NVFP4"
393
- },
394
- "model.layers.2.mlp.up_proj": {
395
- "group_size": 16,
396
- "quant_algo": "NVFP4"
397
- },
398
- "model.layers.2.self_attn.g_proj": {
399
- "quant_algo": "FP8"
400
- },
401
- "model.layers.2.self_attn.k_proj": {
402
- "quant_algo": "FP8"
403
- },
404
- "model.layers.2.self_attn.o_proj": {
405
- "quant_algo": "FP8"
406
- },
407
- "model.layers.2.self_attn.q_proj": {
408
- "quant_algo": "FP8"
409
- },
410
- "model.layers.2.self_attn.v_proj": {
411
- "quant_algo": "FP8"
412
- },
413
- "model.layers.20.mlp.down_proj": {
414
- "group_size": 16,
415
- "quant_algo": "NVFP4"
416
- },
417
- "model.layers.20.mlp.gate_proj": {
418
- "group_size": 16,
419
- "quant_algo": "NVFP4"
420
- },
421
- "model.layers.20.mlp.gate_up_proj": {
422
- "group_size": 16,
423
- "quant_algo": "NVFP4"
424
- },
425
- "model.layers.20.mlp.up_proj": {
426
- "group_size": 16,
427
- "quant_algo": "NVFP4"
428
- },
429
- "model.layers.20.self_attn.g_proj": {
430
- "quant_algo": "FP8"
431
- },
432
- "model.layers.20.self_attn.k_proj": {
433
- "quant_algo": "FP8"
434
- },
435
- "model.layers.20.self_attn.o_proj": {
436
- "quant_algo": "FP8"
437
- },
438
- "model.layers.20.self_attn.q_proj": {
439
- "quant_algo": "FP8"
440
- },
441
- "model.layers.20.self_attn.v_proj": {
442
- "quant_algo": "FP8"
443
- },
444
- "model.layers.21.mlp.down_proj": {
445
- "group_size": 16,
446
- "quant_algo": "NVFP4"
447
- },
448
- "model.layers.21.mlp.gate_proj": {
449
- "group_size": 16,
450
- "quant_algo": "NVFP4"
451
- },
452
- "model.layers.21.mlp.gate_up_proj": {
453
- "group_size": 16,
454
- "quant_algo": "NVFP4"
455
- },
456
- "model.layers.21.mlp.up_proj": {
457
- "group_size": 16,
458
- "quant_algo": "NVFP4"
459
- },
460
- "model.layers.21.self_attn.g_proj": {
461
- "quant_algo": "FP8"
462
- },
463
- "model.layers.21.self_attn.k_proj": {
464
- "quant_algo": "FP8"
465
- },
466
- "model.layers.21.self_attn.o_proj": {
467
- "quant_algo": "FP8"
468
- },
469
- "model.layers.21.self_attn.q_proj": {
470
- "quant_algo": "FP8"
471
- },
472
- "model.layers.21.self_attn.v_proj": {
473
- "quant_algo": "FP8"
474
- },
475
- "model.layers.22.mlp.down_proj": {
476
- "group_size": 16,
477
- "quant_algo": "NVFP4"
478
- },
479
- "model.layers.22.mlp.gate_proj": {
480
- "group_size": 16,
481
- "quant_algo": "NVFP4"
482
- },
483
- "model.layers.22.mlp.gate_up_proj": {
484
- "group_size": 16,
485
- "quant_algo": "NVFP4"
486
- },
487
- "model.layers.22.mlp.up_proj": {
488
- "group_size": 16,
489
- "quant_algo": "NVFP4"
490
- },
491
- "model.layers.22.self_attn.g_proj": {
492
- "quant_algo": "FP8"
493
- },
494
- "model.layers.22.self_attn.k_proj": {
495
- "quant_algo": "FP8"
496
- },
497
- "model.layers.22.self_attn.o_proj": {
498
- "quant_algo": "FP8"
499
- },
500
- "model.layers.22.self_attn.q_proj": {
501
- "quant_algo": "FP8"
502
- },
503
- "model.layers.22.self_attn.v_proj": {
504
- "quant_algo": "FP8"
505
- },
506
- "model.layers.23.mlp.down_proj": {
507
- "group_size": 16,
508
- "quant_algo": "NVFP4"
509
- },
510
- "model.layers.23.mlp.gate_proj": {
511
- "group_size": 16,
512
- "quant_algo": "NVFP4"
513
- },
514
- "model.layers.23.mlp.gate_up_proj": {
515
- "group_size": 16,
516
- "quant_algo": "NVFP4"
517
- },
518
- "model.layers.23.mlp.up_proj": {
519
- "group_size": 16,
520
- "quant_algo": "NVFP4"
521
- },
522
- "model.layers.23.self_attn.g_proj": {
523
- "quant_algo": "FP8"
524
- },
525
- "model.layers.23.self_attn.k_proj": {
526
- "quant_algo": "FP8"
527
- },
528
- "model.layers.23.self_attn.o_proj": {
529
- "quant_algo": "FP8"
530
- },
531
- "model.layers.23.self_attn.q_proj": {
532
- "quant_algo": "FP8"
533
- },
534
- "model.layers.23.self_attn.v_proj": {
535
- "quant_algo": "FP8"
536
- },
537
- "model.layers.24.mlp.down_proj": {
538
- "group_size": 16,
539
- "quant_algo": "NVFP4"
540
- },
541
- "model.layers.24.mlp.gate_proj": {
542
- "group_size": 16,
543
- "quant_algo": "NVFP4"
544
- },
545
- "model.layers.24.mlp.gate_up_proj": {
546
- "group_size": 16,
547
- "quant_algo": "NVFP4"
548
- },
549
- "model.layers.24.mlp.up_proj": {
550
- "group_size": 16,
551
- "quant_algo": "NVFP4"
552
- },
553
- "model.layers.24.self_attn.g_proj": {
554
- "quant_algo": "FP8"
555
- },
556
- "model.layers.24.self_attn.k_proj": {
557
- "quant_algo": "FP8"
558
- },
559
- "model.layers.24.self_attn.o_proj": {
560
- "quant_algo": "FP8"
561
- },
562
- "model.layers.24.self_attn.q_proj": {
563
- "quant_algo": "FP8"
564
- },
565
- "model.layers.24.self_attn.v_proj": {
566
- "quant_algo": "FP8"
567
- },
568
- "model.layers.25.mlp.down_proj": {
569
- "group_size": 16,
570
- "quant_algo": "NVFP4"
571
- },
572
- "model.layers.25.mlp.gate_proj": {
573
- "group_size": 16,
574
- "quant_algo": "NVFP4"
575
- },
576
- "model.layers.25.mlp.gate_up_proj": {
577
- "group_size": 16,
578
- "quant_algo": "NVFP4"
579
- },
580
- "model.layers.25.mlp.up_proj": {
581
- "group_size": 16,
582
- "quant_algo": "NVFP4"
583
- },
584
- "model.layers.25.self_attn.g_proj": {
585
- "quant_algo": "FP8"
586
- },
587
- "model.layers.25.self_attn.k_proj": {
588
- "quant_algo": "FP8"
589
- },
590
- "model.layers.25.self_attn.o_proj": {
591
- "quant_algo": "FP8"
592
- },
593
- "model.layers.25.self_attn.q_proj": {
594
- "quant_algo": "FP8"
595
- },
596
- "model.layers.25.self_attn.v_proj": {
597
- "quant_algo": "FP8"
598
- },
599
- "model.layers.26.mlp.down_proj": {
600
- "group_size": 16,
601
- "quant_algo": "NVFP4"
602
- },
603
- "model.layers.26.mlp.gate_proj": {
604
- "group_size": 16,
605
- "quant_algo": "NVFP4"
606
- },
607
- "model.layers.26.mlp.gate_up_proj": {
608
- "group_size": 16,
609
- "quant_algo": "NVFP4"
610
- },
611
- "model.layers.26.mlp.up_proj": {
612
- "group_size": 16,
613
- "quant_algo": "NVFP4"
614
- },
615
- "model.layers.26.self_attn.g_proj": {
616
- "quant_algo": "FP8"
617
- },
618
- "model.layers.26.self_attn.k_proj": {
619
- "quant_algo": "FP8"
620
- },
621
- "model.layers.26.self_attn.o_proj": {
622
- "quant_algo": "FP8"
623
- },
624
- "model.layers.26.self_attn.q_proj": {
625
- "quant_algo": "FP8"
626
- },
627
- "model.layers.26.self_attn.v_proj": {
628
- "quant_algo": "FP8"
629
- },
630
- "model.layers.27.mlp.down_proj": {
631
- "group_size": 16,
632
- "quant_algo": "NVFP4"
633
- },
634
- "model.layers.27.mlp.gate_proj": {
635
- "group_size": 16,
636
- "quant_algo": "NVFP4"
637
- },
638
- "model.layers.27.mlp.gate_up_proj": {
639
- "group_size": 16,
640
- "quant_algo": "NVFP4"
641
- },
642
- "model.layers.27.mlp.up_proj": {
643
- "group_size": 16,
644
- "quant_algo": "NVFP4"
645
- },
646
- "model.layers.27.self_attn.g_proj": {
647
- "quant_algo": "FP8"
648
- },
649
- "model.layers.27.self_attn.k_proj": {
650
- "quant_algo": "FP8"
651
- },
652
- "model.layers.27.self_attn.o_proj": {
653
- "quant_algo": "FP8"
654
- },
655
- "model.layers.27.self_attn.q_proj": {
656
- "quant_algo": "FP8"
657
- },
658
- "model.layers.27.self_attn.v_proj": {
659
- "quant_algo": "FP8"
660
- },
661
- "model.layers.28.mlp.down_proj": {
662
- "group_size": 16,
663
- "quant_algo": "NVFP4"
664
- },
665
- "model.layers.28.mlp.gate_proj": {
666
- "group_size": 16,
667
- "quant_algo": "NVFP4"
668
- },
669
- "model.layers.28.mlp.gate_up_proj": {
670
- "group_size": 16,
671
- "quant_algo": "NVFP4"
672
- },
673
- "model.layers.28.mlp.up_proj": {
674
- "group_size": 16,
675
- "quant_algo": "NVFP4"
676
- },
677
- "model.layers.28.self_attn.g_proj": {
678
- "quant_algo": "FP8"
679
- },
680
- "model.layers.28.self_attn.k_proj": {
681
- "quant_algo": "FP8"
682
- },
683
- "model.layers.28.self_attn.o_proj": {
684
- "quant_algo": "FP8"
685
- },
686
- "model.layers.28.self_attn.q_proj": {
687
- "quant_algo": "FP8"
688
- },
689
- "model.layers.28.self_attn.v_proj": {
690
- "quant_algo": "FP8"
691
- },
692
- "model.layers.29.mlp.down_proj": {
693
- "group_size": 16,
694
- "quant_algo": "NVFP4"
695
- },
696
- "model.layers.29.mlp.gate_proj": {
697
- "group_size": 16,
698
- "quant_algo": "NVFP4"
699
- },
700
- "model.layers.29.mlp.gate_up_proj": {
701
- "group_size": 16,
702
- "quant_algo": "NVFP4"
703
- },
704
- "model.layers.29.mlp.up_proj": {
705
- "group_size": 16,
706
- "quant_algo": "NVFP4"
707
- },
708
- "model.layers.29.self_attn.g_proj": {
709
- "quant_algo": "FP8"
710
- },
711
- "model.layers.29.self_attn.k_proj": {
712
- "quant_algo": "FP8"
713
- },
714
- "model.layers.29.self_attn.o_proj": {
715
- "quant_algo": "FP8"
716
- },
717
- "model.layers.29.self_attn.q_proj": {
718
- "quant_algo": "FP8"
719
- },
720
- "model.layers.29.self_attn.v_proj": {
721
- "quant_algo": "FP8"
722
- },
723
- "model.layers.3.mlp.down_proj": {
724
- "group_size": 16,
725
- "quant_algo": "NVFP4"
726
- },
727
- "model.layers.3.mlp.gate_proj": {
728
- "group_size": 16,
729
- "quant_algo": "NVFP4"
730
- },
731
- "model.layers.3.mlp.gate_up_proj": {
732
- "group_size": 16,
733
- "quant_algo": "NVFP4"
734
- },
735
- "model.layers.3.mlp.up_proj": {
736
- "group_size": 16,
737
- "quant_algo": "NVFP4"
738
- },
739
- "model.layers.3.self_attn.g_proj": {
740
- "quant_algo": "FP8"
741
- },
742
- "model.layers.3.self_attn.k_proj": {
743
- "quant_algo": "FP8"
744
- },
745
- "model.layers.3.self_attn.o_proj": {
746
- "quant_algo": "FP8"
747
- },
748
- "model.layers.3.self_attn.q_proj": {
749
- "quant_algo": "FP8"
750
- },
751
- "model.layers.3.self_attn.v_proj": {
752
- "quant_algo": "FP8"
753
- },
754
- "model.layers.30.mlp.down_proj": {
755
- "group_size": 16,
756
- "quant_algo": "NVFP4"
757
- },
758
- "model.layers.30.mlp.gate_proj": {
759
- "group_size": 16,
760
- "quant_algo": "NVFP4"
761
- },
762
- "model.layers.30.mlp.gate_up_proj": {
763
- "group_size": 16,
764
- "quant_algo": "NVFP4"
765
- },
766
- "model.layers.30.mlp.up_proj": {
767
- "group_size": 16,
768
- "quant_algo": "NVFP4"
769
- },
770
- "model.layers.30.self_attn.g_proj": {
771
- "quant_algo": "FP8"
772
- },
773
- "model.layers.30.self_attn.k_proj": {
774
- "quant_algo": "FP8"
775
- },
776
- "model.layers.30.self_attn.o_proj": {
777
- "quant_algo": "FP8"
778
- },
779
- "model.layers.30.self_attn.q_proj": {
780
- "quant_algo": "FP8"
781
- },
782
- "model.layers.30.self_attn.v_proj": {
783
- "quant_algo": "FP8"
784
- },
785
- "model.layers.31.mlp.down_proj": {
786
- "group_size": 16,
787
- "quant_algo": "NVFP4"
788
- },
789
- "model.layers.31.mlp.gate_proj": {
790
- "group_size": 16,
791
- "quant_algo": "NVFP4"
792
- },
793
- "model.layers.31.mlp.gate_up_proj": {
794
- "group_size": 16,
795
- "quant_algo": "NVFP4"
796
- },
797
- "model.layers.31.mlp.up_proj": {
798
- "group_size": 16,
799
- "quant_algo": "NVFP4"
800
- },
801
- "model.layers.31.self_attn.g_proj": {
802
- "quant_algo": "FP8"
803
- },
804
- "model.layers.31.self_attn.k_proj": {
805
- "quant_algo": "FP8"
806
- },
807
- "model.layers.31.self_attn.o_proj": {
808
- "quant_algo": "FP8"
809
- },
810
- "model.layers.31.self_attn.q_proj": {
811
- "quant_algo": "FP8"
812
- },
813
- "model.layers.31.self_attn.v_proj": {
814
- "quant_algo": "FP8"
815
- },
816
- "model.layers.32.mlp.down_proj": {
817
- "group_size": 16,
818
- "quant_algo": "NVFP4"
819
- },
820
- "model.layers.32.mlp.gate_proj": {
821
- "group_size": 16,
822
- "quant_algo": "NVFP4"
823
- },
824
- "model.layers.32.mlp.gate_up_proj": {
825
- "group_size": 16,
826
- "quant_algo": "NVFP4"
827
- },
828
- "model.layers.32.mlp.up_proj": {
829
- "group_size": 16,
830
- "quant_algo": "NVFP4"
831
- },
832
- "model.layers.32.self_attn.g_proj": {
833
- "quant_algo": "FP8"
834
- },
835
- "model.layers.32.self_attn.k_proj": {
836
- "quant_algo": "FP8"
837
- },
838
- "model.layers.32.self_attn.o_proj": {
839
- "quant_algo": "FP8"
840
- },
841
- "model.layers.32.self_attn.q_proj": {
842
- "quant_algo": "FP8"
843
- },
844
- "model.layers.32.self_attn.v_proj": {
845
- "quant_algo": "FP8"
846
- },
847
- "model.layers.33.mlp.down_proj": {
848
- "group_size": 16,
849
- "quant_algo": "NVFP4"
850
- },
851
- "model.layers.33.mlp.gate_proj": {
852
- "group_size": 16,
853
- "quant_algo": "NVFP4"
854
- },
855
- "model.layers.33.mlp.gate_up_proj": {
856
- "group_size": 16,
857
- "quant_algo": "NVFP4"
858
- },
859
- "model.layers.33.mlp.up_proj": {
860
- "group_size": 16,
861
- "quant_algo": "NVFP4"
862
- },
863
- "model.layers.33.self_attn.g_proj": {
864
- "quant_algo": "FP8"
865
- },
866
- "model.layers.33.self_attn.k_proj": {
867
- "quant_algo": "FP8"
868
- },
869
- "model.layers.33.self_attn.o_proj": {
870
- "quant_algo": "FP8"
871
- },
872
- "model.layers.33.self_attn.q_proj": {
873
- "quant_algo": "FP8"
874
- },
875
- "model.layers.33.self_attn.v_proj": {
876
- "quant_algo": "FP8"
877
- },
878
- "model.layers.34.mlp.down_proj": {
879
- "group_size": 16,
880
- "quant_algo": "NVFP4"
881
- },
882
- "model.layers.34.mlp.gate_proj": {
883
- "group_size": 16,
884
- "quant_algo": "NVFP4"
885
- },
886
- "model.layers.34.mlp.gate_up_proj": {
887
- "group_size": 16,
888
- "quant_algo": "NVFP4"
889
- },
890
- "model.layers.34.mlp.up_proj": {
891
- "group_size": 16,
892
- "quant_algo": "NVFP4"
893
- },
894
- "model.layers.34.self_attn.g_proj": {
895
- "quant_algo": "FP8"
896
- },
897
- "model.layers.34.self_attn.k_proj": {
898
- "quant_algo": "FP8"
899
- },
900
- "model.layers.34.self_attn.o_proj": {
901
- "quant_algo": "FP8"
902
- },
903
- "model.layers.34.self_attn.q_proj": {
904
- "quant_algo": "FP8"
905
- },
906
- "model.layers.34.self_attn.v_proj": {
907
- "quant_algo": "FP8"
908
- },
909
- "model.layers.35.mlp.down_proj": {
910
- "group_size": 16,
911
- "quant_algo": "NVFP4"
912
- },
913
- "model.layers.35.mlp.gate_proj": {
914
- "group_size": 16,
915
- "quant_algo": "NVFP4"
916
- },
917
- "model.layers.35.mlp.gate_up_proj": {
918
- "group_size": 16,
919
- "quant_algo": "NVFP4"
920
- },
921
- "model.layers.35.mlp.up_proj": {
922
- "group_size": 16,
923
- "quant_algo": "NVFP4"
924
- },
925
- "model.layers.35.self_attn.g_proj": {
926
- "quant_algo": "FP8"
927
- },
928
- "model.layers.35.self_attn.k_proj": {
929
- "quant_algo": "FP8"
930
- },
931
- "model.layers.35.self_attn.o_proj": {
932
- "quant_algo": "FP8"
933
- },
934
- "model.layers.35.self_attn.q_proj": {
935
- "quant_algo": "FP8"
936
- },
937
- "model.layers.35.self_attn.v_proj": {
938
- "quant_algo": "FP8"
939
- },
940
- "model.layers.36.mlp.down_proj": {
941
- "group_size": 16,
942
- "quant_algo": "NVFP4"
943
- },
944
- "model.layers.36.mlp.gate_proj": {
945
- "group_size": 16,
946
- "quant_algo": "NVFP4"
947
- },
948
- "model.layers.36.mlp.gate_up_proj": {
949
- "group_size": 16,
950
- "quant_algo": "NVFP4"
951
- },
952
- "model.layers.36.mlp.up_proj": {
953
- "group_size": 16,
954
- "quant_algo": "NVFP4"
955
- },
956
- "model.layers.36.self_attn.g_proj": {
957
- "quant_algo": "FP8"
958
- },
959
- "model.layers.36.self_attn.k_proj": {
960
- "quant_algo": "FP8"
961
- },
962
- "model.layers.36.self_attn.o_proj": {
963
- "quant_algo": "FP8"
964
- },
965
- "model.layers.36.self_attn.q_proj": {
966
- "quant_algo": "FP8"
967
- },
968
- "model.layers.36.self_attn.v_proj": {
969
- "quant_algo": "FP8"
970
- },
971
- "model.layers.37.mlp.down_proj": {
972
- "group_size": 16,
973
- "quant_algo": "NVFP4"
974
- },
975
- "model.layers.37.mlp.gate_proj": {
976
- "group_size": 16,
977
- "quant_algo": "NVFP4"
978
- },
979
- "model.layers.37.mlp.gate_up_proj": {
980
- "group_size": 16,
981
- "quant_algo": "NVFP4"
982
- },
983
- "model.layers.37.mlp.up_proj": {
984
- "group_size": 16,
985
- "quant_algo": "NVFP4"
986
- },
987
- "model.layers.37.self_attn.g_proj": {
988
- "quant_algo": "FP8"
989
- },
990
- "model.layers.37.self_attn.k_proj": {
991
- "quant_algo": "FP8"
992
- },
993
- "model.layers.37.self_attn.o_proj": {
994
- "quant_algo": "FP8"
995
- },
996
- "model.layers.37.self_attn.q_proj": {
997
- "quant_algo": "FP8"
998
- },
999
- "model.layers.37.self_attn.v_proj": {
1000
- "quant_algo": "FP8"
1001
- },
1002
- "model.layers.38.mlp.down_proj": {
1003
- "group_size": 16,
1004
- "quant_algo": "NVFP4"
1005
- },
1006
- "model.layers.38.mlp.gate_proj": {
1007
- "group_size": 16,
1008
- "quant_algo": "NVFP4"
1009
- },
1010
- "model.layers.38.mlp.gate_up_proj": {
1011
- "group_size": 16,
1012
- "quant_algo": "NVFP4"
1013
- },
1014
- "model.layers.38.mlp.up_proj": {
1015
- "group_size": 16,
1016
- "quant_algo": "NVFP4"
1017
- },
1018
- "model.layers.38.self_attn.g_proj": {
1019
- "quant_algo": "FP8"
1020
- },
1021
- "model.layers.38.self_attn.k_proj": {
1022
- "quant_algo": "FP8"
1023
- },
1024
- "model.layers.38.self_attn.o_proj": {
1025
- "quant_algo": "FP8"
1026
- },
1027
- "model.layers.38.self_attn.q_proj": {
1028
- "quant_algo": "FP8"
1029
- },
1030
- "model.layers.38.self_attn.v_proj": {
1031
- "quant_algo": "FP8"
1032
- },
1033
- "model.layers.39.mlp.down_proj": {
1034
- "group_size": 16,
1035
- "quant_algo": "NVFP4"
1036
- },
1037
- "model.layers.39.mlp.gate_proj": {
1038
- "group_size": 16,
1039
- "quant_algo": "NVFP4"
1040
- },
1041
- "model.layers.39.mlp.gate_up_proj": {
1042
- "group_size": 16,
1043
- "quant_algo": "NVFP4"
1044
- },
1045
- "model.layers.39.mlp.up_proj": {
1046
- "group_size": 16,
1047
- "quant_algo": "NVFP4"
1048
- },
1049
- "model.layers.39.self_attn.g_proj": {
1050
- "quant_algo": "FP8"
1051
- },
1052
- "model.layers.39.self_attn.k_proj": {
1053
- "quant_algo": "FP8"
1054
- },
1055
- "model.layers.39.self_attn.o_proj": {
1056
- "quant_algo": "FP8"
1057
- },
1058
- "model.layers.39.self_attn.q_proj": {
1059
- "quant_algo": "FP8"
1060
- },
1061
- "model.layers.39.self_attn.v_proj": {
1062
- "quant_algo": "FP8"
1063
- },
1064
- "model.layers.4.mlp.down_proj": {
1065
- "group_size": 16,
1066
- "quant_algo": "NVFP4"
1067
- },
1068
- "model.layers.4.mlp.gate_proj": {
1069
- "group_size": 16,
1070
- "quant_algo": "NVFP4"
1071
- },
1072
- "model.layers.4.mlp.gate_up_proj": {
1073
- "group_size": 16,
1074
- "quant_algo": "NVFP4"
1075
- },
1076
- "model.layers.4.mlp.up_proj": {
1077
- "group_size": 16,
1078
- "quant_algo": "NVFP4"
1079
- },
1080
- "model.layers.4.self_attn.g_proj": {
1081
- "quant_algo": "FP8"
1082
- },
1083
- "model.layers.4.self_attn.k_proj": {
1084
- "quant_algo": "FP8"
1085
- },
1086
- "model.layers.4.self_attn.o_proj": {
1087
- "quant_algo": "FP8"
1088
- },
1089
- "model.layers.4.self_attn.q_proj": {
1090
- "quant_algo": "FP8"
1091
- },
1092
- "model.layers.4.self_attn.v_proj": {
1093
- "quant_algo": "FP8"
1094
- },
1095
- "model.layers.40.mlp.down_proj": {
1096
- "group_size": 16,
1097
- "quant_algo": "NVFP4"
1098
- },
1099
- "model.layers.40.mlp.gate_proj": {
1100
- "group_size": 16,
1101
- "quant_algo": "NVFP4"
1102
- },
1103
- "model.layers.40.mlp.gate_up_proj": {
1104
- "group_size": 16,
1105
- "quant_algo": "NVFP4"
1106
- },
1107
- "model.layers.40.mlp.up_proj": {
1108
- "group_size": 16,
1109
- "quant_algo": "NVFP4"
1110
- },
1111
- "model.layers.40.self_attn.g_proj": {
1112
- "quant_algo": "FP8"
1113
- },
1114
- "model.layers.40.self_attn.k_proj": {
1115
- "quant_algo": "FP8"
1116
- },
1117
- "model.layers.40.self_attn.o_proj": {
1118
- "quant_algo": "FP8"
1119
- },
1120
- "model.layers.40.self_attn.q_proj": {
1121
- "quant_algo": "FP8"
1122
- },
1123
- "model.layers.40.self_attn.v_proj": {
1124
- "quant_algo": "FP8"
1125
- },
1126
- "model.layers.41.mlp.down_proj": {
1127
- "group_size": 16,
1128
- "quant_algo": "NVFP4"
1129
- },
1130
- "model.layers.41.mlp.gate_proj": {
1131
- "group_size": 16,
1132
- "quant_algo": "NVFP4"
1133
- },
1134
- "model.layers.41.mlp.gate_up_proj": {
1135
- "group_size": 16,
1136
- "quant_algo": "NVFP4"
1137
- },
1138
- "model.layers.41.mlp.up_proj": {
1139
- "group_size": 16,
1140
- "quant_algo": "NVFP4"
1141
- },
1142
- "model.layers.41.self_attn.g_proj": {
1143
- "quant_algo": "FP8"
1144
- },
1145
- "model.layers.41.self_attn.k_proj": {
1146
- "quant_algo": "FP8"
1147
- },
1148
- "model.layers.41.self_attn.o_proj": {
1149
- "quant_algo": "FP8"
1150
- },
1151
- "model.layers.41.self_attn.q_proj": {
1152
- "quant_algo": "FP8"
1153
- },
1154
- "model.layers.41.self_attn.v_proj": {
1155
- "quant_algo": "FP8"
1156
- },
1157
- "model.layers.42.mlp.down_proj": {
1158
- "group_size": 16,
1159
- "quant_algo": "NVFP4"
1160
- },
1161
- "model.layers.42.mlp.gate_proj": {
1162
- "group_size": 16,
1163
- "quant_algo": "NVFP4"
1164
- },
1165
- "model.layers.42.mlp.gate_up_proj": {
1166
- "group_size": 16,
1167
- "quant_algo": "NVFP4"
1168
- },
1169
- "model.layers.42.mlp.up_proj": {
1170
- "group_size": 16,
1171
- "quant_algo": "NVFP4"
1172
- },
1173
- "model.layers.42.self_attn.g_proj": {
1174
- "quant_algo": "FP8"
1175
- },
1176
- "model.layers.42.self_attn.k_proj": {
1177
- "quant_algo": "FP8"
1178
- },
1179
- "model.layers.42.self_attn.o_proj": {
1180
- "quant_algo": "FP8"
1181
- },
1182
- "model.layers.42.self_attn.q_proj": {
1183
- "quant_algo": "FP8"
1184
- },
1185
- "model.layers.42.self_attn.v_proj": {
1186
- "quant_algo": "FP8"
1187
- },
1188
- "model.layers.43.mlp.down_proj": {
1189
- "group_size": 16,
1190
- "quant_algo": "NVFP4"
1191
- },
1192
- "model.layers.43.mlp.gate_proj": {
1193
- "group_size": 16,
1194
- "quant_algo": "NVFP4"
1195
- },
1196
- "model.layers.43.mlp.gate_up_proj": {
1197
- "group_size": 16,
1198
- "quant_algo": "NVFP4"
1199
- },
1200
- "model.layers.43.mlp.up_proj": {
1201
- "group_size": 16,
1202
- "quant_algo": "NVFP4"
1203
- },
1204
- "model.layers.43.self_attn.g_proj": {
1205
- "quant_algo": "FP8"
1206
- },
1207
- "model.layers.43.self_attn.k_proj": {
1208
- "quant_algo": "FP8"
1209
- },
1210
- "model.layers.43.self_attn.o_proj": {
1211
- "quant_algo": "FP8"
1212
- },
1213
- "model.layers.43.self_attn.q_proj": {
1214
- "quant_algo": "FP8"
1215
- },
1216
- "model.layers.43.self_attn.v_proj": {
1217
- "quant_algo": "FP8"
1218
- },
1219
- "model.layers.44.mlp.down_proj": {
1220
- "group_size": 16,
1221
- "quant_algo": "NVFP4"
1222
- },
1223
- "model.layers.44.mlp.gate_proj": {
1224
- "group_size": 16,
1225
- "quant_algo": "NVFP4"
1226
- },
1227
- "model.layers.44.mlp.gate_up_proj": {
1228
- "group_size": 16,
1229
- "quant_algo": "NVFP4"
1230
- },
1231
- "model.layers.44.mlp.up_proj": {
1232
- "group_size": 16,
1233
- "quant_algo": "NVFP4"
1234
- },
1235
- "model.layers.44.self_attn.g_proj": {
1236
- "quant_algo": "FP8"
1237
- },
1238
- "model.layers.44.self_attn.k_proj": {
1239
- "quant_algo": "FP8"
1240
- },
1241
- "model.layers.44.self_attn.o_proj": {
1242
- "quant_algo": "FP8"
1243
- },
1244
- "model.layers.44.self_attn.q_proj": {
1245
- "quant_algo": "FP8"
1246
- },
1247
- "model.layers.44.self_attn.v_proj": {
1248
- "quant_algo": "FP8"
1249
- },
1250
- "model.layers.45.mlp.down_proj": {
1251
- "group_size": 16,
1252
- "quant_algo": "NVFP4"
1253
- },
1254
- "model.layers.45.mlp.gate_proj": {
1255
- "group_size": 16,
1256
- "quant_algo": "NVFP4"
1257
- },
1258
- "model.layers.45.mlp.gate_up_proj": {
1259
- "group_size": 16,
1260
- "quant_algo": "NVFP4"
1261
- },
1262
- "model.layers.45.mlp.up_proj": {
1263
- "group_size": 16,
1264
- "quant_algo": "NVFP4"
1265
- },
1266
- "model.layers.45.self_attn.g_proj": {
1267
- "quant_algo": "FP8"
1268
- },
1269
- "model.layers.45.self_attn.k_proj": {
1270
- "quant_algo": "FP8"
1271
- },
1272
- "model.layers.45.self_attn.o_proj": {
1273
- "quant_algo": "FP8"
1274
- },
1275
- "model.layers.45.self_attn.q_proj": {
1276
- "quant_algo": "FP8"
1277
- },
1278
- "model.layers.45.self_attn.v_proj": {
1279
- "quant_algo": "FP8"
1280
- },
1281
- "model.layers.46.mlp.down_proj": {
1282
- "group_size": 16,
1283
- "quant_algo": "NVFP4"
1284
- },
1285
- "model.layers.46.mlp.gate_proj": {
1286
- "group_size": 16,
1287
- "quant_algo": "NVFP4"
1288
- },
1289
- "model.layers.46.mlp.gate_up_proj": {
1290
- "group_size": 16,
1291
- "quant_algo": "NVFP4"
1292
- },
1293
- "model.layers.46.mlp.up_proj": {
1294
- "group_size": 16,
1295
- "quant_algo": "NVFP4"
1296
- },
1297
- "model.layers.46.self_attn.g_proj": {
1298
- "quant_algo": "FP8"
1299
- },
1300
- "model.layers.46.self_attn.k_proj": {
1301
- "quant_algo": "FP8"
1302
- },
1303
- "model.layers.46.self_attn.o_proj": {
1304
- "quant_algo": "FP8"
1305
- },
1306
- "model.layers.46.self_attn.q_proj": {
1307
- "quant_algo": "FP8"
1308
- },
1309
- "model.layers.46.self_attn.v_proj": {
1310
- "quant_algo": "FP8"
1311
- },
1312
- "model.layers.47.mlp.down_proj": {
1313
- "group_size": 16,
1314
- "quant_algo": "NVFP4"
1315
- },
1316
- "model.layers.47.mlp.gate_proj": {
1317
- "group_size": 16,
1318
- "quant_algo": "NVFP4"
1319
- },
1320
- "model.layers.47.mlp.gate_up_proj": {
1321
- "group_size": 16,
1322
- "quant_algo": "NVFP4"
1323
- },
1324
- "model.layers.47.mlp.up_proj": {
1325
- "group_size": 16,
1326
- "quant_algo": "NVFP4"
1327
- },
1328
- "model.layers.47.self_attn.g_proj": {
1329
- "quant_algo": "FP8"
1330
- },
1331
- "model.layers.47.self_attn.k_proj": {
1332
- "quant_algo": "FP8"
1333
- },
1334
- "model.layers.47.self_attn.o_proj": {
1335
- "quant_algo": "FP8"
1336
- },
1337
- "model.layers.47.self_attn.q_proj": {
1338
- "quant_algo": "FP8"
1339
- },
1340
- "model.layers.47.self_attn.v_proj": {
1341
- "quant_algo": "FP8"
1342
- },
1343
- "model.layers.48.mlp.down_proj": {
1344
- "group_size": 16,
1345
- "quant_algo": "NVFP4"
1346
- },
1347
- "model.layers.48.mlp.gate_proj": {
1348
- "group_size": 16,
1349
- "quant_algo": "NVFP4"
1350
- },
1351
- "model.layers.48.mlp.gate_up_proj": {
1352
- "group_size": 16,
1353
- "quant_algo": "NVFP4"
1354
- },
1355
- "model.layers.48.mlp.up_proj": {
1356
- "group_size": 16,
1357
- "quant_algo": "NVFP4"
1358
- },
1359
- "model.layers.48.self_attn.g_proj": {
1360
- "quant_algo": "FP8"
1361
- },
1362
- "model.layers.48.self_attn.k_proj": {
1363
- "quant_algo": "FP8"
1364
- },
1365
- "model.layers.48.self_attn.o_proj": {
1366
- "quant_algo": "FP8"
1367
- },
1368
- "model.layers.48.self_attn.q_proj": {
1369
- "quant_algo": "FP8"
1370
- },
1371
- "model.layers.48.self_attn.v_proj": {
1372
- "quant_algo": "FP8"
1373
- },
1374
- "model.layers.49.mlp.down_proj": {
1375
- "group_size": 16,
1376
- "quant_algo": "NVFP4"
1377
- },
1378
- "model.layers.49.mlp.gate_proj": {
1379
- "group_size": 16,
1380
- "quant_algo": "NVFP4"
1381
- },
1382
- "model.layers.49.mlp.gate_up_proj": {
1383
- "group_size": 16,
1384
- "quant_algo": "NVFP4"
1385
- },
1386
- "model.layers.49.mlp.up_proj": {
1387
- "group_size": 16,
1388
- "quant_algo": "NVFP4"
1389
- },
1390
- "model.layers.49.self_attn.g_proj": {
1391
- "quant_algo": "FP8"
1392
- },
1393
- "model.layers.49.self_attn.k_proj": {
1394
- "quant_algo": "FP8"
1395
- },
1396
- "model.layers.49.self_attn.o_proj": {
1397
- "quant_algo": "FP8"
1398
- },
1399
- "model.layers.49.self_attn.q_proj": {
1400
- "quant_algo": "FP8"
1401
- },
1402
- "model.layers.49.self_attn.v_proj": {
1403
- "quant_algo": "FP8"
1404
- },
1405
- "model.layers.5.mlp.down_proj": {
1406
- "group_size": 16,
1407
- "quant_algo": "NVFP4"
1408
- },
1409
- "model.layers.5.mlp.gate_proj": {
1410
- "group_size": 16,
1411
- "quant_algo": "NVFP4"
1412
- },
1413
- "model.layers.5.mlp.gate_up_proj": {
1414
- "group_size": 16,
1415
- "quant_algo": "NVFP4"
1416
- },
1417
- "model.layers.5.mlp.up_proj": {
1418
- "group_size": 16,
1419
- "quant_algo": "NVFP4"
1420
- },
1421
- "model.layers.5.self_attn.g_proj": {
1422
- "quant_algo": "FP8"
1423
- },
1424
- "model.layers.5.self_attn.k_proj": {
1425
- "quant_algo": "FP8"
1426
- },
1427
- "model.layers.5.self_attn.o_proj": {
1428
- "quant_algo": "FP8"
1429
- },
1430
- "model.layers.5.self_attn.q_proj": {
1431
- "quant_algo": "FP8"
1432
- },
1433
- "model.layers.5.self_attn.v_proj": {
1434
- "quant_algo": "FP8"
1435
- },
1436
- "model.layers.50.mlp.down_proj": {
1437
- "group_size": 16,
1438
- "quant_algo": "NVFP4"
1439
- },
1440
- "model.layers.50.mlp.gate_proj": {
1441
- "group_size": 16,
1442
- "quant_algo": "NVFP4"
1443
- },
1444
- "model.layers.50.mlp.gate_up_proj": {
1445
- "group_size": 16,
1446
- "quant_algo": "NVFP4"
1447
- },
1448
- "model.layers.50.mlp.up_proj": {
1449
- "group_size": 16,
1450
- "quant_algo": "NVFP4"
1451
- },
1452
- "model.layers.50.self_attn.g_proj": {
1453
- "quant_algo": "FP8"
1454
- },
1455
- "model.layers.50.self_attn.k_proj": {
1456
- "quant_algo": "FP8"
1457
- },
1458
- "model.layers.50.self_attn.o_proj": {
1459
- "quant_algo": "FP8"
1460
- },
1461
- "model.layers.50.self_attn.q_proj": {
1462
- "quant_algo": "FP8"
1463
- },
1464
- "model.layers.50.self_attn.v_proj": {
1465
- "quant_algo": "FP8"
1466
- },
1467
- "model.layers.51.mlp.down_proj": {
1468
- "group_size": 16,
1469
- "quant_algo": "NVFP4"
1470
- },
1471
- "model.layers.51.mlp.gate_proj": {
1472
- "group_size": 16,
1473
- "quant_algo": "NVFP4"
1474
- },
1475
- "model.layers.51.mlp.gate_up_proj": {
1476
- "group_size": 16,
1477
- "quant_algo": "NVFP4"
1478
- },
1479
- "model.layers.51.mlp.up_proj": {
1480
- "group_size": 16,
1481
- "quant_algo": "NVFP4"
1482
- },
1483
- "model.layers.51.self_attn.g_proj": {
1484
- "quant_algo": "FP8"
1485
- },
1486
- "model.layers.51.self_attn.k_proj": {
1487
- "quant_algo": "FP8"
1488
- },
1489
- "model.layers.51.self_attn.o_proj": {
1490
- "quant_algo": "FP8"
1491
- },
1492
- "model.layers.51.self_attn.q_proj": {
1493
- "quant_algo": "FP8"
1494
- },
1495
- "model.layers.51.self_attn.v_proj": {
1496
- "quant_algo": "FP8"
1497
- },
1498
- "model.layers.52.mlp.down_proj": {
1499
- "group_size": 16,
1500
- "quant_algo": "NVFP4"
1501
- },
1502
- "model.layers.52.mlp.gate_proj": {
1503
- "group_size": 16,
1504
- "quant_algo": "NVFP4"
1505
- },
1506
- "model.layers.52.mlp.gate_up_proj": {
1507
- "group_size": 16,
1508
- "quant_algo": "NVFP4"
1509
- },
1510
- "model.layers.52.mlp.up_proj": {
1511
- "group_size": 16,
1512
- "quant_algo": "NVFP4"
1513
- },
1514
- "model.layers.52.self_attn.g_proj": {
1515
- "quant_algo": "FP8"
1516
- },
1517
- "model.layers.52.self_attn.k_proj": {
1518
- "quant_algo": "FP8"
1519
- },
1520
- "model.layers.52.self_attn.o_proj": {
1521
- "quant_algo": "FP8"
1522
- },
1523
- "model.layers.52.self_attn.q_proj": {
1524
- "quant_algo": "FP8"
1525
- },
1526
- "model.layers.52.self_attn.v_proj": {
1527
- "quant_algo": "FP8"
1528
- },
1529
- "model.layers.53.mlp.down_proj": {
1530
- "group_size": 16,
1531
- "quant_algo": "NVFP4"
1532
- },
1533
- "model.layers.53.mlp.gate_proj": {
1534
- "group_size": 16,
1535
- "quant_algo": "NVFP4"
1536
- },
1537
- "model.layers.53.mlp.gate_up_proj": {
1538
- "group_size": 16,
1539
- "quant_algo": "NVFP4"
1540
- },
1541
- "model.layers.53.mlp.up_proj": {
1542
- "group_size": 16,
1543
- "quant_algo": "NVFP4"
1544
- },
1545
- "model.layers.53.self_attn.g_proj": {
1546
- "quant_algo": "FP8"
1547
- },
1548
- "model.layers.53.self_attn.k_proj": {
1549
- "quant_algo": "FP8"
1550
- },
1551
- "model.layers.53.self_attn.o_proj": {
1552
- "quant_algo": "FP8"
1553
- },
1554
- "model.layers.53.self_attn.q_proj": {
1555
- "quant_algo": "FP8"
1556
- },
1557
- "model.layers.53.self_attn.v_proj": {
1558
- "quant_algo": "FP8"
1559
- },
1560
- "model.layers.54.mlp.down_proj": {
1561
- "group_size": 16,
1562
- "quant_algo": "NVFP4"
1563
- },
1564
- "model.layers.54.mlp.gate_proj": {
1565
- "group_size": 16,
1566
- "quant_algo": "NVFP4"
1567
- },
1568
- "model.layers.54.mlp.gate_up_proj": {
1569
- "group_size": 16,
1570
- "quant_algo": "NVFP4"
1571
- },
1572
- "model.layers.54.mlp.up_proj": {
1573
- "group_size": 16,
1574
- "quant_algo": "NVFP4"
1575
- },
1576
- "model.layers.54.self_attn.g_proj": {
1577
- "quant_algo": "FP8"
1578
- },
1579
- "model.layers.54.self_attn.k_proj": {
1580
- "quant_algo": "FP8"
1581
- },
1582
- "model.layers.54.self_attn.o_proj": {
1583
- "quant_algo": "FP8"
1584
- },
1585
- "model.layers.54.self_attn.q_proj": {
1586
- "quant_algo": "FP8"
1587
- },
1588
- "model.layers.54.self_attn.v_proj": {
1589
- "quant_algo": "FP8"
1590
- },
1591
- "model.layers.55.mlp.down_proj": {
1592
- "group_size": 16,
1593
- "quant_algo": "NVFP4"
1594
- },
1595
- "model.layers.55.mlp.gate_proj": {
1596
- "group_size": 16,
1597
- "quant_algo": "NVFP4"
1598
- },
1599
- "model.layers.55.mlp.gate_up_proj": {
1600
- "group_size": 16,
1601
- "quant_algo": "NVFP4"
1602
- },
1603
- "model.layers.55.mlp.up_proj": {
1604
- "group_size": 16,
1605
- "quant_algo": "NVFP4"
1606
- },
1607
- "model.layers.55.self_attn.g_proj": {
1608
- "quant_algo": "FP8"
1609
- },
1610
- "model.layers.55.self_attn.k_proj": {
1611
- "quant_algo": "FP8"
1612
- },
1613
- "model.layers.55.self_attn.o_proj": {
1614
- "quant_algo": "FP8"
1615
- },
1616
- "model.layers.55.self_attn.q_proj": {
1617
- "quant_algo": "FP8"
1618
- },
1619
- "model.layers.55.self_attn.v_proj": {
1620
- "quant_algo": "FP8"
1621
- },
1622
- "model.layers.56.mlp.down_proj": {
1623
- "group_size": 16,
1624
- "quant_algo": "NVFP4"
1625
- },
1626
- "model.layers.56.mlp.gate_proj": {
1627
- "group_size": 16,
1628
- "quant_algo": "NVFP4"
1629
- },
1630
- "model.layers.56.mlp.gate_up_proj": {
1631
- "group_size": 16,
1632
- "quant_algo": "NVFP4"
1633
- },
1634
- "model.layers.56.mlp.up_proj": {
1635
- "group_size": 16,
1636
- "quant_algo": "NVFP4"
1637
- },
1638
- "model.layers.56.self_attn.g_proj": {
1639
- "quant_algo": "FP8"
1640
- },
1641
- "model.layers.56.self_attn.k_proj": {
1642
- "quant_algo": "FP8"
1643
- },
1644
- "model.layers.56.self_attn.o_proj": {
1645
- "quant_algo": "FP8"
1646
- },
1647
- "model.layers.56.self_attn.q_proj": {
1648
- "quant_algo": "FP8"
1649
- },
1650
- "model.layers.56.self_attn.v_proj": {
1651
- "quant_algo": "FP8"
1652
- },
1653
- "model.layers.57.mlp.down_proj": {
1654
- "group_size": 16,
1655
- "quant_algo": "NVFP4"
1656
- },
1657
- "model.layers.57.mlp.gate_proj": {
1658
- "group_size": 16,
1659
- "quant_algo": "NVFP4"
1660
- },
1661
- "model.layers.57.mlp.gate_up_proj": {
1662
- "group_size": 16,
1663
- "quant_algo": "NVFP4"
1664
- },
1665
- "model.layers.57.mlp.up_proj": {
1666
- "group_size": 16,
1667
- "quant_algo": "NVFP4"
1668
- },
1669
- "model.layers.57.self_attn.g_proj": {
1670
- "quant_algo": "FP8"
1671
- },
1672
- "model.layers.57.self_attn.k_proj": {
1673
- "quant_algo": "FP8"
1674
- },
1675
- "model.layers.57.self_attn.o_proj": {
1676
- "quant_algo": "FP8"
1677
- },
1678
- "model.layers.57.self_attn.q_proj": {
1679
- "quant_algo": "FP8"
1680
- },
1681
- "model.layers.57.self_attn.v_proj": {
1682
- "quant_algo": "FP8"
1683
- },
1684
- "model.layers.58.mlp.down_proj": {
1685
- "group_size": 16,
1686
- "quant_algo": "NVFP4"
1687
- },
1688
- "model.layers.58.mlp.gate_proj": {
1689
- "group_size": 16,
1690
- "quant_algo": "NVFP4"
1691
- },
1692
- "model.layers.58.mlp.gate_up_proj": {
1693
- "group_size": 16,
1694
- "quant_algo": "NVFP4"
1695
- },
1696
- "model.layers.58.mlp.up_proj": {
1697
- "group_size": 16,
1698
- "quant_algo": "NVFP4"
1699
- },
1700
- "model.layers.58.self_attn.g_proj": {
1701
- "quant_algo": "FP8"
1702
- },
1703
- "model.layers.58.self_attn.k_proj": {
1704
- "quant_algo": "FP8"
1705
- },
1706
- "model.layers.58.self_attn.o_proj": {
1707
- "quant_algo": "FP8"
1708
- },
1709
- "model.layers.58.self_attn.q_proj": {
1710
- "quant_algo": "FP8"
1711
- },
1712
- "model.layers.58.self_attn.v_proj": {
1713
- "quant_algo": "FP8"
1714
- },
1715
- "model.layers.59.mlp.down_proj": {
1716
- "group_size": 16,
1717
- "quant_algo": "NVFP4"
1718
- },
1719
- "model.layers.59.mlp.gate_proj": {
1720
- "group_size": 16,
1721
- "quant_algo": "NVFP4"
1722
- },
1723
- "model.layers.59.mlp.gate_up_proj": {
1724
- "group_size": 16,
1725
- "quant_algo": "NVFP4"
1726
- },
1727
- "model.layers.59.mlp.up_proj": {
1728
- "group_size": 16,
1729
- "quant_algo": "NVFP4"
1730
- },
1731
- "model.layers.59.self_attn.g_proj": {
1732
- "quant_algo": "FP8"
1733
- },
1734
- "model.layers.59.self_attn.k_proj": {
1735
- "quant_algo": "FP8"
1736
- },
1737
- "model.layers.59.self_attn.o_proj": {
1738
- "quant_algo": "FP8"
1739
- },
1740
- "model.layers.59.self_attn.q_proj": {
1741
- "quant_algo": "FP8"
1742
- },
1743
- "model.layers.59.self_attn.v_proj": {
1744
- "quant_algo": "FP8"
1745
- },
1746
- "model.layers.6.mlp.down_proj": {
1747
- "group_size": 16,
1748
- "quant_algo": "NVFP4"
1749
- },
1750
- "model.layers.6.mlp.gate_proj": {
1751
- "group_size": 16,
1752
- "quant_algo": "NVFP4"
1753
- },
1754
- "model.layers.6.mlp.gate_up_proj": {
1755
- "group_size": 16,
1756
- "quant_algo": "NVFP4"
1757
- },
1758
- "model.layers.6.mlp.up_proj": {
1759
- "group_size": 16,
1760
- "quant_algo": "NVFP4"
1761
- },
1762
- "model.layers.6.self_attn.g_proj": {
1763
- "quant_algo": "FP8"
1764
- },
1765
- "model.layers.6.self_attn.k_proj": {
1766
- "quant_algo": "FP8"
1767
- },
1768
- "model.layers.6.self_attn.o_proj": {
1769
- "quant_algo": "FP8"
1770
- },
1771
- "model.layers.6.self_attn.q_proj": {
1772
- "quant_algo": "FP8"
1773
- },
1774
- "model.layers.6.self_attn.v_proj": {
1775
- "quant_algo": "FP8"
1776
- },
1777
- "model.layers.60.mlp.down_proj": {
1778
- "group_size": 16,
1779
- "quant_algo": "NVFP4"
1780
- },
1781
- "model.layers.60.mlp.gate_proj": {
1782
- "group_size": 16,
1783
- "quant_algo": "NVFP4"
1784
- },
1785
- "model.layers.60.mlp.gate_up_proj": {
1786
- "group_size": 16,
1787
- "quant_algo": "NVFP4"
1788
- },
1789
- "model.layers.60.mlp.up_proj": {
1790
- "group_size": 16,
1791
- "quant_algo": "NVFP4"
1792
- },
1793
- "model.layers.60.self_attn.g_proj": {
1794
- "quant_algo": "FP8"
1795
- },
1796
- "model.layers.60.self_attn.k_proj": {
1797
- "quant_algo": "FP8"
1798
- },
1799
- "model.layers.60.self_attn.o_proj": {
1800
- "quant_algo": "FP8"
1801
- },
1802
- "model.layers.60.self_attn.q_proj": {
1803
- "quant_algo": "FP8"
1804
- },
1805
- "model.layers.60.self_attn.v_proj": {
1806
- "quant_algo": "FP8"
1807
- },
1808
- "model.layers.61.mlp.down_proj": {
1809
- "group_size": 16,
1810
- "quant_algo": "NVFP4"
1811
- },
1812
- "model.layers.61.mlp.gate_proj": {
1813
- "group_size": 16,
1814
- "quant_algo": "NVFP4"
1815
- },
1816
- "model.layers.61.mlp.gate_up_proj": {
1817
- "group_size": 16,
1818
- "quant_algo": "NVFP4"
1819
- },
1820
- "model.layers.61.mlp.up_proj": {
1821
- "group_size": 16,
1822
- "quant_algo": "NVFP4"
1823
- },
1824
- "model.layers.61.self_attn.g_proj": {
1825
- "quant_algo": "FP8"
1826
- },
1827
- "model.layers.61.self_attn.k_proj": {
1828
- "quant_algo": "FP8"
1829
- },
1830
- "model.layers.61.self_attn.o_proj": {
1831
- "quant_algo": "FP8"
1832
- },
1833
- "model.layers.61.self_attn.q_proj": {
1834
- "quant_algo": "FP8"
1835
- },
1836
- "model.layers.61.self_attn.v_proj": {
1837
- "quant_algo": "FP8"
1838
- },
1839
- "model.layers.62.mlp.down_proj": {
1840
- "group_size": 16,
1841
- "quant_algo": "NVFP4"
1842
- },
1843
- "model.layers.62.mlp.gate_proj": {
1844
- "group_size": 16,
1845
- "quant_algo": "NVFP4"
1846
- },
1847
- "model.layers.62.mlp.gate_up_proj": {
1848
- "group_size": 16,
1849
- "quant_algo": "NVFP4"
1850
- },
1851
- "model.layers.62.mlp.up_proj": {
1852
- "group_size": 16,
1853
- "quant_algo": "NVFP4"
1854
- },
1855
- "model.layers.62.self_attn.g_proj": {
1856
- "quant_algo": "FP8"
1857
- },
1858
- "model.layers.62.self_attn.k_proj": {
1859
- "quant_algo": "FP8"
1860
- },
1861
- "model.layers.62.self_attn.o_proj": {
1862
- "quant_algo": "FP8"
1863
- },
1864
- "model.layers.62.self_attn.q_proj": {
1865
- "quant_algo": "FP8"
1866
- },
1867
- "model.layers.62.self_attn.v_proj": {
1868
- "quant_algo": "FP8"
1869
- },
1870
- "model.layers.63.mlp.down_proj": {
1871
- "group_size": 16,
1872
- "quant_algo": "NVFP4"
1873
- },
1874
- "model.layers.63.mlp.gate_proj": {
1875
- "group_size": 16,
1876
- "quant_algo": "NVFP4"
1877
- },
1878
- "model.layers.63.mlp.gate_up_proj": {
1879
- "group_size": 16,
1880
- "quant_algo": "NVFP4"
1881
- },
1882
- "model.layers.63.mlp.up_proj": {
1883
- "group_size": 16,
1884
- "quant_algo": "NVFP4"
1885
- },
1886
- "model.layers.63.self_attn.g_proj": {
1887
- "quant_algo": "FP8"
1888
- },
1889
- "model.layers.63.self_attn.k_proj": {
1890
- "quant_algo": "FP8"
1891
- },
1892
- "model.layers.63.self_attn.o_proj": {
1893
- "quant_algo": "FP8"
1894
- },
1895
- "model.layers.63.self_attn.q_proj": {
1896
- "quant_algo": "FP8"
1897
- },
1898
- "model.layers.63.self_attn.v_proj": {
1899
- "quant_algo": "FP8"
1900
- },
1901
- "model.layers.64.mlp.down_proj": {
1902
- "group_size": 16,
1903
- "quant_algo": "NVFP4"
1904
- },
1905
- "model.layers.64.mlp.gate_proj": {
1906
- "group_size": 16,
1907
- "quant_algo": "NVFP4"
1908
- },
1909
- "model.layers.64.mlp.gate_up_proj": {
1910
- "group_size": 16,
1911
- "quant_algo": "NVFP4"
1912
- },
1913
- "model.layers.64.mlp.up_proj": {
1914
- "group_size": 16,
1915
- "quant_algo": "NVFP4"
1916
- },
1917
- "model.layers.64.self_attn.g_proj": {
1918
- "quant_algo": "FP8"
1919
- },
1920
- "model.layers.64.self_attn.k_proj": {
1921
- "quant_algo": "FP8"
1922
- },
1923
- "model.layers.64.self_attn.o_proj": {
1924
- "quant_algo": "FP8"
1925
- },
1926
- "model.layers.64.self_attn.q_proj": {
1927
- "quant_algo": "FP8"
1928
- },
1929
- "model.layers.64.self_attn.v_proj": {
1930
- "quant_algo": "FP8"
1931
- },
1932
- "model.layers.65.mlp.down_proj": {
1933
- "group_size": 16,
1934
- "quant_algo": "NVFP4"
1935
- },
1936
- "model.layers.65.mlp.gate_proj": {
1937
- "group_size": 16,
1938
- "quant_algo": "NVFP4"
1939
- },
1940
- "model.layers.65.mlp.gate_up_proj": {
1941
- "group_size": 16,
1942
- "quant_algo": "NVFP4"
1943
- },
1944
- "model.layers.65.mlp.up_proj": {
1945
- "group_size": 16,
1946
- "quant_algo": "NVFP4"
1947
- },
1948
- "model.layers.65.self_attn.g_proj": {
1949
- "quant_algo": "FP8"
1950
- },
1951
- "model.layers.65.self_attn.k_proj": {
1952
- "quant_algo": "FP8"
1953
- },
1954
- "model.layers.65.self_attn.o_proj": {
1955
- "quant_algo": "FP8"
1956
- },
1957
- "model.layers.65.self_attn.q_proj": {
1958
- "quant_algo": "FP8"
1959
- },
1960
- "model.layers.65.self_attn.v_proj": {
1961
- "quant_algo": "FP8"
1962
- },
1963
- "model.layers.66.mlp.down_proj": {
1964
- "group_size": 16,
1965
- "quant_algo": "NVFP4"
1966
- },
1967
- "model.layers.66.mlp.gate_proj": {
1968
- "group_size": 16,
1969
- "quant_algo": "NVFP4"
1970
- },
1971
- "model.layers.66.mlp.gate_up_proj": {
1972
- "group_size": 16,
1973
- "quant_algo": "NVFP4"
1974
- },
1975
- "model.layers.66.mlp.up_proj": {
1976
- "group_size": 16,
1977
- "quant_algo": "NVFP4"
1978
- },
1979
- "model.layers.66.self_attn.g_proj": {
1980
- "quant_algo": "FP8"
1981
- },
1982
- "model.layers.66.self_attn.k_proj": {
1983
- "quant_algo": "FP8"
1984
- },
1985
- "model.layers.66.self_attn.o_proj": {
1986
- "quant_algo": "FP8"
1987
- },
1988
- "model.layers.66.self_attn.q_proj": {
1989
- "quant_algo": "FP8"
1990
- },
1991
- "model.layers.66.self_attn.v_proj": {
1992
- "quant_algo": "FP8"
1993
- },
1994
- "model.layers.67.mlp.down_proj": {
1995
- "group_size": 16,
1996
- "quant_algo": "NVFP4"
1997
- },
1998
- "model.layers.67.mlp.gate_proj": {
1999
- "group_size": 16,
2000
- "quant_algo": "NVFP4"
2001
- },
2002
- "model.layers.67.mlp.gate_up_proj": {
2003
- "group_size": 16,
2004
- "quant_algo": "NVFP4"
2005
- },
2006
- "model.layers.67.mlp.up_proj": {
2007
- "group_size": 16,
2008
- "quant_algo": "NVFP4"
2009
- },
2010
- "model.layers.67.self_attn.g_proj": {
2011
- "quant_algo": "FP8"
2012
- },
2013
- "model.layers.67.self_attn.k_proj": {
2014
- "quant_algo": "FP8"
2015
- },
2016
- "model.layers.67.self_attn.o_proj": {
2017
- "quant_algo": "FP8"
2018
- },
2019
- "model.layers.67.self_attn.q_proj": {
2020
- "quant_algo": "FP8"
2021
- },
2022
- "model.layers.67.self_attn.v_proj": {
2023
- "quant_algo": "FP8"
2024
- },
2025
- "model.layers.68.mlp.down_proj": {
2026
- "group_size": 16,
2027
- "quant_algo": "NVFP4"
2028
- },
2029
- "model.layers.68.mlp.gate_proj": {
2030
- "group_size": 16,
2031
- "quant_algo": "NVFP4"
2032
- },
2033
- "model.layers.68.mlp.gate_up_proj": {
2034
- "group_size": 16,
2035
- "quant_algo": "NVFP4"
2036
- },
2037
- "model.layers.68.mlp.up_proj": {
2038
- "group_size": 16,
2039
- "quant_algo": "NVFP4"
2040
- },
2041
- "model.layers.68.self_attn.g_proj": {
2042
- "quant_algo": "FP8"
2043
- },
2044
- "model.layers.68.self_attn.k_proj": {
2045
- "quant_algo": "FP8"
2046
- },
2047
- "model.layers.68.self_attn.o_proj": {
2048
- "quant_algo": "FP8"
2049
- },
2050
- "model.layers.68.self_attn.q_proj": {
2051
- "quant_algo": "FP8"
2052
- },
2053
- "model.layers.68.self_attn.v_proj": {
2054
- "quant_algo": "FP8"
2055
- },
2056
- "model.layers.69.mlp.down_proj": {
2057
- "group_size": 16,
2058
- "quant_algo": "NVFP4"
2059
- },
2060
- "model.layers.69.mlp.gate_proj": {
2061
- "group_size": 16,
2062
- "quant_algo": "NVFP4"
2063
- },
2064
- "model.layers.69.mlp.gate_up_proj": {
2065
- "group_size": 16,
2066
- "quant_algo": "NVFP4"
2067
- },
2068
- "model.layers.69.mlp.up_proj": {
2069
- "group_size": 16,
2070
- "quant_algo": "NVFP4"
2071
- },
2072
- "model.layers.69.self_attn.g_proj": {
2073
- "quant_algo": "FP8"
2074
- },
2075
- "model.layers.69.self_attn.k_proj": {
2076
- "quant_algo": "FP8"
2077
- },
2078
- "model.layers.69.self_attn.o_proj": {
2079
- "quant_algo": "FP8"
2080
- },
2081
- "model.layers.69.self_attn.q_proj": {
2082
- "quant_algo": "FP8"
2083
- },
2084
- "model.layers.69.self_attn.v_proj": {
2085
- "quant_algo": "FP8"
2086
- },
2087
- "model.layers.7.mlp.down_proj": {
2088
- "group_size": 16,
2089
- "quant_algo": "NVFP4"
2090
- },
2091
- "model.layers.7.mlp.gate_proj": {
2092
- "group_size": 16,
2093
- "quant_algo": "NVFP4"
2094
- },
2095
- "model.layers.7.mlp.gate_up_proj": {
2096
- "group_size": 16,
2097
- "quant_algo": "NVFP4"
2098
- },
2099
- "model.layers.7.mlp.up_proj": {
2100
- "group_size": 16,
2101
- "quant_algo": "NVFP4"
2102
- },
2103
- "model.layers.7.self_attn.g_proj": {
2104
- "quant_algo": "FP8"
2105
- },
2106
- "model.layers.7.self_attn.k_proj": {
2107
- "quant_algo": "FP8"
2108
- },
2109
- "model.layers.7.self_attn.o_proj": {
2110
- "quant_algo": "FP8"
2111
- },
2112
- "model.layers.7.self_attn.q_proj": {
2113
- "quant_algo": "FP8"
2114
- },
2115
- "model.layers.7.self_attn.v_proj": {
2116
- "quant_algo": "FP8"
2117
- },
2118
- "model.layers.70.mlp.down_proj": {
2119
- "group_size": 16,
2120
- "quant_algo": "NVFP4"
2121
- },
2122
- "model.layers.70.mlp.gate_proj": {
2123
- "group_size": 16,
2124
- "quant_algo": "NVFP4"
2125
- },
2126
- "model.layers.70.mlp.gate_up_proj": {
2127
- "group_size": 16,
2128
- "quant_algo": "NVFP4"
2129
- },
2130
- "model.layers.70.mlp.up_proj": {
2131
- "group_size": 16,
2132
- "quant_algo": "NVFP4"
2133
- },
2134
- "model.layers.70.self_attn.g_proj": {
2135
- "quant_algo": "FP8"
2136
- },
2137
- "model.layers.70.self_attn.k_proj": {
2138
- "quant_algo": "FP8"
2139
- },
2140
- "model.layers.70.self_attn.o_proj": {
2141
- "quant_algo": "FP8"
2142
- },
2143
- "model.layers.70.self_attn.q_proj": {
2144
- "quant_algo": "FP8"
2145
- },
2146
- "model.layers.70.self_attn.v_proj": {
2147
- "quant_algo": "FP8"
2148
- },
2149
- "model.layers.71.mlp.down_proj": {
2150
- "group_size": 16,
2151
- "quant_algo": "NVFP4"
2152
- },
2153
- "model.layers.71.mlp.gate_proj": {
2154
- "group_size": 16,
2155
- "quant_algo": "NVFP4"
2156
- },
2157
- "model.layers.71.mlp.gate_up_proj": {
2158
- "group_size": 16,
2159
- "quant_algo": "NVFP4"
2160
- },
2161
- "model.layers.71.mlp.up_proj": {
2162
- "group_size": 16,
2163
- "quant_algo": "NVFP4"
2164
- },
2165
- "model.layers.71.self_attn.g_proj": {
2166
- "quant_algo": "FP8"
2167
- },
2168
- "model.layers.71.self_attn.k_proj": {
2169
- "quant_algo": "FP8"
2170
- },
2171
- "model.layers.71.self_attn.o_proj": {
2172
- "quant_algo": "FP8"
2173
- },
2174
- "model.layers.71.self_attn.q_proj": {
2175
- "quant_algo": "FP8"
2176
- },
2177
- "model.layers.71.self_attn.v_proj": {
2178
- "quant_algo": "FP8"
2179
- },
2180
- "model.layers.72.mlp.down_proj": {
2181
- "group_size": 16,
2182
- "quant_algo": "NVFP4"
2183
- },
2184
- "model.layers.72.mlp.gate_proj": {
2185
- "group_size": 16,
2186
- "quant_algo": "NVFP4"
2187
- },
2188
- "model.layers.72.mlp.gate_up_proj": {
2189
- "group_size": 16,
2190
- "quant_algo": "NVFP4"
2191
- },
2192
- "model.layers.72.mlp.up_proj": {
2193
- "group_size": 16,
2194
- "quant_algo": "NVFP4"
2195
- },
2196
- "model.layers.72.self_attn.g_proj": {
2197
- "quant_algo": "FP8"
2198
- },
2199
- "model.layers.72.self_attn.k_proj": {
2200
- "quant_algo": "FP8"
2201
- },
2202
- "model.layers.72.self_attn.o_proj": {
2203
- "quant_algo": "FP8"
2204
- },
2205
- "model.layers.72.self_attn.q_proj": {
2206
- "quant_algo": "FP8"
2207
- },
2208
- "model.layers.72.self_attn.v_proj": {
2209
- "quant_algo": "FP8"
2210
- },
2211
- "model.layers.73.mlp.down_proj": {
2212
- "group_size": 16,
2213
- "quant_algo": "NVFP4"
2214
- },
2215
- "model.layers.73.mlp.gate_proj": {
2216
- "group_size": 16,
2217
- "quant_algo": "NVFP4"
2218
- },
2219
- "model.layers.73.mlp.gate_up_proj": {
2220
- "group_size": 16,
2221
- "quant_algo": "NVFP4"
2222
- },
2223
- "model.layers.73.mlp.up_proj": {
2224
- "group_size": 16,
2225
- "quant_algo": "NVFP4"
2226
- },
2227
- "model.layers.73.self_attn.g_proj": {
2228
- "quant_algo": "FP8"
2229
- },
2230
- "model.layers.73.self_attn.k_proj": {
2231
- "quant_algo": "FP8"
2232
- },
2233
- "model.layers.73.self_attn.o_proj": {
2234
- "quant_algo": "FP8"
2235
- },
2236
- "model.layers.73.self_attn.q_proj": {
2237
- "quant_algo": "FP8"
2238
- },
2239
- "model.layers.73.self_attn.v_proj": {
2240
- "quant_algo": "FP8"
2241
- },
2242
- "model.layers.74.mlp.down_proj": {
2243
- "group_size": 16,
2244
- "quant_algo": "NVFP4"
2245
- },
2246
- "model.layers.74.mlp.gate_proj": {
2247
- "group_size": 16,
2248
- "quant_algo": "NVFP4"
2249
- },
2250
- "model.layers.74.mlp.gate_up_proj": {
2251
- "group_size": 16,
2252
- "quant_algo": "NVFP4"
2253
- },
2254
- "model.layers.74.mlp.up_proj": {
2255
- "group_size": 16,
2256
- "quant_algo": "NVFP4"
2257
- },
2258
- "model.layers.74.self_attn.g_proj": {
2259
- "quant_algo": "FP8"
2260
- },
2261
- "model.layers.74.self_attn.k_proj": {
2262
- "quant_algo": "FP8"
2263
- },
2264
- "model.layers.74.self_attn.o_proj": {
2265
- "quant_algo": "FP8"
2266
- },
2267
- "model.layers.74.self_attn.q_proj": {
2268
- "quant_algo": "FP8"
2269
- },
2270
- "model.layers.74.self_attn.v_proj": {
2271
- "quant_algo": "FP8"
2272
- },
2273
- "model.layers.75.mlp.down_proj": {
2274
- "group_size": 16,
2275
- "quant_algo": "NVFP4"
2276
- },
2277
- "model.layers.75.mlp.gate_proj": {
2278
- "group_size": 16,
2279
- "quant_algo": "NVFP4"
2280
- },
2281
- "model.layers.75.mlp.gate_up_proj": {
2282
- "group_size": 16,
2283
- "quant_algo": "NVFP4"
2284
- },
2285
- "model.layers.75.mlp.up_proj": {
2286
- "group_size": 16,
2287
- "quant_algo": "NVFP4"
2288
- },
2289
- "model.layers.75.self_attn.g_proj": {
2290
- "quant_algo": "FP8"
2291
- },
2292
- "model.layers.75.self_attn.k_proj": {
2293
- "quant_algo": "FP8"
2294
- },
2295
- "model.layers.75.self_attn.o_proj": {
2296
- "quant_algo": "FP8"
2297
- },
2298
- "model.layers.75.self_attn.q_proj": {
2299
- "quant_algo": "FP8"
2300
- },
2301
- "model.layers.75.self_attn.v_proj": {
2302
- "quant_algo": "FP8"
2303
- },
2304
- "model.layers.76.mlp.down_proj": {
2305
- "group_size": 16,
2306
- "quant_algo": "NVFP4"
2307
- },
2308
- "model.layers.76.mlp.gate_proj": {
2309
- "group_size": 16,
2310
- "quant_algo": "NVFP4"
2311
- },
2312
- "model.layers.76.mlp.gate_up_proj": {
2313
- "group_size": 16,
2314
- "quant_algo": "NVFP4"
2315
- },
2316
- "model.layers.76.mlp.up_proj": {
2317
- "group_size": 16,
2318
- "quant_algo": "NVFP4"
2319
- },
2320
- "model.layers.76.self_attn.g_proj": {
2321
- "quant_algo": "FP8"
2322
- },
2323
- "model.layers.76.self_attn.k_proj": {
2324
- "quant_algo": "FP8"
2325
- },
2326
- "model.layers.76.self_attn.o_proj": {
2327
- "quant_algo": "FP8"
2328
- },
2329
- "model.layers.76.self_attn.q_proj": {
2330
- "quant_algo": "FP8"
2331
- },
2332
- "model.layers.76.self_attn.v_proj": {
2333
- "quant_algo": "FP8"
2334
- },
2335
- "model.layers.77.mlp.down_proj": {
2336
- "group_size": 16,
2337
- "quant_algo": "NVFP4"
2338
- },
2339
- "model.layers.77.mlp.gate_proj": {
2340
- "group_size": 16,
2341
- "quant_algo": "NVFP4"
2342
- },
2343
- "model.layers.77.mlp.gate_up_proj": {
2344
- "group_size": 16,
2345
- "quant_algo": "NVFP4"
2346
- },
2347
- "model.layers.77.mlp.up_proj": {
2348
- "group_size": 16,
2349
- "quant_algo": "NVFP4"
2350
- },
2351
- "model.layers.77.self_attn.g_proj": {
2352
- "quant_algo": "FP8"
2353
- },
2354
- "model.layers.77.self_attn.k_proj": {
2355
- "quant_algo": "FP8"
2356
- },
2357
- "model.layers.77.self_attn.o_proj": {
2358
- "quant_algo": "FP8"
2359
- },
2360
- "model.layers.77.self_attn.q_proj": {
2361
- "quant_algo": "FP8"
2362
- },
2363
- "model.layers.77.self_attn.v_proj": {
2364
- "quant_algo": "FP8"
2365
- },
2366
- "model.layers.78.mlp.down_proj": {
2367
- "group_size": 16,
2368
- "quant_algo": "NVFP4"
2369
- },
2370
- "model.layers.78.mlp.gate_proj": {
2371
- "group_size": 16,
2372
- "quant_algo": "NVFP4"
2373
- },
2374
- "model.layers.78.mlp.gate_up_proj": {
2375
- "group_size": 16,
2376
- "quant_algo": "NVFP4"
2377
- },
2378
- "model.layers.78.mlp.up_proj": {
2379
- "group_size": 16,
2380
- "quant_algo": "NVFP4"
2381
- },
2382
- "model.layers.78.self_attn.g_proj": {
2383
- "quant_algo": "FP8"
2384
- },
2385
- "model.layers.78.self_attn.k_proj": {
2386
- "quant_algo": "FP8"
2387
- },
2388
- "model.layers.78.self_attn.o_proj": {
2389
- "quant_algo": "FP8"
2390
- },
2391
- "model.layers.78.self_attn.q_proj": {
2392
- "quant_algo": "FP8"
2393
- },
2394
- "model.layers.78.self_attn.v_proj": {
2395
- "quant_algo": "FP8"
2396
- },
2397
- "model.layers.79.mlp.down_proj": {
2398
- "group_size": 16,
2399
- "quant_algo": "NVFP4"
2400
- },
2401
- "model.layers.79.mlp.gate_proj": {
2402
- "group_size": 16,
2403
- "quant_algo": "NVFP4"
2404
- },
2405
- "model.layers.79.mlp.gate_up_proj": {
2406
- "group_size": 16,
2407
- "quant_algo": "NVFP4"
2408
- },
2409
- "model.layers.79.mlp.up_proj": {
2410
- "group_size": 16,
2411
- "quant_algo": "NVFP4"
2412
- },
2413
- "model.layers.79.self_attn.g_proj": {
2414
- "quant_algo": "FP8"
2415
- },
2416
- "model.layers.79.self_attn.k_proj": {
2417
- "quant_algo": "FP8"
2418
- },
2419
- "model.layers.79.self_attn.o_proj": {
2420
- "quant_algo": "FP8"
2421
- },
2422
- "model.layers.79.self_attn.q_proj": {
2423
- "quant_algo": "FP8"
2424
- },
2425
- "model.layers.79.self_attn.v_proj": {
2426
- "quant_algo": "FP8"
2427
- },
2428
- "model.layers.8.mlp.down_proj": {
2429
- "group_size": 16,
2430
- "quant_algo": "NVFP4"
2431
- },
2432
- "model.layers.8.mlp.gate_proj": {
2433
- "group_size": 16,
2434
- "quant_algo": "NVFP4"
2435
- },
2436
- "model.layers.8.mlp.gate_up_proj": {
2437
- "group_size": 16,
2438
- "quant_algo": "NVFP4"
2439
- },
2440
- "model.layers.8.mlp.up_proj": {
2441
- "group_size": 16,
2442
- "quant_algo": "NVFP4"
2443
- },
2444
- "model.layers.8.self_attn.g_proj": {
2445
- "quant_algo": "FP8"
2446
- },
2447
- "model.layers.8.self_attn.k_proj": {
2448
- "quant_algo": "FP8"
2449
- },
2450
- "model.layers.8.self_attn.o_proj": {
2451
- "quant_algo": "FP8"
2452
- },
2453
- "model.layers.8.self_attn.q_proj": {
2454
- "quant_algo": "FP8"
2455
- },
2456
- "model.layers.8.self_attn.v_proj": {
2457
- "quant_algo": "FP8"
2458
- },
2459
- "model.layers.9.mlp.down_proj": {
2460
- "group_size": 16,
2461
- "quant_algo": "NVFP4"
2462
- },
2463
- "model.layers.9.mlp.gate_proj": {
2464
- "group_size": 16,
2465
- "quant_algo": "NVFP4"
2466
- },
2467
- "model.layers.9.mlp.gate_up_proj": {
2468
- "group_size": 16,
2469
- "quant_algo": "NVFP4"
2470
- },
2471
- "model.layers.9.mlp.up_proj": {
2472
- "group_size": 16,
2473
- "quant_algo": "NVFP4"
2474
- },
2475
- "model.layers.9.self_attn.g_proj": {
2476
- "quant_algo": "FP8"
2477
- },
2478
- "model.layers.9.self_attn.k_proj": {
2479
- "quant_algo": "FP8"
2480
- },
2481
- "model.layers.9.self_attn.o_proj": {
2482
- "quant_algo": "FP8"
2483
- },
2484
- "model.layers.9.self_attn.q_proj": {
2485
- "quant_algo": "FP8"
2486
- },
2487
- "model.layers.9.self_attn.v_proj": {
2488
- "quant_algo": "FP8"
2489
- }
2490
  }
2491
- }
2492
- }
 
1
  {
2
+ "producer": {
3
+ "name": "modelopt",
4
+ "version": "0.46.0"
5
+ },
6
+ "quantization": {
7
+ "quant_algo": "NVFP4",
8
+ "kv_cache_quant_algo": null,
9
+ "group_size": 16,
10
+ "exclude_modules": [
11
+ "lm_head",
12
+ "model.embed_tokens",
13
+ "model.layers.0.self_attn*",
14
+ "model.layers.1.self_attn*",
15
+ "model.layers.10.self_attn*",
16
+ "model.layers.11.self_attn*",
17
+ "model.layers.12.self_attn*",
18
+ "model.layers.13.self_attn*",
19
+ "model.layers.14.self_attn*",
20
+ "model.layers.15.self_attn*",
21
+ "model.layers.16.self_attn*",
22
+ "model.layers.17.self_attn*",
23
+ "model.layers.18.self_attn*",
24
+ "model.layers.19.self_attn*",
25
+ "model.layers.2.self_attn*",
26
+ "model.layers.20.self_attn*",
27
+ "model.layers.21.self_attn*",
28
+ "model.layers.22.self_attn*",
29
+ "model.layers.23.self_attn*",
30
+ "model.layers.24.self_attn*",
31
+ "model.layers.25.self_attn*",
32
+ "model.layers.26.self_attn*",
33
+ "model.layers.27.self_attn*",
34
+ "model.layers.28.self_attn*",
35
+ "model.layers.29.self_attn*",
36
+ "model.layers.3.self_attn*",
37
+ "model.layers.30.self_attn*",
38
+ "model.layers.31.self_attn*",
39
+ "model.layers.32.self_attn*",
40
+ "model.layers.33.self_attn*",
41
+ "model.layers.34.self_attn*",
42
+ "model.layers.35.self_attn*",
43
+ "model.layers.36.self_attn*",
44
+ "model.layers.37.self_attn*",
45
+ "model.layers.38.self_attn*",
46
+ "model.layers.39.self_attn*",
47
+ "model.layers.4.self_attn*",
48
+ "model.layers.40.self_attn*",
49
+ "model.layers.41.self_attn*",
50
+ "model.layers.42.self_attn*",
51
+ "model.layers.43.self_attn*",
52
+ "model.layers.44.self_attn*",
53
+ "model.layers.45.self_attn*",
54
+ "model.layers.46.self_attn*",
55
+ "model.layers.47.self_attn*",
56
+ "model.layers.48.self_attn*",
57
+ "model.layers.49.self_attn*",
58
+ "model.layers.5.self_attn*",
59
+ "model.layers.50.self_attn*",
60
+ "model.layers.51.self_attn*",
61
+ "model.layers.52.self_attn*",
62
+ "model.layers.53.self_attn*",
63
+ "model.layers.54.self_attn*",
64
+ "model.layers.55.self_attn*",
65
+ "model.layers.56.self_attn*",
66
+ "model.layers.57.self_attn*",
67
+ "model.layers.58.self_attn*",
68
+ "model.layers.59.self_attn*",
69
+ "model.layers.6.self_attn*",
70
+ "model.layers.60.self_attn*",
71
+ "model.layers.61.self_attn*",
72
+ "model.layers.62.self_attn*",
73
+ "model.layers.63.self_attn*",
74
+ "model.layers.64.self_attn*",
75
+ "model.layers.65.self_attn*",
76
+ "model.layers.66.self_attn*",
77
+ "model.layers.67.self_attn*",
78
+ "model.layers.68.self_attn*",
79
+ "model.layers.69.self_attn*",
80
+ "model.layers.7.self_attn*",
81
+ "model.layers.70.self_attn*",
82
+ "model.layers.71.self_attn*",
83
+ "model.layers.72.self_attn*",
84
+ "model.layers.73.self_attn*",
85
+ "model.layers.74.self_attn*",
86
+ "model.layers.75.self_attn*",
87
+ "model.layers.76.self_attn*",
88
+ "model.layers.77.self_attn*",
89
+ "model.layers.78.self_attn*",
90
+ "model.layers.79.self_attn*",
91
+ "model.layers.8.self_attn*",
92
+ "model.layers.9.self_attn*"
93
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  }
95
+ }
 
inference/local/chat_template.jinja DELETED
@@ -1,52 +0,0 @@
1
- {%- if tools %}
2
- {{- '<|im_start|>system\n' }}
3
- {%- if messages[0]['role'] == 'system' %}
4
- {{- messages[0]['content'] + '\n\n' }}
5
- {%- endif %}
6
- {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools> XML tags:\n<tools>" }}
7
- {%- for tool in tools %}
8
- {{- "\n" }}
9
- {{- tool | tojson }}
10
- {%- endfor %}
11
- {{- "\n</tools>\n\nFor each function call, return exactly one valid JSON object with function name and arguments inside XML tool-call tags. The arguments field must be a JSON object, not a string, and strings must use valid JSON escaping:\n<tool_call>\n{\"name\": \"execute_bash\", \"arguments\": {\"command\": \"pwd\"}}\n</tool_call><|im_end|>\n" }}
12
- {%- else %}
13
- {%- if messages[0]['role'] == 'system' %}
14
- {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
15
- {%- endif %}
16
- {%- endif %}
17
- {%- for message in messages %}
18
- {%- set content = message.content or '' %}
19
- {%- if message.role == 'assistant' and message.reasoning_content and not (clear_thinking | default(false)) and not content.lstrip().startswith('<think>') %}
20
- {%- set content = '<think>\n' + message.reasoning_content + '\n</think>\n' + content %}
21
- {%- endif %}
22
- {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
- {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
24
- {%- elif message.role == "assistant" %}
25
- {{- '<|im_start|>' + message.role }}
26
- {%- if content %}
27
- {{- '\n' + content }}
28
- {%- endif %}
29
- {%- for tool_call in message.tool_calls %}
30
- {%- if tool_call.function is defined %}
31
- {%- set tool_call = tool_call.function %}
32
- {%- endif %}
33
- {{- '\n<tool_call>\n{"name": ' }}
34
- {{- tool_call.name | tojson }}
35
- {{- ', "arguments": ' }}
36
- {{- tool_call.arguments | tojson }}
37
- {{- '}\n</tool_call>' }}
38
- {%- endfor %}
39
- {{- '<|im_end|>\n' }}
40
- {%- elif message.role == "tool" %}
41
- {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
- {{- '<|im_start|>user' }}
43
- {%- endif %}
44
- {{- '\n<tool_response>\n' + message.content + '\n</tool_response>' }}
45
- {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
46
- {{- '<|im_end|>\n' }}
47
- {%- endif %}
48
- {%- endif %}
49
- {%- endfor %}
50
- {%- if add_generation_prompt %}
51
- {{- '<|im_start|>assistant\n' }}
52
- {%- endif %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
inference/local/model.py CHANGED
@@ -1,6 +1,6 @@
1
  """Ricursive transformer implemented with SGLang layers.
2
 
3
- Used here for mixed NVFP4-MLP/FP8-attention checkpoints with DFlash2.
4
  The checkpoint supplies configuration, tokenizer and weights.
5
  """
6
 
@@ -35,7 +35,7 @@ def rotary_cache_length(config) -> int:
35
  from sglang.srt.server_args import get_global_server_args
36
 
37
  args = get_global_server_args()
38
- native = int(config.max_position_embeddings)
39
  return native + (
40
  int(args.speculative_num_draft_tokens) if args.speculative_algorithm == "DFLASH" else 0
41
  )
@@ -433,3 +433,100 @@ class RicursiveForCausalLM(nn.Module):
433
  if name in params:
434
  param = params[name]
435
  getattr(param, "weight_loader", default_weight_loader)(param, weight)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """Ricursive transformer implemented with SGLang layers.
2
 
3
+ Used here for NVFP4-MLP/BF16-attention checkpoints with DFlash2.
4
  The checkpoint supplies configuration, tokenizer and weights.
5
  """
6
 
 
35
  from sglang.srt.server_args import get_global_server_args
36
 
37
  args = get_global_server_args()
38
+ native = int(args.context_length or config.max_position_embeddings)
39
  return native + (
40
  int(args.speculative_num_draft_tokens) if args.speculative_algorithm == "DFLASH" else 0
41
  )
 
433
  if name in params:
434
  param = params[name]
435
  getattr(param, "weight_loader", default_weight_loader)(param, weight)
436
+
437
+
438
+ class _NoRope(nn.Module):
439
+ """Identity RoPE for DFlash2 draft layers listed in config.no_rope_layers."""
440
+
441
+ # SGLang's fused draft context-KV path applies one RoPE table to every layer; it
442
+ # only accepts neox-style RoPE, so this marker turns it off for drafts with NoPE layers.
443
+ is_neox_style = False
444
+
445
+ def forward(self, positions, q, k):
446
+ return q, k
447
+
448
+
449
+ # DFlash2 drafts are trained on 16K-token rows (12,288 tokens of context + 4,096 predicted),
450
+ # so a global draft layer never attends further back than this.
451
+ DRAFT_TRAINING_CONTEXT = 16384
452
+
453
+
454
+ def _patch_draft_attention():
455
+ """Fit SGLang 0.5.19's DFlash draft attention to DFlash2 drafts with global layers.
456
+
457
+ It applies RoPE in every layer, ignoring no_rope_layers, and its draft window
458
+ (--speculative-draft-window-size) truncates every layer, including full attention,
459
+ so the window must cover the draft's training context.
460
+ """
461
+ from sglang.srt.models.dflash import DFlashAttention
462
+ from sglang.srt.server_args import get_global_server_args
463
+
464
+ init = DFlashAttention.__init__
465
+
466
+ def __init__(self, config, layer_id, *args, **kwargs):
467
+ init(self, config, layer_id, *args, **kwargs)
468
+ if layer_id in (getattr(config, "no_rope_layers", None) or ()):
469
+ self.rotary_emb = _NoRope()
470
+ self.use_table_qk_norm_rope = False
471
+ layer_types = getattr(config, "layer_types", None) or ()
472
+ window = get_global_server_args().speculative_draft_window_size
473
+ if (
474
+ layer_id < len(layer_types)
475
+ and layer_types[layer_id] == "full_attention"
476
+ and window is not None
477
+ and window < DRAFT_TRAINING_CONTEXT
478
+ ):
479
+ raise ValueError(
480
+ f"Draft layer {layer_id} attends globally over its {DRAFT_TRAINING_CONTEXT}-token "
481
+ f"training rows; use --speculative-draft-window-size >= {DRAFT_TRAINING_CONTEXT}"
482
+ )
483
+
484
+ DFlashAttention.__init__ = __init__
485
+
486
+
487
+ _patch_draft_attention()
488
+
489
+
490
+ def _raise_on_load_gaps(model: str, unused, unloaded) -> None:
491
+ """Refuse checkpoint tensors nothing consumed and parameters nothing loaded."""
492
+ if unused or unloaded:
493
+ raise ValueError(f"{model}: unused tensors {sorted(unused)[:20]}, "
494
+ f"unloaded parameters {sorted(unloaded)[:20]}")
495
+
496
+
497
+ def _strict_draft_loading():
498
+ """SGLang 0.5.19's DFlash loader skips tensors it cannot place: refuse such drafts."""
499
+ from sglang.srt.models.dflash import DFlashDraftModel
500
+
501
+ load, shared = DFlashDraftModel.load_weights, ("inv_freq", "embed_tokens.weight", "lm_head.weight")
502
+
503
+ def load_weights(self, weights):
504
+ if (getattr(self.config, "dflash_config", None) or {}).get("block_norm", "pre") != "pre":
505
+ raise ValueError("Only pre-norm DFlash drafts are supported")
506
+ params, calls, unused = dict(self.named_parameters()), [], []
507
+ own = {n: p.weight_loader for n, p in params.items() if "weight_loader" in p.__dict__}
508
+ for n, p in params.items(): # record which parameters receive a tensor
509
+ f = own.get(n, default_weight_loader)
510
+ p.weight_loader = lambda q, *a, _f=f, _n=n: (calls.append(_n), _f(q, *a))[1]
511
+
512
+ def watch(stream): # a tensor that reached no weight_loader was skipped
513
+ for name, weight in stream:
514
+ before = len(calls)
515
+ yield name, weight
516
+ if len(calls) == before and not name.endswith(shared):
517
+ unused.append(name)
518
+
519
+ try:
520
+ load(self, watch(weights))
521
+ finally:
522
+ for n, p in params.items():
523
+ del p.weight_loader
524
+ if n in own:
525
+ p.weight_loader = own[n]
526
+ unloaded = {n for n in params if not n.endswith(("k_scale", "v_scale"))} - set(calls)
527
+ _raise_on_load_gaps("DFlash draft", unused, unloaded)
528
+
529
+ DFlashDraftModel.load_weights = load_weights
530
+
531
+
532
+ _strict_draft_loading()
inference/local/ripple.ipynb CHANGED
@@ -7,24 +7,21 @@
7
  "source": [
8
  "# Ripple on DGX Spark\n",
9
  "\n",
10
- "<video controls playsinline preload=\"metadata\" width=\"100%\" src=\"https://huggingface.co/ricursiveAI/Ripple-65B-Preview-NVFP4/resolve/main/dgx-demo.mp4\"></video>\n",
11
- "\n",
12
- "[Watch the DGX Spark demo](https://huggingface.co/ricursiveAI/Ripple-65B-Preview-NVFP4/resolve/main/dgx-demo.mp4)\n",
13
- "\n",
14
  "Open this notebook anywhere on your Spark using a **Python 3.12 kernel**.\n",
15
  "It downloads the supporting Python files, chat template, and checkpoints from Hugging Face;\n",
16
  "no other repository files or prior checkpoint downloads are needed.\n",
17
- "The Spark needs CUDA Toolkit 13, a compatible driver, and a C++ compiler (`build-essential`).\n",
 
18
  "\n",
19
- "Only mixed **NVFP4 MLP / FP8 attention** targets with a matching **BF16 DFlash2 draft**\n",
20
- "are supported. Up to 128K context, with space reserved for draft lookahead. One request at a time.\n",
21
  "Inference runs through SGLang's offline engine, with no HTTP server.\n",
22
  "\n",
23
  "To create a dedicated notebook environment on Spark:\n",
24
  "\n",
25
  "```bash\n",
26
  "python3.12 -m venv ~/.venvs/ripple-spark\n",
27
- "~/.venvs/ripple-spark/bin/pip install --upgrade pip jupyterlab ipykernel\n",
28
  "~/.venvs/ripple-spark/bin/jupyter lab\n",
29
  "```\n",
30
  "\n",
@@ -68,22 +65,16 @@
68
  "source": [
69
  "import os\n",
70
  "import platform\n",
71
- "import shutil\n",
72
  "import subprocess\n",
73
  "import sys\n",
74
  "from pathlib import Path\n",
75
  "\n",
76
- "if (platform.system() != \"Linux\" or platform.machine() not in (\"aarch64\", \"arm64\")\n",
77
- " or sys.version_info[:2] != (3, 12)):\n",
78
  " raise RuntimeError(\"Connect to a Python 3.12 Jupyter kernel on the Spark\")\n",
79
  "sys.dont_write_bytecode = True\n",
80
  "os.environ[\"PYTHONDONTWRITEBYTECODE\"] = \"1\"\n",
81
  "os.environ.setdefault(\"CUDA_HOME\", \"/usr/local/cuda-13.0\")\n",
82
  "os.environ[\"PATH\"] = str(Path(sys.executable).parent) + \":\" + os.environ[\"CUDA_HOME\"] + \"/bin:\" + os.environ[\"PATH\"]\n",
83
- "if not (Path(os.environ[\"CUDA_HOME\"]) / \"bin/nvcc\").is_file():\n",
84
- " raise RuntimeError(\"Install CUDA Toolkit 13 or set CUDA_HOME to its installation directory\")\n",
85
- "if shutil.which(\"g++\") is None:\n",
86
- " raise RuntimeError(\"Install the C++ compiler: sudo apt install build-essential\")\n",
87
  "subprocess.check_call(\n",
88
  " [\n",
89
  " sys.executable,\n",
@@ -97,8 +88,8 @@
97
  " \"torch==2.13.0+cu130\",\n",
98
  " \"transformers==5.12.1\",\n",
99
  " \"flashinfer-python==0.6.18\",\n",
100
- " \"nest_asyncio==1.6.0\",\n",
101
  " \"ninja==1.13.2\",\n",
 
102
  " ]\n",
103
  ")"
104
  ]
@@ -110,11 +101,11 @@
110
  "source": [
111
  "## 2. Download and load\n",
112
  "\n",
113
- "The public repositories below supply the code, NVFP4/FP8 target, and BF16 DFlash2 draft.\n",
114
- "No Hugging Face login is required. Downloads are cached for later runs.\n",
115
- "The two Python files and chat template are downloaded from the NVFP4 repository;\n",
116
  "`Spark` downloads the target and draft checkpoints automatically.\n",
117
- "First startup downloads about 52 GB of weights and compiles kernels. Allow at least 70 GB of free disk space.\n",
118
  "Stop other GPU workloads first. Run this cell once; call `model.shutdown()` before reloading.\n"
119
  ]
120
  },
@@ -131,10 +122,12 @@
131
  "import nest_asyncio\n",
132
  "from huggingface_hub import snapshot_download\n",
133
  "\n",
134
- "code_dir = Path(snapshot_download(\n",
 
135
  " TARGET,\n",
136
- " allow_patterns=[\"inference/local/spark.py\", \"inference/local/model.py\", \"inference/local/chat_template.jinja\"],\n",
137
- "))\n",
 
138
  "sys.path.insert(0, str(code_dir / \"inference/local\"))\n",
139
  "nest_asyncio.apply()\n",
140
  "\n",
@@ -189,7 +182,7 @@
189
  "messages = [{\"role\": \"user\", \"content\": \"Explain speculative decoding in three short sentences.\"}]\n",
190
  "prompt = model.tokenizer_manager.tokenizer.apply_chat_template(\n",
191
  " messages,\n",
192
- " chat_template=model.chat_template_path.read_text(),\n",
193
  " tokenize=False,\n",
194
  " add_generation_prompt=True,\n",
195
  " clear_thinking=False,\n",
 
7
  "source": [
8
  "# Ripple on DGX Spark\n",
9
  "\n",
 
 
 
 
10
  "Open this notebook anywhere on your Spark using a **Python 3.12 kernel**.\n",
11
  "It downloads the supporting Python files, chat template, and checkpoints from Hugging Face;\n",
12
  "no other repository files or prior checkpoint downloads are needed.\n",
13
+ "The Spark needs CUDA Toolkit 13, a compatible driver, a C++ compiler (`build-essential`),\n",
14
+ "and Python headers (`python3.12-dev`), which Triton uses to build its kernel launcher.\n",
15
  "\n",
16
+ "Only **NVFP4 MLP / BF16 attention** targets with a matching **BF16 DFlash2 draft**\n",
17
+ "are supported. One request at a time.\n",
18
  "Inference runs through SGLang's offline engine, with no HTTP server.\n",
19
  "\n",
20
  "To create a dedicated notebook environment on Spark:\n",
21
  "\n",
22
  "```bash\n",
23
  "python3.12 -m venv ~/.venvs/ripple-spark\n",
24
+ "~/.venvs/ripple-spark/bin/pip install jupyterlab\n",
25
  "~/.venvs/ripple-spark/bin/jupyter lab\n",
26
  "```\n",
27
  "\n",
 
65
  "source": [
66
  "import os\n",
67
  "import platform\n",
 
68
  "import subprocess\n",
69
  "import sys\n",
70
  "from pathlib import Path\n",
71
  "\n",
72
+ "if platform.system() != \"Linux\" or sys.version_info[:2] != (3, 12):\n",
 
73
  " raise RuntimeError(\"Connect to a Python 3.12 Jupyter kernel on the Spark\")\n",
74
  "sys.dont_write_bytecode = True\n",
75
  "os.environ[\"PYTHONDONTWRITEBYTECODE\"] = \"1\"\n",
76
  "os.environ.setdefault(\"CUDA_HOME\", \"/usr/local/cuda-13.0\")\n",
77
  "os.environ[\"PATH\"] = str(Path(sys.executable).parent) + \":\" + os.environ[\"CUDA_HOME\"] + \"/bin:\" + os.environ[\"PATH\"]\n",
 
 
 
 
78
  "subprocess.check_call(\n",
79
  " [\n",
80
  " sys.executable,\n",
 
88
  " \"torch==2.13.0+cu130\",\n",
89
  " \"transformers==5.12.1\",\n",
90
  " \"flashinfer-python==0.6.18\",\n",
 
91
  " \"ninja==1.13.2\",\n",
92
+ " \"nest_asyncio==1.6.0\",\n",
93
  " ]\n",
94
  ")"
95
  ]
 
101
  "source": [
102
  "## 2. Download and load\n",
103
  "\n",
104
+ "The public repositories below supply the code, NVFP4/BF16 target, and BF16 DFlash2 draft.\n",
105
+ "For private repositories, run `hf auth login` in the Spark terminal first. Downloads are cached for later runs.\n",
106
+ "The local Python files, shared format parser, and chat template are downloaded from the target repository;\n",
107
  "`Spark` downloads the target and draft checkpoints automatically.\n",
108
+ "First startup downloads the weights and compiles kernels, so allow extra time and disk space.\n",
109
  "Stop other GPU workloads first. Run this cell once; call `model.shutdown()` before reloading.\n"
110
  ]
111
  },
 
122
  "import nest_asyncio\n",
123
  "from huggingface_hub import snapshot_download\n",
124
  "\n",
125
+ "code_dir = Path.home() / \".cache/ripple/code\"\n",
126
+ "snapshot_download(\n",
127
  " TARGET,\n",
128
+ " allow_patterns=[\"inference/local/spark.py\", \"inference/local/model.py\", \"inference/server/ripple_format.py\", \"chat_template.jinja\"],\n",
129
+ " local_dir=code_dir,\n",
130
+ ")\n",
131
  "sys.path.insert(0, str(code_dir / \"inference/local\"))\n",
132
  "nest_asyncio.apply()\n",
133
  "\n",
 
182
  "messages = [{\"role\": \"user\", \"content\": \"Explain speculative decoding in three short sentences.\"}]\n",
183
  "prompt = model.tokenizer_manager.tokenizer.apply_chat_template(\n",
184
  " messages,\n",
185
+ " chat_template=(code_dir / \"chat_template.jinja\").read_text(),\n",
186
  " tokenize=False,\n",
187
  " add_generation_prompt=True,\n",
188
  " clear_thinking=False,\n",
inference/local/spark.py CHANGED
@@ -1,9 +1,11 @@
1
- """Offline Ripple inference on DGX Spark: mixed NVFP4/FP8 + BF16 DFlash2."""
2
 
 
3
  import functools
4
  import json
5
  import os
6
  import shlex
 
7
  from pathlib import Path
8
 
9
  # Keep the Spark kernel choices used by the original notebook.
@@ -22,7 +24,11 @@ from sglang.srt.server_args import prepare_server_args
22
 
23
  from model import RicursiveForCausalLM
24
 
 
 
 
25
  ModelRegistry.models["RicursiveForCausalLM"] = RicursiveForCausalLM
 
26
 
27
 
28
  def checkpoint(value):
@@ -32,32 +38,39 @@ def checkpoint(value):
32
  return path.resolve()
33
  if str(value).startswith(("/", "~", ".", "gs://")):
34
  raise FileNotFoundError(value)
35
- return Path(snapshot_download(str(value), ignore_patterns=["*.mp4", "*.ipynb"]))
36
 
37
 
38
  def check_checkpoints(target, draft):
39
- """Require NVFP4 MLPs, FP8 attention, and an unquantized DFlash2 draft."""
40
  config = json.loads((target / "config.json").read_text())
41
  if config.get("architectures") != ["RicursiveForCausalLM"]:
42
  raise ValueError("Target must be a RicursiveForCausalLM checkpoint")
43
  quant = config.get("quantization_config") or {}
44
  if (target / "hf_quant_config.json").exists():
45
  quant = json.loads((target / "hf_quant_config.json").read_text())["quantization"]
46
- if quant.get("quant_algo") != "MIXED_PRECISION":
47
- raise ValueError("Spark requires a mixed NVFP4/FP8 ModelOpt checkpoint")
48
- layers = quant.get("quantized_layers", {})
 
 
 
 
 
 
 
 
 
 
 
 
49
  for index in range(config["num_hidden_layers"]):
50
- projections = {f"mlp.{name}_proj": "NVFP4" for name in ("gate", "up", "down")}
51
- heads = (
52
- ("q", "k", "v", "o", "g")
53
- if config.get("use_gated_attention", True)
54
- else ("q", "k", "v", "o")
55
- )
56
- projections.update({f"self_attn.{name}_proj": "FP8" for name in heads})
57
- for name, algorithm in projections.items():
58
- key = f"model.layers.{index}.{name}"
59
- if layers.get(key, {}).get("quant_algo") != algorithm:
60
- raise ValueError(f"Expected {algorithm} for {key}")
61
  draft_config = json.loads((draft / "config.json").read_text())
62
  if draft_config.get("architectures") != ["DFlash2DraftModel"]:
63
  raise ValueError("Draft must be a DFlash2DraftModel checkpoint")
@@ -65,13 +78,6 @@ def check_checkpoints(target, draft):
65
  raise ValueError("The DFlash2 draft must be unquantized BF16")
66
 
67
 
68
- class NoRotaryEmbedding(torch.nn.Module):
69
- """Leave Q/K unchanged for a draft layer configured without RoPE."""
70
-
71
- def forward(self, positions, query, key):
72
- return query, key
73
-
74
-
75
  def run_worker(*args, **kwargs):
76
  """Retain the two numerical fixes required by the Spark attention path."""
77
  from sglang.kernels.ops.speculative.dspark import dspark_accept
@@ -92,16 +98,6 @@ def run_worker(*args, **kwargs):
92
  @functools.wraps(load_model)
93
  def load(self):
94
  result = load_model(self)
95
- if self.is_draft_worker:
96
- no_rope_layers = getattr(self.model.config, "no_rope_layers", [])
97
- for index in no_rope_layers:
98
- attention = self.model.layers[index].self_attn
99
- attention.rotary_emb = NoRotaryEmbedding()
100
- attention.use_table_qk_norm_rope = False
101
- if no_rope_layers:
102
- # SGLang 0.5.19's fused context-KV path shares one RoPE table
103
- # across all layers; the sequential path preserves NoPE.
104
- self.model.supports_fused_context_kv = False
105
  for name, module in self.model.named_modules():
106
  if isinstance(module, RadixAttention):
107
  # Quantized exports can carry FP8 scales although our KV is BF16.
@@ -124,41 +120,35 @@ class Spark(Engine):
124
 
125
  run_scheduler_process_func = staticmethod(run_worker)
126
 
127
- def __init__(self, target, draft):
128
  if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 1):
129
  raise RuntimeError("Run this notebook on a DGX Spark / GB10 GPU")
130
  target, draft = checkpoint(target), checkpoint(draft)
131
  check_checkpoints(target, draft)
132
- self.chat_template_path = Path(__file__).with_name("chat_template.jinja")
133
- if not self.chat_template_path.is_file():
134
- # Also support the original ripple-inference checkout layout.
135
- self.chat_template_path = Path(__file__).resolve().parents[2] / "chat_template.jinja"
136
- if not self.chat_template_path.is_file():
137
- raise FileNotFoundError("Download chat_template.jinja alongside spark.py")
138
  args = prepare_server_args(
139
  [
140
  "--model-path",
141
  str(target),
142
- "--speculative-draft-model-path",
143
- str(draft),
144
  "--chat-template",
145
- str(self.chat_template_path),
146
  ]
147
- + shlex.split("""
148
  --trust-remote-code --dtype bfloat16 --kv-cache-dtype bfloat16 --tp-size 1
149
  --attention-backend flashinfer --fp4-gemm-backend flashinfer_cutlass
150
  --flashinfer-autotune-skip-ops fp8_gemm
151
- --speculative-algorithm DFLASH --speculative-draft-model-quantization unquant
152
- --speculative-num-draft-tokens 32
153
- --speculative-draft-attention-backend triton
154
- --context-length 131040 --max-total-tokens 139264
155
- --json-model-override-args '{"max_position_embeddings": 131072}' --mem-fraction-static 0.78
156
  --max-running-requests 1 --cuda-graph-bs-decode 1
157
  --cuda-graph-backend-prefill disabled --disable-radix-cache
158
  --weight-loader-disable-mmap --weight-loader-drop-cache-after-load
159
  --model-loader-extra-config '{"enable_multithread_load": false}'
160
  --random-seed 42 --watchdog-timeout 1800 --log-level warning
161
- --tool-call-parser hermes --reasoning-parser deepseek-v3
162
  """)
163
  )
164
  super().__init__(server_args=args)
 
1
+ """Offline Ripple inference on DGX Spark: NVFP4 MLPs, BF16 attention + BF16 DFlash2."""
2
 
3
+ import fnmatch
4
  import functools
5
  import json
6
  import os
7
  import shlex
8
+ import sys
9
  from pathlib import Path
10
 
11
  # Keep the Spark kernel choices used by the original notebook.
 
24
 
25
  from model import RicursiveForCausalLM
26
 
27
+ sys.path.append(str(Path(__file__).resolve().parents[1] / "server"))
28
+ import ripple_format # noqa: E402
29
+
30
  ModelRegistry.models["RicursiveForCausalLM"] = RicursiveForCausalLM
31
+ ripple_format.register()
32
 
33
 
34
  def checkpoint(value):
 
38
  return path.resolve()
39
  if str(value).startswith(("/", "~", ".", "gs://")):
40
  raise FileNotFoundError(value)
41
+ return Path(snapshot_download(str(value)))
42
 
43
 
44
  def check_checkpoints(target, draft):
45
+ """Require NVFP4 MLPs, BF16 attention, and an unquantized DFlash2 draft."""
46
  config = json.loads((target / "config.json").read_text())
47
  if config.get("architectures") != ["RicursiveForCausalLM"]:
48
  raise ValueError("Target must be a RicursiveForCausalLM checkpoint")
49
  quant = config.get("quantization_config") or {}
50
  if (target / "hf_quant_config.json").exists():
51
  quant = json.loads((target / "hf_quant_config.json").read_text())["quantization"]
52
+ if quant.get("quant_algo") != "NVFP4" or quant.get("group_size") != 16:
53
+ raise ValueError("Spark requires an NVFP4/BF16 ModelOpt checkpoint (MLP group size 16)")
54
+ if quant.get("kv_cache_quant_algo") is not None:
55
+ raise ValueError("Spark requires an unquantized BF16 KV cache")
56
+ if quant.get("quantized_layers"):
57
+ raise ValueError("Per-layer mixed-precision overrides are not supported")
58
+ excluded = quant.get("exclude_modules", [])
59
+
60
+ def require(name, bf16):
61
+ is_excluded = any(fnmatch.fnmatchcase(name, pattern) for pattern in excluded)
62
+ if is_excluded != bf16:
63
+ raise ValueError(f"Expected {'BF16' if bf16 else 'NVFP4'} for {name}")
64
+
65
+ for name in ("lm_head", "model.embed_tokens"):
66
+ require(name, True)
67
  for index in range(config["num_hidden_layers"]):
68
+ prefix = f"model.layers.{index}"
69
+ for name in ("gate", "up", "gate_up", "down"):
70
+ require(f"{prefix}.mlp.{name}_proj", False)
71
+ heads = ("q", "k", "v", "o", "g") if config.get("use_gated_attention", True) else ("q", "k", "v", "o")
72
+ for name in heads:
73
+ require(f"{prefix}.self_attn.{name}_proj", True)
 
 
 
 
 
74
  draft_config = json.loads((draft / "config.json").read_text())
75
  if draft_config.get("architectures") != ["DFlash2DraftModel"]:
76
  raise ValueError("Draft must be a DFlash2DraftModel checkpoint")
 
78
  raise ValueError("The DFlash2 draft must be unquantized BF16")
79
 
80
 
 
 
 
 
 
 
 
81
  def run_worker(*args, **kwargs):
82
  """Retain the two numerical fixes required by the Spark attention path."""
83
  from sglang.kernels.ops.speculative.dspark import dspark_accept
 
98
  @functools.wraps(load_model)
99
  def load(self):
100
  result = load_model(self)
 
 
 
 
 
 
 
 
 
 
101
  for name, module in self.model.named_modules():
102
  if isinstance(module, RadixAttention):
103
  # Quantized exports can carry FP8 scales although our KV is BF16.
 
120
 
121
  run_scheduler_process_func = staticmethod(run_worker)
122
 
123
+ def __init__(self, target, draft, *, speculative=True):
124
  if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 1):
125
  raise RuntimeError("Run this notebook on a DGX Spark / GB10 GPU")
126
  target, draft = checkpoint(target), checkpoint(draft)
127
  check_checkpoints(target, draft)
128
+ spec_args = [
129
+ "--speculative-draft-model-path", str(draft),
130
+ "--speculative-algorithm", "DFLASH", "--speculative-draft-model-quantization", "unquant",
131
+ "--speculative-num-draft-tokens", "32", "--speculative-draft-window-size", "16384",
132
+ "--speculative-draft-attention-backend", "triton",
133
+ ] if speculative else []
134
  args = prepare_server_args(
135
  [
136
  "--model-path",
137
  str(target),
 
 
138
  "--chat-template",
139
+ str(Path(__file__).resolve().parents[2] / "chat_template.jinja"),
140
  ]
141
+ + spec_args + shlex.split("""
142
  --trust-remote-code --dtype bfloat16 --kv-cache-dtype bfloat16 --tp-size 1
143
  --attention-backend flashinfer --fp4-gemm-backend flashinfer_cutlass
144
  --flashinfer-autotune-skip-ops fp8_gemm
145
+ --context-length 131072 --max-total-tokens 139264 --mem-fraction-static 0.78
 
 
 
 
146
  --max-running-requests 1 --cuda-graph-bs-decode 1
147
  --cuda-graph-backend-prefill disabled --disable-radix-cache
148
  --weight-loader-disable-mmap --weight-loader-drop-cache-after-load
149
  --model-loader-extra-config '{"enable_multithread_load": false}'
150
  --random-seed 42 --watchdog-timeout 1800 --log-level warning
151
+ --tool-call-parser ripple --reasoning-parser ripple
152
  """)
153
  )
154
  super().__init__(server_args=args)
inference/server/ripple_format.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ripple's chat format for SGLang: reasoning parser, tool-call parser and tokenizer check.
2
+
3
+ The model generates the layout chat_template.jinja renders:
4
+
5
+ <think>\\n{reasoning}\\n</think>\\n{content}
6
+ <think>\\n{reasoning}\\n</think>\\n\\n<tool_call>\\n{"name": "f", "arguments": {...}}\\n</tool_call>
7
+
8
+ `--reasoning-parser ripple` returns reasoning and content without the separator newlines the
9
+ template adds back, so an assistant message sent back unchanged re-renders to the tokens the
10
+ model generated (SGLang's deepseek-v3 parser keeps them, doubling each newline on every turn).
11
+ Output counts as reasoning only when it opens with <think>.
12
+
13
+ `--tool-call-parser ripple` is hermes with the template's call layout, so strict tools and
14
+ tool_choice=required constrain decoding to `<tool_call>\\n{"name": "f", "arguments": ...}`
15
+ (hermes forces `<tool_call>{"name":"f"`, which the model was not trained on). Text before the
16
+ first call keeps its whitespace, minus the newline the template puts before <tool_call>.
17
+
18
+ Output the template cannot render (`</think>\\n<tool_call>` with one newline, text after a call)
19
+ re-renders in the template's canonical layout.
20
+
21
+ `register()` adds both parsers and checks every tokenizer SGLang loads against the checkpoint's
22
+ tokenizer.json (Transformers 5 can rebuild the pre-tokenizer from `tokenizer_class`).
23
+ """
24
+
25
+ import json
26
+ import logging
27
+
28
+ THINK_START, THINK_END = "<think>", "</think>"
29
+
30
+ # Exercise pre-tokenizer differences: digit runs, contractions, whitespace runs, non-ASCII.
31
+ TOKENIZER_PROBES = (
32
+ "x = 1234567890 + 3.14159; y=-42",
33
+ "I'M here, you'RE there, it's 4:05pm. DON'T",
34
+ "a b c\n\n\n d\t\te \r\n f",
35
+ " def f(self):\n return {'k': [1, 2]}\n",
36
+ "café naïve 日本語テキスト 한국어 🙂👍 Ελληνικά",
37
+ "<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n<think>\nok\n</think>\nHello<|im_end|>",
38
+ )
39
+
40
+
41
+ def _drop_newline(text, lead):
42
+ if lead:
43
+ return text[1:] if text.startswith("\n") else text
44
+ return text[:-1] if text.endswith("\n") else text
45
+
46
+
47
+ def _partial_suffix(text, token):
48
+ """Length of the longest suffix of text that is a strict prefix of token."""
49
+ for i in range(min(len(text), len(token) - 1), 0, -1):
50
+ if token.startswith(text[-i:]):
51
+ return i
52
+ return 0
53
+
54
+
55
+ def _reasoning_detector_class():
56
+ from sglang.srt.parser.reasoning_parser import (
57
+ BaseReasoningFormatDetector,
58
+ StreamingParseResult,
59
+ )
60
+
61
+ class RippleReasoningDetector(BaseReasoningFormatDetector):
62
+ def __init__(self, stream_reasoning=True, force_reasoning=False, **kwargs):
63
+ # force_reasoning is ignored: the generation prompt does not open <think>.
64
+ super().__init__(
65
+ THINK_START,
66
+ THINK_END,
67
+ force_reasoning=False,
68
+ stream_reasoning=stream_reasoning,
69
+ reasoning_default="explicit_thinking", # as deepseek-v3
70
+ **kwargs,
71
+ )
72
+ self._state = "start" # start -> lead -> think -> after -> content
73
+
74
+ def _detect_and_parse_impl(self, text):
75
+ if self.continue_final_message:
76
+ return super()._detect_and_parse_impl(text)
77
+ if not text.startswith(THINK_START):
78
+ return StreamingParseResult(normal_text=text)
79
+ body = _drop_newline(text[len(THINK_START):], lead=True)
80
+ reasoning, end, content = body.partition(THINK_END)
81
+ if not end: # truncated inside reasoning
82
+ return StreamingParseResult(reasoning_text=reasoning)
83
+ return StreamingParseResult(
84
+ normal_text=_drop_newline(content, lead=True),
85
+ reasoning_text=_drop_newline(reasoning, lead=False),
86
+ )
87
+
88
+ def _parse_streaming_increment_impl(self, new_text):
89
+ if self.continue_final_message:
90
+ return super()._parse_streaming_increment_impl(new_text)
91
+ self._buffer += new_text
92
+ reasoning, normal = "", ""
93
+ while self._buffer:
94
+ buf = self._buffer
95
+ if self._state == "start":
96
+ if buf.startswith(THINK_START):
97
+ self._buffer, self._state, self._in_reasoning = buf[len(THINK_START):], "lead", True
98
+ elif THINK_START.startswith(buf):
99
+ break
100
+ else:
101
+ self._state = "content"
102
+ elif self._state == "lead":
103
+ self._buffer, self._state = _drop_newline(buf, lead=True), "think"
104
+ elif self._state == "think":
105
+ i = buf.find(THINK_END)
106
+ if i >= 0:
107
+ reasoning += _drop_newline(buf[:i], lead=False)
108
+ self._buffer, self._state, self._in_reasoning = buf[i + len(THINK_END):], "after", False
109
+ continue
110
+ if not self.stream_reasoning:
111
+ break
112
+ # Hold back a possible "\n</think>" prefix: its newline is the separator.
113
+ hold = max(_partial_suffix(buf, "\n" + THINK_END), _partial_suffix(buf, THINK_END))
114
+ reasoning += buf[:len(buf) - hold]
115
+ self._buffer = buf[len(buf) - hold:]
116
+ break
117
+ elif self._state == "after":
118
+ self._buffer, self._state = _drop_newline(buf, lead=True), "content"
119
+ else:
120
+ normal += buf
121
+ self._buffer = ""
122
+ return StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
123
+
124
+ def finish(self):
125
+ if self.continue_final_message:
126
+ return super().finish()
127
+ buf, self._buffer = self._buffer, ""
128
+ if self._state in ("lead", "think"): # truncated inside reasoning
129
+ if self._force_nonempty_content:
130
+ text, self._accumulated_reasoning = self._accumulated_reasoning + buf, ""
131
+ return StreamingParseResult(normal_text=text)
132
+ return StreamingParseResult(reasoning_text=buf)
133
+ return StreamingParseResult(normal_text=buf)
134
+
135
+ return RippleReasoningDetector
136
+
137
+
138
+ def _tool_detector_class():
139
+ from sglang.srt.function_call.core_types import StructureInfo
140
+ from sglang.srt.function_call.hermes_detector import HermesDetector
141
+
142
+ class RippleToolDetector(HermesDetector):
143
+ """Text before the first call keeps its whitespace except the one newline the template
144
+ puts before <tool_call> (hermes strips it all)."""
145
+
146
+ def __init__(self):
147
+ super().__init__()
148
+ self._held_newline = ""
149
+ self._in_calls = False
150
+
151
+ def detect_and_parse(self, text, tools):
152
+ res = super().detect_and_parse(text, tools)
153
+ idx = text.find(self.bot_token)
154
+ if idx != -1 and res.normal_text == text[:idx].strip(): # parsed (not the error path)
155
+ res.normal_text = _drop_newline(text[:idx], lead=False)
156
+ return res
157
+
158
+ def parse_streaming_increment(self, new_text, tools):
159
+ res = super().parse_streaming_increment(new_text, tools)
160
+ if self._in_calls: # text after the first call is dropped, as in detect_and_parse
161
+ res.normal_text = ""
162
+ return res
163
+ call_started = bool(self._buffer.startswith(self.bot_token) or self.current_tool_id >= 0 or res.calls)
164
+ self._in_calls = call_started
165
+ if res.normal_text:
166
+ text, self._held_newline = self._held_newline + res.normal_text, ""
167
+ if call_started: # its trailing newline is the separator before <tool_call>
168
+ text = _drop_newline(text, lead=False)
169
+ elif text.endswith("\n"): # undecided until the next text arrives
170
+ text, self._held_newline = text[:-1], "\n"
171
+ res.normal_text = text
172
+ elif call_started:
173
+ self._held_newline = ""
174
+ return res
175
+
176
+ def finish(self, tools):
177
+ res = super().finish(tools)
178
+ if self._in_calls:
179
+ res.normal_text = ""
180
+ elif self._held_newline:
181
+ res.normal_text, self._held_newline = self._held_newline + (res.normal_text or ""), ""
182
+ return res
183
+
184
+ def structure_info(self):
185
+ return lambda name: StructureInfo(
186
+ begin='<tool_call>\n{"name": ' + json.dumps(name, ensure_ascii=False) + ', "arguments": ',
187
+ end="}\n</tool_call>",
188
+ trigger="<tool_call>",
189
+ )
190
+
191
+ return RippleToolDetector
192
+
193
+
194
+ def check_tokenizer(tokenizer, tokenizer_json):
195
+ """Make `tokenizer` encode and decode like the raw tokenizer.json, or raise.
196
+
197
+ Restores the normalizer, pre-tokenizer and decoder from tokenizer.json when they differ."""
198
+ from tokenizers import Tokenizer
199
+
200
+ raw = Tokenizer.from_file(tokenizer_json)
201
+
202
+ def mismatches():
203
+ bad = []
204
+ for text in TOKENIZER_PROBES:
205
+ ids = raw.encode(text, add_special_tokens=False).ids
206
+ got = tokenizer.encode(text, add_special_tokens=False)
207
+ if got != ids or tokenizer.decode(ids) != raw.decode(ids, skip_special_tokens=False):
208
+ bad.append(text)
209
+ return bad
210
+
211
+ if not mismatches():
212
+ return False
213
+ backend = getattr(tokenizer, "_tokenizer", None)
214
+ if backend is not None:
215
+ backend.normalizer, backend.pre_tokenizer = raw.normalizer, raw.pre_tokenizer
216
+ backend.decoder = raw.decoder
217
+ bad = mismatches()
218
+ if bad:
219
+ raise RuntimeError(
220
+ f"{type(tokenizer).__name__} does not match {tokenizer_json} on {bad!r}; "
221
+ "export the tokenizer with tokenizer_class PreTrainedTokenizerFast"
222
+ )
223
+ return True
224
+
225
+
226
+ def _tokenizer_json(name, revision=None):
227
+ from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
228
+
229
+ try:
230
+ return hf_tok._resolve_local_or_cached_file(name, "tokenizer.json", revision)
231
+ except (OSError, ValueError): # no tokenizer.json (e.g. a slow-only tokenizer)
232
+ return None
233
+
234
+
235
+ def register():
236
+ """Add the `ripple` parsers to SGLang and check tokenizers on load. Idempotent; call at
237
+ import time so SGLang's spawned processes register too."""
238
+ from sglang.srt.function_call.function_call_parser import FunctionCallParser
239
+ from sglang.srt.parser.reasoning_parser import ReasoningParser
240
+ from sglang.srt.utils.hf_transformers import tokenizer as hf_tok
241
+
242
+ ReasoningParser.DetectorMap.setdefault("ripple", _reasoning_detector_class())
243
+ FunctionCallParser.ToolCallParserEnum.setdefault("ripple", _tool_detector_class())
244
+
245
+ fixes = hf_tok._apply_post_load_fixes # fail loudly if SGLang renames it
246
+ if getattr(fixes, "_ripple", False):
247
+ return
248
+
249
+ def apply_post_load_fixes(tokenizer, name, revision=None):
250
+ tokenizer = fixes(tokenizer, name, revision)
251
+ path = _tokenizer_json(name, revision)
252
+ if path is not None and check_tokenizer(tokenizer, path):
253
+ logging.getLogger(__name__).warning(
254
+ "Restored %s's normalizer, pre-tokenizer and decoder from tokenizer.json", name)
255
+ return tokenizer
256
+
257
+ apply_post_load_fixes._ripple = True
258
+ hf_tok._apply_post_load_fixes = apply_post_load_fixes
dgx-demo.mp4 → model-00001-of-00006.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:710c0830b0a8cd0d02edf207b622bf04b2faee88e0ccf4a4726d24316827af3b
3
- size 2186284
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eaa330c20b3774b201349acbbdc72d2c37fc08cdda5b57a241ef3df0580cb497
3
+ size 9990882364
model-00003-of-00005.safetensors → model-00002-of-00006.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:acbb995c8ab2891dcfe52b379b78fdfbc1c39d24b6eb3b8f62bc1a31a7fbb512
3
- size 9964952382
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f460bca35a7b99faaee131f898a54c9d54b808d05fd1ad59493c2403538f8c96
3
+ size 9988306372
model-00001-of-00005.safetensors → model-00003-of-00006.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a5065a14fe94dcc61bb586140deb8781edf1fe96f8a6d0079aa7698faac361b7
3
- size 9959620668
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c570c1435b733c51183221cf5473918c6f9cbb658c97eb322879e5995d96416
3
+ size 9993548868
model-00004-of-00005.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:09c6b842621e8c83f310d49c31cd7ad4ff093705a8f7806f31c34059c292f8ab
3
- size 9912523246
 
 
 
 
model-00002-of-00005.safetensors → model-00004-of-00006.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b587eea5b4573206623267713f50c444a90a6cf789c44a5153af7d4d4f6d3e60
3
- size 9964953102
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:372a737fb2e758e3911dfb49f1b6be2a26f2c99533337dc7e4390cc082b747ad
3
+ size 9896597694
model-00005-of-00005.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f0f083da8ab6100961129b587aad9d6837f3504a1bbc2cffa5e6a1b77fe30e67
3
- size 3990111074
 
 
 
 
model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2448f900b27958228fa703903fa9b4ff0bd14590a31fe0c7627bf88715e98ce
3
+ size 9993548908
model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7cf2508852befa8303153f441a588745270ef3ce163188072d35963d7413578
3
+ size 7350967818
model.safetensors.index.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer.json CHANGED
@@ -222,7 +222,32 @@
222
  }
223
  ]
224
  },
225
- "post_processor": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
  "decoder": {
227
  "type": "ByteLevel",
228
  "add_prefix_space": true,
 
222
  }
223
  ]
224
  },
225
+ "post_processor": {
226
+ "type": "TemplateProcessing",
227
+ "single": [
228
+ {
229
+ "Sequence": {
230
+ "id": "A",
231
+ "type_id": 0
232
+ }
233
+ }
234
+ ],
235
+ "pair": [
236
+ {
237
+ "Sequence": {
238
+ "id": "A",
239
+ "type_id": 0
240
+ }
241
+ },
242
+ {
243
+ "Sequence": {
244
+ "id": "B",
245
+ "type_id": 1
246
+ }
247
+ }
248
+ ],
249
+ "special_tokens": {}
250
+ },
251
  "decoder": {
252
  "type": "ByteLevel",
253
  "add_prefix_space": true,
tokenizer_config.json CHANGED
@@ -1,188 +1,12 @@
1
  {
2
  "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "100256": {
5
- "content": "<|extra_id_0|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": false
11
- },
12
- "100257": {
13
- "content": "<|endoftext|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- },
20
- "100258": {
21
- "content": "<|fim_prefix|>",
22
- "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": true
27
- },
28
- "100259": {
29
- "content": "<|fim_middle|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": true
35
- },
36
- "100260": {
37
- "content": "<|fim_suffix|>",
38
- "lstrip": false,
39
- "normalized": false,
40
- "rstrip": false,
41
- "single_word": false,
42
- "special": true
43
- },
44
- "100261": {
45
- "content": "|||PHONE_NUMBER|||",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": false,
49
- "single_word": false,
50
- "special": false
51
- },
52
- "100262": {
53
- "content": "|||EMAIL_ADDRESS|||",
54
- "lstrip": false,
55
- "normalized": false,
56
- "rstrip": false,
57
- "single_word": false,
58
- "special": false
59
- },
60
- "100263": {
61
- "content": "|||IP_ADDRESS|||",
62
- "lstrip": false,
63
- "normalized": false,
64
- "rstrip": false,
65
- "single_word": false,
66
- "special": false
67
- },
68
- "100264": {
69
- "content": "<|im_start|>",
70
- "lstrip": false,
71
- "normalized": false,
72
- "rstrip": false,
73
- "single_word": false,
74
- "special": true
75
- },
76
- "100265": {
77
- "content": "<|im_end|>",
78
- "lstrip": false,
79
- "normalized": false,
80
- "rstrip": false,
81
- "single_word": false,
82
- "special": true
83
- },
84
- "100266": {
85
- "content": "<|extra_id_1|>",
86
- "lstrip": false,
87
- "normalized": false,
88
- "rstrip": false,
89
- "single_word": false,
90
- "special": false
91
- },
92
- "100267": {
93
- "content": "<|extra_id_2|>",
94
- "lstrip": false,
95
- "normalized": false,
96
- "rstrip": false,
97
- "single_word": false,
98
- "special": false
99
- },
100
- "100268": {
101
- "content": "<|extra_id_3|>",
102
- "lstrip": false,
103
- "normalized": false,
104
- "rstrip": false,
105
- "single_word": false,
106
- "special": false
107
- },
108
- "100269": {
109
- "content": "<|extra_id_4|>",
110
- "lstrip": false,
111
- "normalized": false,
112
- "rstrip": false,
113
- "single_word": false,
114
- "special": false
115
- },
116
- "100270": {
117
- "content": "<|extra_id_5|>",
118
- "lstrip": false,
119
- "normalized": false,
120
- "rstrip": false,
121
- "single_word": false,
122
- "special": false
123
- },
124
- "100271": {
125
- "content": "<|extra_id_6|>",
126
- "lstrip": false,
127
- "normalized": false,
128
- "rstrip": false,
129
- "single_word": false,
130
- "special": false
131
- },
132
- "100272": {
133
- "content": "<|extra_id_7|>",
134
- "lstrip": false,
135
- "normalized": false,
136
- "rstrip": false,
137
- "single_word": false,
138
- "special": false
139
- },
140
- "100273": {
141
- "content": "<|extra_id_8|>",
142
- "lstrip": false,
143
- "normalized": false,
144
- "rstrip": false,
145
- "single_word": false,
146
- "special": false
147
- },
148
- "100274": {
149
- "content": "<|extra_id_9|>",
150
- "lstrip": false,
151
- "normalized": false,
152
- "rstrip": false,
153
- "single_word": false,
154
- "special": false
155
- },
156
- "100275": {
157
- "content": "<|extra_id_10|>",
158
- "lstrip": false,
159
- "normalized": false,
160
- "rstrip": false,
161
- "single_word": false,
162
- "special": false
163
- },
164
- "100276": {
165
- "content": "<|endofprompt|>",
166
- "lstrip": false,
167
- "normalized": false,
168
- "rstrip": false,
169
- "single_word": false,
170
- "special": true
171
- },
172
- "100277": {
173
- "content": "<|pad|>",
174
- "lstrip": false,
175
- "normalized": false,
176
- "rstrip": false,
177
- "single_word": false,
178
- "special": true
179
- }
180
- },
181
  "bos_token": "<|endoftext|>",
182
- "chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}",
183
  "clean_up_tokenization_spaces": false,
184
  "eos_token": "<|im_end|>",
185
- "extra_special_tokens": {},
 
 
186
  "model_max_length": 131072,
187
  "pad_token": "<|pad|>",
188
  "tokenizer_class": "PreTrainedTokenizerFast",
 
1
  {
2
  "add_prefix_space": false,
3
+ "backend": "tokenizers",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  "bos_token": "<|endoftext|>",
 
5
  "clean_up_tokenization_spaces": false,
6
  "eos_token": "<|im_end|>",
7
+ "fix_mistral_regex": false,
8
+ "is_local": true,
9
+ "local_files_only": true,
10
  "model_max_length": 131072,
11
  "pad_token": "<|pad|>",
12
  "tokenizer_class": "PreTrainedTokenizerFast",