Hugging Face's logo

nvidia
/
GLM-5.3-Flash-NVFP4

Image-Text-to-Text
Safetensors
Model Optimizer
glm5_next
nvidia
ModelOpt
GLM-5
quantized
FP4
fp4
conversational
8-bit precision
modelopt
Model card Files
xet
 Community
15

fix: declare NextN (MTP) layer 45 as BF16 in quantization ignore lists

#10
by lucifer1004 - opened
base: refs/heads/main
←
from: refs/pr/10
Discussion Files changed
+608
-604
Files changed (2) hide show
  1. config.json +463 -461
  2. hf_quant_config.json +145 -143
config.json CHANGED
@@ -1,469 +1,471 @@
1
  {
2
- "architectures": [
3
- "Glm5NextForConditionalGeneration"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "dtype": "bfloat16",
6
- "image_end_token_id": 154831,
7
- "image_start_token_id": 154830,
8
- "image_token_id": 154854,
9
- "model_type": "glm5_next",
10
- "text_config": {
11
- "attention_bias": false,
12
- "attention_dropout": 0.0,
13
- "base_model_ep_plan": {
14
- "layers.*.mlp.experts": "torch_a2a_experts",
15
- "layers.*.mlp.experts.down_proj": "grouped_gemm",
16
- "layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
17
- "layers.*.mlp.gate": "torch_a2a_router"
 
 
 
 
 
 
 
 
 
 
 
 
18
  },
19
- "bos_token_id": null,
20
- "dtype": "bfloat16",
21
- "eos_token_id": [
22
- 154820,
23
- 154827,
24
- 154829
25
- ],
26
- "first_k_dense_replace": 3,
27
- "hc_eps": 1e-06,
28
- "hc_mult": 4,
29
- "hc_sinkhorn_iters": 20,
30
- "head_dim": 0,
31
- "hidden_act": "silu",
32
- "hidden_size": 4096,
33
- "index_head_dim": 128,
34
- "index_kpool": 4,
35
- "index_kpool_always_select_tail": true,
36
- "index_kpool_compress": true,
37
- "index_n_heads": 32,
38
- "index_share_for_mtp_iteration": true,
39
- "index_topk": 2048,
40
- "indexer_rope_interleave": true,
41
- "indexer_types": [
42
- "full",
43
- "full",
44
- "full",
45
- "full",
46
- "full",
47
- "full",
48
- "full",
49
- "full",
50
- "full",
51
- "full",
52
- "full",
53
- "full",
54
- "full",
55
- "full",
56
- "full",
57
- "full",
58
- "full",
59
- "full",
60
- "full",
61
- "full",
62
- "full",
63
- "full",
64
- "full",
65
- "full",
66
- "full",
67
- "full",
68
- "full",
69
- "full",
70
- "full",
71
- "full",
72
- "full",
73
- "full",
74
- "full",
75
- "full",
76
- "full",
77
- "full",
78
- "full",
79
- "full",
80
- "full",
81
- "full",
82
- "full",
83
- "full",
84
- "full",
85
- "full",
86
- "full"
87
- ],
88
- "initializer_range": 0.02,
89
- "intermediate_size": 12288,
90
- "kv_lora_rank": 512,
91
- "layer_types": [
92
- "linear_attention",
93
- "linear_attention",
94
- "linear_attention",
95
- "deepseek_sparse_attention",
96
- "linear_attention",
97
- "linear_attention",
98
- "linear_attention",
99
- "deepseek_sparse_attention",
100
- "linear_attention",
101
- "linear_attention",
102
- "linear_attention",
103
- "deepseek_sparse_attention",
104
- "linear_attention",
105
- "linear_attention",
106
- "linear_attention",
107
- "deepseek_sparse_attention",
108
- "linear_attention",
109
- "linear_attention",
110
- "linear_attention",
111
- "deepseek_sparse_attention",
112
- "linear_attention",
113
- "linear_attention",
114
- "linear_attention",
115
- "deepseek_sparse_attention",
116
- "linear_attention",
117
- "linear_attention",
118
- "linear_attention",
119
- "deepseek_sparse_attention",
120
- "linear_attention",
121
- "linear_attention",
122
- "linear_attention",
123
- "deepseek_sparse_attention",
124
- "linear_attention",
125
- "linear_attention",
126
- "linear_attention",
127
- "deepseek_sparse_attention",
128
- "linear_attention",
129
- "linear_attention",
130
- "linear_attention",
131
- "deepseek_sparse_attention",
132
- "linear_attention",
133
- "linear_attention",
134
- "linear_attention",
135
- "deepseek_sparse_attention",
136
- "linear_attention"
137
- ],
138
- "linear_attn_config": {
139
- "full_attn_layers": [
140
- 3,
141
- 7,
142
- 11,
143
- 15,
144
- 19,
145
- 23,
146
- 27,
147
- 31,
148
- 35,
149
- 39,
150
- 43
151
- ],
152
- "gate_lower_bound": -5.0,
153
- "head_dim": 128,
154
- "kda_layers": [
155
- 0,
156
- 1,
157
- 2,
158
- 4,
159
- 5,
160
- 6,
161
- 8,
162
- 9,
163
- 10,
164
- 12,
165
- 13,
166
- 14,
167
- 16,
168
- 17,
169
- 18,
170
- 20,
171
- 21,
172
- 22,
173
- 24,
174
- 25,
175
- 26,
176
- 28,
177
- 29,
178
- 30,
179
- 32,
180
- 33,
181
- 34,
182
- 36,
183
- 37,
184
- 38,
185
- 40,
186
- 41,
187
- 42,
188
- 44
189
- ],
190
- "num_heads": 64,
191
- "short_conv_kernel_size": 4
192
  },
193
- "linear_conv_kernel_dim": 4,
194
- "linear_head_dim": 128,
195
- "linear_lower_bound": -5.0,
196
- "linear_num_heads": 64,
197
- "max_position_embeddings": 1048576,
198
- "mhc": true,
199
- "mla_use_nope": true,
200
- "mlp_layer_types": [
201
- "dense",
202
- "dense",
203
- "dense",
204
- "sparse",
205
- "sparse",
206
- "sparse",
207
- "sparse",
208
- "sparse",
209
- "sparse",
210
- "sparse",
211
- "sparse",
212
- "sparse",
213
- "sparse",
214
- "sparse",
215
- "sparse",
216
- "sparse",
217
- "sparse",
218
- "sparse",
219
- "sparse",
220
- "sparse",
221
- "sparse",
222
- "sparse",
223
- "sparse",
224
- "sparse",
225
- "sparse",
226
- "sparse",
227
- "sparse",
228
- "sparse",
229
- "sparse",
230
- "sparse",
231
- "sparse",
232
- "sparse",
233
- "sparse",
234
- "sparse",
235
- "sparse",
236
- "sparse",
237
- "sparse",
238
- "sparse",
239
- "sparse",
240
- "sparse",
241
- "sparse",
242
- "sparse",
243
- "sparse",
244
- "sparse",
245
- "sparse"
246
- ],
247
- "model_type": "glm5_next_text",
248
- "moe_intermediate_size": 2048,
249
- "moe_router_dtype": "float32",
250
- "n_group": 1,
251
- "n_routed_experts": 288,
252
- "n_shared_experts": 1,
253
- "norm_topk_prob": true,
254
- "num_attention_heads": 64,
255
- "num_experts_per_tok": 8,
256
- "num_hidden_layers": 45,
257
- "num_key_value_heads": 64,
258
- "num_nextn_predict_layers": 1,
259
- "output_router_logits": false,
260
- "pad_token_id": 154820,
261
- "q_lora_rank": 1536,
262
- "qk_head_dim": 256,
263
- "qk_nope_head_dim": 256,
264
- "qk_rope_head_dim": 0,
265
- "rms_norm_eps": 1e-05,
266
- "routed_scaling_factor": 2.5,
267
- "router_aux_loss_coef": 0.001,
268
- "scoring_func": "sigmoid",
269
- "swiglu_limit": 10.0,
270
- "tie_word_embeddings": false,
271
- "topk_group": 1,
272
- "topk_method": "noaux_tc",
273
- "use_cache": true,
274
- "v_head_dim": 256,
275
- "vocab_size": 154880
276
  },
277
- "tie_word_embeddings": false,
278
- "transformers_version": "5.16.1",
279
- "video_end_token_id": 154833,
280
- "video_start_token_id": 154832,
281
- "video_token_id": 154855,
282
- "vision_config": {
283
- "attention_bias": true,
284
- "attention_dropout": 0.0,
285
- "depth": 24,
286
- "dtype": "bfloat16",
287
- "hidden_act": "silu",
288
- "hidden_size": 1024,
289
- "image_size": 448,
290
- "in_channels": 3,
291
- "initializer_range": 0.02,
292
- "intermediate_size": 4096,
293
- "model_type": "glm5_next_vision",
294
- "num_heads": 16,
295
- "out_hidden_size": 4096,
296
- "patch_size": 14,
297
- "projection_intermediate_size": 10240,
298
- "rms_norm_eps": 1e-05,
299
- "spatial_merge_size": 2,
300
- "swiglu_limit": 10.0,
301
- "temporal_patch_size": 2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
302
  },
303
- "quantization_config": {
304
- "config_groups": {
305
- "group_0": {
306
- "input_activations": {
307
- "dynamic": false,
308
- "num_bits": 4,
309
- "type": "float",
310
- "group_size": 16
311
- },
312
- "weights": {
313
- "dynamic": false,
314
- "num_bits": 4,
315
- "type": "float",
316
- "group_size": 16
317
- },
318
- "targets": [
319
- "Linear"
320
- ]
321
- }
322
- },
323
- "ignore": [
324
- "lm_head",
325
- "model.language_model.embed_tokens",
326
- "model.language_model.layers.0.self_attn*",
327
- "model.language_model.layers.1.self_attn*",
328
- "model.language_model.layers.10.mlp.gate",
329
- "model.language_model.layers.10.mlp.shared_experts*",
330
- "model.language_model.layers.10.self_attn*",
331
- "model.language_model.layers.11.mlp.gate",
332
- "model.language_model.layers.11.mlp.shared_experts*",
333
- "model.language_model.layers.11.self_attn*",
334
- "model.language_model.layers.12.mlp.gate",
335
- "model.language_model.layers.12.mlp.shared_experts*",
336
- "model.language_model.layers.12.self_attn*",
337
- "model.language_model.layers.13.mlp.gate",
338
- "model.language_model.layers.13.mlp.shared_experts*",
339
- "model.language_model.layers.13.self_attn*",
340
- "model.language_model.layers.14.mlp.gate",
341
- "model.language_model.layers.14.mlp.shared_experts*",
342
- "model.language_model.layers.14.self_attn*",
343
- "model.language_model.layers.15.mlp.gate",
344
- "model.language_model.layers.15.mlp.shared_experts*",
345
- "model.language_model.layers.15.self_attn*",
346
- "model.language_model.layers.16.mlp.gate",
347
- "model.language_model.layers.16.mlp.shared_experts*",
348
- "model.language_model.layers.16.self_attn*",
349
- "model.language_model.layers.17.mlp.gate",
350
- "model.language_model.layers.17.mlp.shared_experts*",
351
- "model.language_model.layers.17.self_attn*",
352
- "model.language_model.layers.18.mlp.gate",
353
- "model.language_model.layers.18.mlp.shared_experts*",
354
- "model.language_model.layers.18.self_attn*",
355
- "model.language_model.layers.19.mlp.gate",
356
- "model.language_model.layers.19.mlp.shared_experts*",
357
- "model.language_model.layers.19.self_attn*",
358
- "model.language_model.layers.2.self_attn*",
359
- "model.language_model.layers.20.mlp.gate",
360
- "model.language_model.layers.20.mlp.shared_experts*",
361
- "model.language_model.layers.20.self_attn*",
362
- "model.language_model.layers.21.mlp.gate",
363
- "model.language_model.layers.21.mlp.shared_experts*",
364
- "model.language_model.layers.21.self_attn*",
365
- "model.language_model.layers.22.mlp.gate",
366
- "model.language_model.layers.22.mlp.shared_experts*",
367
- "model.language_model.layers.22.self_attn*",
368
- "model.language_model.layers.23.mlp.gate",
369
- "model.language_model.layers.23.mlp.shared_experts*",
370
- "model.language_model.layers.23.self_attn*",
371
- "model.language_model.layers.24.mlp.gate",
372
- "model.language_model.layers.24.mlp.shared_experts*",
373
- "model.language_model.layers.24.self_attn*",
374
- "model.language_model.layers.25.mlp.gate",
375
- "model.language_model.layers.25.mlp.shared_experts*",
376
- "model.language_model.layers.25.self_attn*",
377
- "model.language_model.layers.26.mlp.gate",
378
- "model.language_model.layers.26.mlp.shared_experts*",
379
- "model.language_model.layers.26.self_attn*",
380
- "model.language_model.layers.27.mlp.gate",
381
- "model.language_model.layers.27.mlp.shared_experts*",
382
- "model.language_model.layers.27.self_attn*",
383
- "model.language_model.layers.28.mlp.gate",
384
- "model.language_model.layers.28.mlp.shared_experts*",
385
- "model.language_model.layers.28.self_attn*",
386
- "model.language_model.layers.29.mlp.gate",
387
- "model.language_model.layers.29.mlp.shared_experts*",
388
- "model.language_model.layers.29.self_attn*",
389
- "model.language_model.layers.3.mlp.gate",
390
- "model.language_model.layers.3.mlp.shared_experts*",
391
- "model.language_model.layers.3.self_attn*",
392
- "model.language_model.layers.30.mlp.gate",
393
- "model.language_model.layers.30.mlp.shared_experts*",
394
- "model.language_model.layers.30.self_attn*",
395
- "model.language_model.layers.31.mlp.gate",
396
- "model.language_model.layers.31.mlp.shared_experts*",
397
- "model.language_model.layers.31.self_attn*",
398
- "model.language_model.layers.32.mlp.gate",
399
- "model.language_model.layers.32.mlp.shared_experts*",
400
- "model.language_model.layers.32.self_attn*",
401
- "model.language_model.layers.33.mlp.gate",
402
- "model.language_model.layers.33.mlp.shared_experts*",
403
- "model.language_model.layers.33.self_attn*",
404
- "model.language_model.layers.34.mlp.gate",
405
- "model.language_model.layers.34.mlp.shared_experts*",
406
- "model.language_model.layers.34.self_attn*",
407
- "model.language_model.layers.35.mlp.gate",
408
- "model.language_model.layers.35.mlp.shared_experts*",
409
- "model.language_model.layers.35.self_attn*",
410
- "model.language_model.layers.36.mlp.gate",
411
- "model.language_model.layers.36.mlp.shared_experts*",
412
- "model.language_model.layers.36.self_attn*",
413
- "model.language_model.layers.37.mlp.gate",
414
- "model.language_model.layers.37.mlp.shared_experts*",
415
- "model.language_model.layers.37.self_attn*",
416
- "model.language_model.layers.38.mlp.gate",
417
- "model.language_model.layers.38.mlp.shared_experts*",
418
- "model.language_model.layers.38.self_attn*",
419
- "model.language_model.layers.39.mlp.gate",
420
- "model.language_model.layers.39.mlp.shared_experts*",
421
- "model.language_model.layers.39.self_attn*",
422
- "model.language_model.layers.4.mlp.gate",
423
- "model.language_model.layers.4.mlp.shared_experts*",
424
- "model.language_model.layers.4.self_attn*",
425
- "model.language_model.layers.40.mlp.gate",
426
- "model.language_model.layers.40.mlp.shared_experts*",
427
- "model.language_model.layers.40.self_attn*",
428
- "model.language_model.layers.41.mlp.gate",
429
- "model.language_model.layers.41.mlp.shared_experts*",
430
- "model.language_model.layers.41.self_attn*",
431
- "model.language_model.layers.42.mlp.gate",
432
- "model.language_model.layers.42.mlp.shared_experts*",
433
- "model.language_model.layers.42.self_attn*",
434
- "model.language_model.layers.43.mlp.gate",
435
- "model.language_model.layers.43.mlp.shared_experts*",
436
- "model.language_model.layers.43.self_attn*",
437
- "model.language_model.layers.44.mlp.gate",
438
- "model.language_model.layers.44.mlp.shared_experts*",
439
- "model.language_model.layers.44.self_attn*",
440
- "model.language_model.layers.5.mlp.gate",
441
- "model.language_model.layers.5.mlp.shared_experts*",
442
- "model.language_model.layers.5.self_attn*",
443
- "model.language_model.layers.6.mlp.gate",
444
- "model.language_model.layers.6.mlp.shared_experts*",
445
- "model.language_model.layers.6.self_attn*",
446
- "model.language_model.layers.7.mlp.gate",
447
- "model.language_model.layers.7.mlp.shared_experts*",
448
- "model.language_model.layers.7.self_attn*",
449
- "model.language_model.layers.8.mlp.gate",
450
- "model.language_model.layers.8.mlp.shared_experts*",
451
- "model.language_model.layers.8.self_attn*",
452
- "model.language_model.layers.9.mlp.gate",
453
- "model.language_model.layers.9.mlp.shared_experts*",
454
- "model.language_model.layers.9.self_attn*",
455
- "model.visual*"
456
- ],
457
- "quant_algo": "NVFP4",
458
- "kv_cache_scheme": {
459
- "dynamic": false,
460
- "num_bits": 8,
461
- "type": "float"
462
- },
463
- "producer": {
464
- "name": "modelopt",
465
- "version": "0.47.0.dev393+ga4bc45b30.d20260828"
466
- },
467
- "quant_method": "modelopt"
468
- }
469
  }
 
1
  {
2
+ "architectures": [
3
+ "Glm5NextForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "image_end_token_id": 154831,
7
+ "image_start_token_id": 154830,
8
+ "image_token_id": 154854,
9
+ "model_type": "glm5_next",
10
+ "text_config": {
11
+ "attention_bias": false,
12
+ "attention_dropout": 0.0,
13
+ "base_model_ep_plan": {
14
+ "layers.*.mlp.experts": "torch_a2a_experts",
15
+ "layers.*.mlp.experts.down_proj": "grouped_gemm",
16
+ "layers.*.mlp.experts.gate_up_proj": "grouped_gemm",
17
+ "layers.*.mlp.gate": "torch_a2a_router"
18
+ },
19
+ "bos_token_id": null,
20
+ "dtype": "bfloat16",
21
+ "eos_token_id": [
22
+ 154820,
23
+ 154827,
24
+ 154829
25
+ ],
26
+ "first_k_dense_replace": 3,
27
+ "hc_eps": 1e-06,
28
+ "hc_mult": 4,
29
+ "hc_sinkhorn_iters": 20,
30
+ "head_dim": 0,
31
+ "hidden_act": "silu",
32
+ "hidden_size": 4096,
33
+ "index_head_dim": 128,
34
+ "index_kpool": 4,
35
+ "index_kpool_always_select_tail": true,
36
+ "index_kpool_compress": true,
37
+ "index_n_heads": 32,
38
+ "index_share_for_mtp_iteration": true,
39
+ "index_topk": 2048,
40
+ "indexer_rope_interleave": true,
41
+ "indexer_types": [
42
+ "full",
43
+ "full",
44
+ "full",
45
+ "full",
46
+ "full",
47
+ "full",
48
+ "full",
49
+ "full",
50
+ "full",
51
+ "full",
52
+ "full",
53
+ "full",
54
+ "full",
55
+ "full",
56
+ "full",
57
+ "full",
58
+ "full",
59
+ "full",
60
+ "full",
61
+ "full",
62
+ "full",
63
+ "full",
64
+ "full",
65
+ "full",
66
+ "full",
67
+ "full",
68
+ "full",
69
+ "full",
70
+ "full",
71
+ "full",
72
+ "full",
73
+ "full",
74
+ "full",
75
+ "full",
76
+ "full",
77
+ "full",
78
+ "full",
79
+ "full",
80
+ "full",
81
+ "full",
82
+ "full",
83
+ "full",
84
+ "full",
85
+ "full",
86
+ "full"
87
+ ],
88
+ "initializer_range": 0.02,
89
+ "intermediate_size": 12288,
90
+ "kv_lora_rank": 512,
91
+ "layer_types": [
92
+ "linear_attention",
93
+ "linear_attention",
94
+ "linear_attention",
95
+ "deepseek_sparse_attention",
96
+ "linear_attention",
97
+ "linear_attention",
98
+ "linear_attention",
99
+ "deepseek_sparse_attention",
100
+ "linear_attention",
101
+ "linear_attention",
102
+ "linear_attention",
103
+ "deepseek_sparse_attention",
104
+ "linear_attention",
105
+ "linear_attention",
106
+ "linear_attention",
107
+ "deepseek_sparse_attention",
108
+ "linear_attention",
109
+ "linear_attention",
110
+ "linear_attention",
111
+ "deepseek_sparse_attention",
112
+ "linear_attention",
113
+ "linear_attention",
114
+ "linear_attention",
115
+ "deepseek_sparse_attention",
116
+ "linear_attention",
117
+ "linear_attention",
118
+ "linear_attention",
119
+ "deepseek_sparse_attention",
120
+ "linear_attention",
121
+ "linear_attention",
122
+ "linear_attention",
123
+ "deepseek_sparse_attention",
124
+ "linear_attention",
125
+ "linear_attention",
126
+ "linear_attention",
127
+ "deepseek_sparse_attention",
128
+ "linear_attention",
129
+ "linear_attention",
130
+ "linear_attention",
131
+ "deepseek_sparse_attention",
132
+ "linear_attention",
133
+ "linear_attention",
134
+ "linear_attention",
135
+ "deepseek_sparse_attention",
136
+ "linear_attention"
137
  ],
138
+ "linear_attn_config": {
139
+ "full_attn_layers": [
140
+ 3,
141
+ 7,
142
+ 11,
143
+ 15,
144
+ 19,
145
+ 23,
146
+ 27,
147
+ 31,
148
+ 35,
149
+ 39,
150
+ 43
151
+ ],
152
+ "gate_lower_bound": -5.0,
153
+ "head_dim": 128,
154
+ "kda_layers": [
155
+ 0,
156
+ 1,
157
+ 2,
158
+ 4,
159
+ 5,
160
+ 6,
161
+ 8,
162
+ 9,
163
+ 10,
164
+ 12,
165
+ 13,
166
+ 14,
167
+ 16,
168
+ 17,
169
+ 18,
170
+ 20,
171
+ 21,
172
+ 22,
173
+ 24,
174
+ 25,
175
+ 26,
176
+ 28,
177
+ 29,
178
+ 30,
179
+ 32,
180
+ 33,
181
+ 34,
182
+ 36,
183
+ 37,
184
+ 38,
185
+ 40,
186
+ 41,
187
+ 42,
188
+ 44
189
+ ],
190
+ "num_heads": 64,
191
+ "short_conv_kernel_size": 4
192
+ },
193
+ "linear_conv_kernel_dim": 4,
194
+ "linear_head_dim": 128,
195
+ "linear_lower_bound": -5.0,
196
+ "linear_num_heads": 64,
197
+ "max_position_embeddings": 1048576,
198
+ "mhc": true,
199
+ "mla_use_nope": true,
200
+ "mlp_layer_types": [
201
+ "dense",
202
+ "dense",
203
+ "dense",
204
+ "sparse",
205
+ "sparse",
206
+ "sparse",
207
+ "sparse",
208
+ "sparse",
209
+ "sparse",
210
+ "sparse",
211
+ "sparse",
212
+ "sparse",
213
+ "sparse",
214
+ "sparse",
215
+ "sparse",
216
+ "sparse",
217
+ "sparse",
218
+ "sparse",
219
+ "sparse",
220
+ "sparse",
221
+ "sparse",
222
+ "sparse",
223
+ "sparse",
224
+ "sparse",
225
+ "sparse",
226
+ "sparse",
227
+ "sparse",
228
+ "sparse",
229
+ "sparse",
230
+ "sparse",
231
+ "sparse",
232
+ "sparse",
233
+ "sparse",
234
+ "sparse",
235
+ "sparse",
236
+ "sparse",
237
+ "sparse",
238
+ "sparse",
239
+ "sparse",
240
+ "sparse",
241
+ "sparse",
242
+ "sparse",
243
+ "sparse",
244
+ "sparse",
245
+ "sparse"
246
+ ],
247
+ "model_type": "glm5_next_text",
248
+ "moe_intermediate_size": 2048,
249
+ "moe_router_dtype": "float32",
250
+ "n_group": 1,
251
+ "n_routed_experts": 288,
252
+ "n_shared_experts": 1,
253
+ "norm_topk_prob": true,
254
+ "num_attention_heads": 64,
255
+ "num_experts_per_tok": 8,
256
+ "num_hidden_layers": 45,
257
+ "num_key_value_heads": 64,
258
+ "num_nextn_predict_layers": 1,
259
+ "output_router_logits": false,
260
+ "pad_token_id": 154820,
261
+ "q_lora_rank": 1536,
262
+ "qk_head_dim": 256,
263
+ "qk_nope_head_dim": 256,
264
+ "qk_rope_head_dim": 0,
265
+ "rms_norm_eps": 1e-05,
266
+ "routed_scaling_factor": 2.5,
267
+ "router_aux_loss_coef": 0.001,
268
+ "scoring_func": "sigmoid",
269
+ "swiglu_limit": 10.0,
270
+ "tie_word_embeddings": false,
271
+ "topk_group": 1,
272
+ "topk_method": "noaux_tc",
273
+ "use_cache": true,
274
+ "v_head_dim": 256,
275
+ "vocab_size": 154880
276
+ },
277
+ "tie_word_embeddings": false,
278
+ "transformers_version": "5.16.1",
279
+ "video_end_token_id": 154833,
280
+ "video_start_token_id": 154832,
281
+ "video_token_id": 154855,
282
+ "vision_config": {
283
+ "attention_bias": true,
284
+ "attention_dropout": 0.0,
285
+ "depth": 24,
286
  "dtype": "bfloat16",
287
+ "hidden_act": "silu",
288
+ "hidden_size": 1024,
289
+ "image_size": 448,
290
+ "in_channels": 3,
291
+ "initializer_range": 0.02,
292
+ "intermediate_size": 4096,
293
+ "model_type": "glm5_next_vision",
294
+ "num_heads": 16,
295
+ "out_hidden_size": 4096,
296
+ "patch_size": 14,
297
+ "projection_intermediate_size": 10240,
298
+ "rms_norm_eps": 1e-05,
299
+ "spatial_merge_size": 2,
300
+ "swiglu_limit": 10.0,
301
+ "temporal_patch_size": 2
302
+ },
303
+ "quantization_config": {
304
+ "config_groups": {
305
+ "group_0": {
306
+ "input_activations": {
307
+ "dynamic": false,
308
+ "num_bits": 4,
309
+ "type": "float",
310
+ "group_size": 16
311
  },
312
+ "weights": {
313
+ "dynamic": false,
314
+ "num_bits": 4,
315
+ "type": "float",
316
+ "group_size": 16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  },
318
+ "targets": [
319
+ "Linear"
320
+ ]
321
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
322
  },
323
+ "ignore": [
324
+ "lm_head",
325
+ "model.language_model.embed_tokens",
326
+ "model.language_model.layers.0.self_attn*",
327
+ "model.language_model.layers.1.self_attn*",
328
+ "model.language_model.layers.10.mlp.gate",
329
+ "model.language_model.layers.10.mlp.shared_experts*",
330
+ "model.language_model.layers.10.self_attn*",
331
+ "model.language_model.layers.11.mlp.gate",
332
+ "model.language_model.layers.11.mlp.shared_experts*",
333
+ "model.language_model.layers.11.self_attn*",
334
+ "model.language_model.layers.12.mlp.gate",
335
+ "model.language_model.layers.12.mlp.shared_experts*",
336
+ "model.language_model.layers.12.self_attn*",
337
+ "model.language_model.layers.13.mlp.gate",
338
+ "model.language_model.layers.13.mlp.shared_experts*",
339
+ "model.language_model.layers.13.self_attn*",
340
+ "model.language_model.layers.14.mlp.gate",
341
+ "model.language_model.layers.14.mlp.shared_experts*",
342
+ "model.language_model.layers.14.self_attn*",
343
+ "model.language_model.layers.15.mlp.gate",
344
+ "model.language_model.layers.15.mlp.shared_experts*",
345
+ "model.language_model.layers.15.self_attn*",
346
+ "model.language_model.layers.16.mlp.gate",
347
+ "model.language_model.layers.16.mlp.shared_experts*",
348
+ "model.language_model.layers.16.self_attn*",
349
+ "model.language_model.layers.17.mlp.gate",
350
+ "model.language_model.layers.17.mlp.shared_experts*",
351
+ "model.language_model.layers.17.self_attn*",
352
+ "model.language_model.layers.18.mlp.gate",
353
+ "model.language_model.layers.18.mlp.shared_experts*",
354
+ "model.language_model.layers.18.self_attn*",
355
+ "model.language_model.layers.19.mlp.gate",
356
+ "model.language_model.layers.19.mlp.shared_experts*",
357
+ "model.language_model.layers.19.self_attn*",
358
+ "model.language_model.layers.2.self_attn*",
359
+ "model.language_model.layers.20.mlp.gate",
360
+ "model.language_model.layers.20.mlp.shared_experts*",
361
+ "model.language_model.layers.20.self_attn*",
362
+ "model.language_model.layers.21.mlp.gate",
363
+ "model.language_model.layers.21.mlp.shared_experts*",
364
+ "model.language_model.layers.21.self_attn*",
365
+ "model.language_model.layers.22.mlp.gate",
366
+ "model.language_model.layers.22.mlp.shared_experts*",
367
+ "model.language_model.layers.22.self_attn*",
368
+ "model.language_model.layers.23.mlp.gate",
369
+ "model.language_model.layers.23.mlp.shared_experts*",
370
+ "model.language_model.layers.23.self_attn*",
371
+ "model.language_model.layers.24.mlp.gate",
372
+ "model.language_model.layers.24.mlp.shared_experts*",
373
+ "model.language_model.layers.24.self_attn*",
374
+ "model.language_model.layers.25.mlp.gate",
375
+ "model.language_model.layers.25.mlp.shared_experts*",
376
+ "model.language_model.layers.25.self_attn*",
377
+ "model.language_model.layers.26.mlp.gate",
378
+ "model.language_model.layers.26.mlp.shared_experts*",
379
+ "model.language_model.layers.26.self_attn*",
380
+ "model.language_model.layers.27.mlp.gate",
381
+ "model.language_model.layers.27.mlp.shared_experts*",
382
+ "model.language_model.layers.27.self_attn*",
383
+ "model.language_model.layers.28.mlp.gate",
384
+ "model.language_model.layers.28.mlp.shared_experts*",
385
+ "model.language_model.layers.28.self_attn*",
386
+ "model.language_model.layers.29.mlp.gate",
387
+ "model.language_model.layers.29.mlp.shared_experts*",
388
+ "model.language_model.layers.29.self_attn*",
389
+ "model.language_model.layers.3.mlp.gate",
390
+ "model.language_model.layers.3.mlp.shared_experts*",
391
+ "model.language_model.layers.3.self_attn*",
392
+ "model.language_model.layers.30.mlp.gate",
393
+ "model.language_model.layers.30.mlp.shared_experts*",
394
+ "model.language_model.layers.30.self_attn*",
395
+ "model.language_model.layers.31.mlp.gate",
396
+ "model.language_model.layers.31.mlp.shared_experts*",
397
+ "model.language_model.layers.31.self_attn*",
398
+ "model.language_model.layers.32.mlp.gate",
399
+ "model.language_model.layers.32.mlp.shared_experts*",
400
+ "model.language_model.layers.32.self_attn*",
401
+ "model.language_model.layers.33.mlp.gate",
402
+ "model.language_model.layers.33.mlp.shared_experts*",
403
+ "model.language_model.layers.33.self_attn*",
404
+ "model.language_model.layers.34.mlp.gate",
405
+ "model.language_model.layers.34.mlp.shared_experts*",
406
+ "model.language_model.layers.34.self_attn*",
407
+ "model.language_model.layers.35.mlp.gate",
408
+ "model.language_model.layers.35.mlp.shared_experts*",
409
+ "model.language_model.layers.35.self_attn*",
410
+ "model.language_model.layers.36.mlp.gate",
411
+ "model.language_model.layers.36.mlp.shared_experts*",
412
+ "model.language_model.layers.36.self_attn*",
413
+ "model.language_model.layers.37.mlp.gate",
414
+ "model.language_model.layers.37.mlp.shared_experts*",
415
+ "model.language_model.layers.37.self_attn*",
416
+ "model.language_model.layers.38.mlp.gate",
417
+ "model.language_model.layers.38.mlp.shared_experts*",
418
+ "model.language_model.layers.38.self_attn*",
419
+ "model.language_model.layers.39.mlp.gate",
420
+ "model.language_model.layers.39.mlp.shared_experts*",
421
+ "model.language_model.layers.39.self_attn*",
422
+ "model.language_model.layers.4.mlp.gate",
423
+ "model.language_model.layers.4.mlp.shared_experts*",
424
+ "model.language_model.layers.4.self_attn*",
425
+ "model.language_model.layers.40.mlp.gate",
426
+ "model.language_model.layers.40.mlp.shared_experts*",
427
+ "model.language_model.layers.40.self_attn*",
428
+ "model.language_model.layers.41.mlp.gate",
429
+ "model.language_model.layers.41.mlp.shared_experts*",
430
+ "model.language_model.layers.41.self_attn*",
431
+ "model.language_model.layers.42.mlp.gate",
432
+ "model.language_model.layers.42.mlp.shared_experts*",
433
+ "model.language_model.layers.42.self_attn*",
434
+ "model.language_model.layers.43.mlp.gate",
435
+ "model.language_model.layers.43.mlp.shared_experts*",
436
+ "model.language_model.layers.43.self_attn*",
437
+ "model.language_model.layers.44.mlp.gate",
438
+ "model.language_model.layers.44.mlp.shared_experts*",
439
+ "model.language_model.layers.44.self_attn*",
440
+ "model.language_model.layers.5.mlp.gate",
441
+ "model.language_model.layers.5.mlp.shared_experts*",
442
+ "model.language_model.layers.5.self_attn*",
443
+ "model.language_model.layers.6.mlp.gate",
444
+ "model.language_model.layers.6.mlp.shared_experts*",
445
+ "model.language_model.layers.6.self_attn*",
446
+ "model.language_model.layers.7.mlp.gate",
447
+ "model.language_model.layers.7.mlp.shared_experts*",
448
+ "model.language_model.layers.7.self_attn*",
449
+ "model.language_model.layers.8.mlp.gate",
450
+ "model.language_model.layers.8.mlp.shared_experts*",
451
+ "model.language_model.layers.8.self_attn*",
452
+ "model.language_model.layers.9.mlp.gate",
453
+ "model.language_model.layers.9.mlp.shared_experts*",
454
+ "model.language_model.layers.9.self_attn*",
455
+ "model.visual*",
456
+ "model.language_model.layers.45*",
457
+ "model.layers.45*"
458
+ ],
459
+ "quant_algo": "NVFP4",
460
+ "kv_cache_scheme": {
461
+ "dynamic": false,
462
+ "num_bits": 8,
463
+ "type": "float"
464
  },
465
+ "producer": {
466
+ "name": "modelopt",
467
+ "version": "0.47.0.dev393+ga4bc45b30.d20260828"
468
+ },
469
+ "quant_method": "modelopt"
470
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
471
  }
hf_quant_config.json CHANGED
@@ -1,145 +1,147 @@
1
  {
2
- "producer": {
3
- "name": "modelopt",
4
- "version": "0.47.0.dev393+ga4bc45b30.d20260828"
5
- },
6
- "quantization": {
7
- "quant_algo": "NVFP4",
8
- "kv_cache_quant_algo": "FP8",
9
- "group_size": 16,
10
- "exclude_modules": [
11
- "lm_head",
12
- "model.language_model.embed_tokens",
13
- "model.language_model.layers.0.self_attn*",
14
- "model.language_model.layers.1.self_attn*",
15
- "model.language_model.layers.10.mlp.gate",
16
- "model.language_model.layers.10.mlp.shared_experts*",
17
- "model.language_model.layers.10.self_attn*",
18
- "model.language_model.layers.11.mlp.gate",
19
- "model.language_model.layers.11.mlp.shared_experts*",
20
- "model.language_model.layers.11.self_attn*",
21
- "model.language_model.layers.12.mlp.gate",
22
- "model.language_model.layers.12.mlp.shared_experts*",
23
- "model.language_model.layers.12.self_attn*",
24
- "model.language_model.layers.13.mlp.gate",
25
- "model.language_model.layers.13.mlp.shared_experts*",
26
- "model.language_model.layers.13.self_attn*",
27
- "model.language_model.layers.14.mlp.gate",
28
- "model.language_model.layers.14.mlp.shared_experts*",
29
- "model.language_model.layers.14.self_attn*",
30
- "model.language_model.layers.15.mlp.gate",
31
- "model.language_model.layers.15.mlp.shared_experts*",
32
- "model.language_model.layers.15.self_attn*",
33
- "model.language_model.layers.16.mlp.gate",
34
- "model.language_model.layers.16.mlp.shared_experts*",
35
- "model.language_model.layers.16.self_attn*",
36
- "model.language_model.layers.17.mlp.gate",
37
- "model.language_model.layers.17.mlp.shared_experts*",
38
- "model.language_model.layers.17.self_attn*",
39
- "model.language_model.layers.18.mlp.gate",
40
- "model.language_model.layers.18.mlp.shared_experts*",
41
- "model.language_model.layers.18.self_attn*",
42
- "model.language_model.layers.19.mlp.gate",
43
- "model.language_model.layers.19.mlp.shared_experts*",
44
- "model.language_model.layers.19.self_attn*",
45
- "model.language_model.layers.2.self_attn*",
46
- "model.language_model.layers.20.mlp.gate",
47
- "model.language_model.layers.20.mlp.shared_experts*",
48
- "model.language_model.layers.20.self_attn*",
49
- "model.language_model.layers.21.mlp.gate",
50
- "model.language_model.layers.21.mlp.shared_experts*",
51
- "model.language_model.layers.21.self_attn*",
52
- "model.language_model.layers.22.mlp.gate",
53
- "model.language_model.layers.22.mlp.shared_experts*",
54
- "model.language_model.layers.22.self_attn*",
55
- "model.language_model.layers.23.mlp.gate",
56
- "model.language_model.layers.23.mlp.shared_experts*",
57
- "model.language_model.layers.23.self_attn*",
58
- "model.language_model.layers.24.mlp.gate",
59
- "model.language_model.layers.24.mlp.shared_experts*",
60
- "model.language_model.layers.24.self_attn*",
61
- "model.language_model.layers.25.mlp.gate",
62
- "model.language_model.layers.25.mlp.shared_experts*",
63
- "model.language_model.layers.25.self_attn*",
64
- "model.language_model.layers.26.mlp.gate",
65
- "model.language_model.layers.26.mlp.shared_experts*",
66
- "model.language_model.layers.26.self_attn*",
67
- "model.language_model.layers.27.mlp.gate",
68
- "model.language_model.layers.27.mlp.shared_experts*",
69
- "model.language_model.layers.27.self_attn*",
70
- "model.language_model.layers.28.mlp.gate",
71
- "model.language_model.layers.28.mlp.shared_experts*",
72
- "model.language_model.layers.28.self_attn*",
73
- "model.language_model.layers.29.mlp.gate",
74
- "model.language_model.layers.29.mlp.shared_experts*",
75
- "model.language_model.layers.29.self_attn*",
76
- "model.language_model.layers.3.mlp.gate",
77
- "model.language_model.layers.3.mlp.shared_experts*",
78
- "model.language_model.layers.3.self_attn*",
79
- "model.language_model.layers.30.mlp.gate",
80
- "model.language_model.layers.30.mlp.shared_experts*",
81
- "model.language_model.layers.30.self_attn*",
82
- "model.language_model.layers.31.mlp.gate",
83
- "model.language_model.layers.31.mlp.shared_experts*",
84
- "model.language_model.layers.31.self_attn*",
85
- "model.language_model.layers.32.mlp.gate",
86
- "model.language_model.layers.32.mlp.shared_experts*",
87
- "model.language_model.layers.32.self_attn*",
88
- "model.language_model.layers.33.mlp.gate",
89
- "model.language_model.layers.33.mlp.shared_experts*",
90
- "model.language_model.layers.33.self_attn*",
91
- "model.language_model.layers.34.mlp.gate",
92
- "model.language_model.layers.34.mlp.shared_experts*",
93
- "model.language_model.layers.34.self_attn*",
94
- "model.language_model.layers.35.mlp.gate",
95
- "model.language_model.layers.35.mlp.shared_experts*",
96
- "model.language_model.layers.35.self_attn*",
97
- "model.language_model.layers.36.mlp.gate",
98
- "model.language_model.layers.36.mlp.shared_experts*",
99
- "model.language_model.layers.36.self_attn*",
100
- "model.language_model.layers.37.mlp.gate",
101
- "model.language_model.layers.37.mlp.shared_experts*",
102
- "model.language_model.layers.37.self_attn*",
103
- "model.language_model.layers.38.mlp.gate",
104
- "model.language_model.layers.38.mlp.shared_experts*",
105
- "model.language_model.layers.38.self_attn*",
106
- "model.language_model.layers.39.mlp.gate",
107
- "model.language_model.layers.39.mlp.shared_experts*",
108
- "model.language_model.layers.39.self_attn*",
109
- "model.language_model.layers.4.mlp.gate",
110
- "model.language_model.layers.4.mlp.shared_experts*",
111
- "model.language_model.layers.4.self_attn*",
112
- "model.language_model.layers.40.mlp.gate",
113
- "model.language_model.layers.40.mlp.shared_experts*",
114
- "model.language_model.layers.40.self_attn*",
115
- "model.language_model.layers.41.mlp.gate",
116
- "model.language_model.layers.41.mlp.shared_experts*",
117
- "model.language_model.layers.41.self_attn*",
118
- "model.language_model.layers.42.mlp.gate",
119
- "model.language_model.layers.42.mlp.shared_experts*",
120
- "model.language_model.layers.42.self_attn*",
121
- "model.language_model.layers.43.mlp.gate",
122
- "model.language_model.layers.43.mlp.shared_experts*",
123
- "model.language_model.layers.43.self_attn*",
124
- "model.language_model.layers.44.mlp.gate",
125
- "model.language_model.layers.44.mlp.shared_experts*",
126
- "model.language_model.layers.44.self_attn*",
127
- "model.language_model.layers.5.mlp.gate",
128
- "model.language_model.layers.5.mlp.shared_experts*",
129
- "model.language_model.layers.5.self_attn*",
130
- "model.language_model.layers.6.mlp.gate",
131
- "model.language_model.layers.6.mlp.shared_experts*",
132
- "model.language_model.layers.6.self_attn*",
133
- "model.language_model.layers.7.mlp.gate",
134
- "model.language_model.layers.7.mlp.shared_experts*",
135
- "model.language_model.layers.7.self_attn*",
136
- "model.language_model.layers.8.mlp.gate",
137
- "model.language_model.layers.8.mlp.shared_experts*",
138
- "model.language_model.layers.8.self_attn*",
139
- "model.language_model.layers.9.mlp.gate",
140
- "model.language_model.layers.9.mlp.shared_experts*",
141
- "model.language_model.layers.9.self_attn*",
142
- "model.visual*"
143
- ]
144
- }
 
 
145
  }
 
1
  {
2
+ "producer": {
3
+ "name": "modelopt",
4
+ "version": "0.47.0.dev393+ga4bc45b30.d20260828"
5
+ },
6
+ "quantization": {
7
+ "quant_algo": "NVFP4",
8
+ "kv_cache_quant_algo": "FP8",
9
+ "group_size": 16,
10
+ "exclude_modules": [
11
+ "lm_head",
12
+ "model.language_model.embed_tokens",
13
+ "model.language_model.layers.0.self_attn*",
14
+ "model.language_model.layers.1.self_attn*",
15
+ "model.language_model.layers.10.mlp.gate",
16
+ "model.language_model.layers.10.mlp.shared_experts*",
17
+ "model.language_model.layers.10.self_attn*",
18
+ "model.language_model.layers.11.mlp.gate",
19
+ "model.language_model.layers.11.mlp.shared_experts*",
20
+ "model.language_model.layers.11.self_attn*",
21
+ "model.language_model.layers.12.mlp.gate",
22
+ "model.language_model.layers.12.mlp.shared_experts*",
23
+ "model.language_model.layers.12.self_attn*",
24
+ "model.language_model.layers.13.mlp.gate",
25
+ "model.language_model.layers.13.mlp.shared_experts*",
26
+ "model.language_model.layers.13.self_attn*",
27
+ "model.language_model.layers.14.mlp.gate",
28
+ "model.language_model.layers.14.mlp.shared_experts*",
29
+ "model.language_model.layers.14.self_attn*",
30
+ "model.language_model.layers.15.mlp.gate",
31
+ "model.language_model.layers.15.mlp.shared_experts*",
32
+ "model.language_model.layers.15.self_attn*",
33
+ "model.language_model.layers.16.mlp.gate",
34
+ "model.language_model.layers.16.mlp.shared_experts*",
35
+ "model.language_model.layers.16.self_attn*",
36
+ "model.language_model.layers.17.mlp.gate",
37
+ "model.language_model.layers.17.mlp.shared_experts*",
38
+ "model.language_model.layers.17.self_attn*",
39
+ "model.language_model.layers.18.mlp.gate",
40
+ "model.language_model.layers.18.mlp.shared_experts*",
41
+ "model.language_model.layers.18.self_attn*",
42
+ "model.language_model.layers.19.mlp.gate",
43
+ "model.language_model.layers.19.mlp.shared_experts*",
44
+ "model.language_model.layers.19.self_attn*",
45
+ "model.language_model.layers.2.self_attn*",
46
+ "model.language_model.layers.20.mlp.gate",
47
+ "model.language_model.layers.20.mlp.shared_experts*",
48
+ "model.language_model.layers.20.self_attn*",
49
+ "model.language_model.layers.21.mlp.gate",
50
+ "model.language_model.layers.21.mlp.shared_experts*",
51
+ "model.language_model.layers.21.self_attn*",
52
+ "model.language_model.layers.22.mlp.gate",
53
+ "model.language_model.layers.22.mlp.shared_experts*",
54
+ "model.language_model.layers.22.self_attn*",
55
+ "model.language_model.layers.23.mlp.gate",
56
+ "model.language_model.layers.23.mlp.shared_experts*",
57
+ "model.language_model.layers.23.self_attn*",
58
+ "model.language_model.layers.24.mlp.gate",
59
+ "model.language_model.layers.24.mlp.shared_experts*",
60
+ "model.language_model.layers.24.self_attn*",
61
+ "model.language_model.layers.25.mlp.gate",
62
+ "model.language_model.layers.25.mlp.shared_experts*",
63
+ "model.language_model.layers.25.self_attn*",
64
+ "model.language_model.layers.26.mlp.gate",
65
+ "model.language_model.layers.26.mlp.shared_experts*",
66
+ "model.language_model.layers.26.self_attn*",
67
+ "model.language_model.layers.27.mlp.gate",
68
+ "model.language_model.layers.27.mlp.shared_experts*",
69
+ "model.language_model.layers.27.self_attn*",
70
+ "model.language_model.layers.28.mlp.gate",
71
+ "model.language_model.layers.28.mlp.shared_experts*",
72
+ "model.language_model.layers.28.self_attn*",
73
+ "model.language_model.layers.29.mlp.gate",
74
+ "model.language_model.layers.29.mlp.shared_experts*",
75
+ "model.language_model.layers.29.self_attn*",
76
+ "model.language_model.layers.3.mlp.gate",
77
+ "model.language_model.layers.3.mlp.shared_experts*",
78
+ "model.language_model.layers.3.self_attn*",
79
+ "model.language_model.layers.30.mlp.gate",
80
+ "model.language_model.layers.30.mlp.shared_experts*",
81
+ "model.language_model.layers.30.self_attn*",
82
+ "model.language_model.layers.31.mlp.gate",
83
+ "model.language_model.layers.31.mlp.shared_experts*",
84
+ "model.language_model.layers.31.self_attn*",
85
+ "model.language_model.layers.32.mlp.gate",
86
+ "model.language_model.layers.32.mlp.shared_experts*",
87
+ "model.language_model.layers.32.self_attn*",
88
+ "model.language_model.layers.33.mlp.gate",
89
+ "model.language_model.layers.33.mlp.shared_experts*",
90
+ "model.language_model.layers.33.self_attn*",
91
+ "model.language_model.layers.34.mlp.gate",
92
+ "model.language_model.layers.34.mlp.shared_experts*",
93
+ "model.language_model.layers.34.self_attn*",
94
+ "model.language_model.layers.35.mlp.gate",
95
+ "model.language_model.layers.35.mlp.shared_experts*",
96
+ "model.language_model.layers.35.self_attn*",
97
+ "model.language_model.layers.36.mlp.gate",
98
+ "model.language_model.layers.36.mlp.shared_experts*",
99
+ "model.language_model.layers.36.self_attn*",
100
+ "model.language_model.layers.37.mlp.gate",
101
+ "model.language_model.layers.37.mlp.shared_experts*",
102
+ "model.language_model.layers.37.self_attn*",
103
+ "model.language_model.layers.38.mlp.gate",
104
+ "model.language_model.layers.38.mlp.shared_experts*",
105
+ "model.language_model.layers.38.self_attn*",
106
+ "model.language_model.layers.39.mlp.gate",
107
+ "model.language_model.layers.39.mlp.shared_experts*",
108
+ "model.language_model.layers.39.self_attn*",
109
+ "model.language_model.layers.4.mlp.gate",
110
+ "model.language_model.layers.4.mlp.shared_experts*",
111
+ "model.language_model.layers.4.self_attn*",
112
+ "model.language_model.layers.40.mlp.gate",
113
+ "model.language_model.layers.40.mlp.shared_experts*",
114
+ "model.language_model.layers.40.self_attn*",
115
+ "model.language_model.layers.41.mlp.gate",
116
+ "model.language_model.layers.41.mlp.shared_experts*",
117
+ "model.language_model.layers.41.self_attn*",
118
+ "model.language_model.layers.42.mlp.gate",
119
+ "model.language_model.layers.42.mlp.shared_experts*",
120
+ "model.language_model.layers.42.self_attn*",
121
+ "model.language_model.layers.43.mlp.gate",
122
+ "model.language_model.layers.43.mlp.shared_experts*",
123
+ "model.language_model.layers.43.self_attn*",
124
+ "model.language_model.layers.44.mlp.gate",
125
+ "model.language_model.layers.44.mlp.shared_experts*",
126
+ "model.language_model.layers.44.self_attn*",
127
+ "model.language_model.layers.5.mlp.gate",
128
+ "model.language_model.layers.5.mlp.shared_experts*",
129
+ "model.language_model.layers.5.self_attn*",
130
+ "model.language_model.layers.6.mlp.gate",
131
+ "model.language_model.layers.6.mlp.shared_experts*",
132
+ "model.language_model.layers.6.self_attn*",
133
+ "model.language_model.layers.7.mlp.gate",
134
+ "model.language_model.layers.7.mlp.shared_experts*",
135
+ "model.language_model.layers.7.self_attn*",
136
+ "model.language_model.layers.8.mlp.gate",
137
+ "model.language_model.layers.8.mlp.shared_experts*",
138
+ "model.language_model.layers.8.self_attn*",
139
+ "model.language_model.layers.9.mlp.gate",
140
+ "model.language_model.layers.9.mlp.shared_experts*",
141
+ "model.language_model.layers.9.self_attn*",
142
+ "model.visual*",
143
+ "model.language_model.layers.45*",
144
+ "model.layers.45*"
145
+ ]
146
+ }
147
  }