Hugging Face's logo

ukkathva
/
TelecomGPT-R1-27B-FP8-Dynamic

Text Generation
Transformers
Safetensors
English
qwen3_5
image-text-to-text
fp8
compressed-tensors
vllm
telecom
reasoning
conversational
Model card Files
xet
 Community
ukkathva commited on
Commit
69ceed6
·
verified ·
1 Parent(s): 8d16bb8

FP8 Dynamic checkpoint (identity-preserving)

Browse files
Files changed (10) hide show
  1. config.json +340 -104
  2. model-00001-of-00007.safetensors +3 -0
  3. model-00002-of-00007.safetensors +3 -0
  4. model-00003-of-00007.safetensors +3 -0
  5. model-00004-of-00007.safetensors +3 -0
  6. model-00005-of-00007.safetensors +3 -0
  7. model-00006-of-00007.safetensors +3 -0
  8. model-00007-of-00007.safetensors +3 -0
  9. model.safetensors.index.json +0 -0
  10. recipe.yaml +4 -1
config.json CHANGED
@@ -1,101 +1,14 @@
1
  {
2
  "architectures": [
3
- "Qwen3_5ForCausalLM"
4
  ],
5
- "attention_bias": false,
6
- "attention_dropout": 0.0,
7
- "attn_output_gate": true,
8
  "bos_token_id": null,
9
  "dtype": "bfloat16",
10
- "eos_token_id": 248044,
11
- "full_attention_interval": 4,
12
- "head_dim": 256,
13
- "hidden_act": "silu",
14
- "hidden_size": 5120,
15
- "initializer_range": 0.02,
16
- "intermediate_size": 17408,
17
- "layer_types": [
18
- "linear_attention",
19
- "linear_attention",
20
- "linear_attention",
21
- "full_attention",
22
- "linear_attention",
23
- "linear_attention",
24
- "linear_attention",
25
- "full_attention",
26
- "linear_attention",
27
- "linear_attention",
28
- "linear_attention",
29
- "full_attention",
30
- "linear_attention",
31
- "linear_attention",
32
- "linear_attention",
33
- "full_attention",
34
- "linear_attention",
35
- "linear_attention",
36
- "linear_attention",
37
- "full_attention",
38
- "linear_attention",
39
- "linear_attention",
40
- "linear_attention",
41
- "full_attention",
42
- "linear_attention",
43
- "linear_attention",
44
- "linear_attention",
45
- "full_attention",
46
- "linear_attention",
47
- "linear_attention",
48
- "linear_attention",
49
- "full_attention",
50
- "linear_attention",
51
- "linear_attention",
52
- "linear_attention",
53
- "full_attention",
54
- "linear_attention",
55
- "linear_attention",
56
- "linear_attention",
57
- "full_attention",
58
- "linear_attention",
59
- "linear_attention",
60
- "linear_attention",
61
- "full_attention",
62
- "linear_attention",
63
- "linear_attention",
64
- "linear_attention",
65
- "full_attention",
66
- "linear_attention",
67
- "linear_attention",
68
- "linear_attention",
69
- "full_attention",
70
- "linear_attention",
71
- "linear_attention",
72
- "linear_attention",
73
- "full_attention",
74
- "linear_attention",
75
- "linear_attention",
76
- "linear_attention",
77
- "full_attention",
78
- "linear_attention",
79
- "linear_attention",
80
- "linear_attention",
81
- "full_attention"
82
- ],
83
- "linear_conv_kernel_dim": 4,
84
- "linear_key_head_dim": 128,
85
- "linear_num_key_heads": 16,
86
- "linear_num_value_heads": 48,
87
- "linear_value_head_dim": 128,
88
- "mamba_ssm_dtype": "float32",
89
- "max_position_embeddings": 262144,
90
- "mlp_only_layers": [],
91
- "model_type": "qwen3_5_text",
92
- "mtp_num_hidden_layers": 0,
93
- "mtp_use_dedicated_embeddings": false,
94
- "num_attention_heads": 24,
95
- "num_hidden_layers": 64,
96
- "num_key_value_heads": 4,
97
- "pad_token_id": null,
98
- "partial_rotary_factor": 0.25,
99
  "quantization_config": {
100
  "config_groups": {
101
  "group_0": {
@@ -137,102 +50,308 @@
137
  "format": "float-quantized",
138
  "global_compression_ratio": null,
139
  "ignore": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  "model.language_model.layers.0.linear_attn",
141
  "model.language_model.layers.0.linear_attn.norm",
 
 
142
  "model.language_model.layers.1.linear_attn",
143
  "model.language_model.layers.1.linear_attn.norm",
 
 
144
  "model.language_model.layers.2.linear_attn",
145
  "model.language_model.layers.2.linear_attn.norm",
 
 
146
  "model.language_model.layers.4.linear_attn",
147
  "model.language_model.layers.4.linear_attn.norm",
 
 
148
  "model.language_model.layers.5.linear_attn",
149
  "model.language_model.layers.5.linear_attn.norm",
 
 
150
  "model.language_model.layers.6.linear_attn",
151
  "model.language_model.layers.6.linear_attn.norm",
 
 
152
  "model.language_model.layers.8.linear_attn",
153
  "model.language_model.layers.8.linear_attn.norm",
 
 
154
  "model.language_model.layers.9.linear_attn",
155
  "model.language_model.layers.9.linear_attn.norm",
 
 
156
  "model.language_model.layers.10.linear_attn",
157
  "model.language_model.layers.10.linear_attn.norm",
 
 
158
  "model.language_model.layers.12.linear_attn",
159
  "model.language_model.layers.12.linear_attn.norm",
 
 
160
  "model.language_model.layers.13.linear_attn",
161
  "model.language_model.layers.13.linear_attn.norm",
 
 
162
  "model.language_model.layers.14.linear_attn",
163
  "model.language_model.layers.14.linear_attn.norm",
 
 
164
  "model.language_model.layers.16.linear_attn",
165
  "model.language_model.layers.16.linear_attn.norm",
 
 
166
  "model.language_model.layers.17.linear_attn",
167
  "model.language_model.layers.17.linear_attn.norm",
 
 
168
  "model.language_model.layers.18.linear_attn",
169
  "model.language_model.layers.18.linear_attn.norm",
 
 
170
  "model.language_model.layers.20.linear_attn",
171
  "model.language_model.layers.20.linear_attn.norm",
 
 
172
  "model.language_model.layers.21.linear_attn",
173
  "model.language_model.layers.21.linear_attn.norm",
 
 
174
  "model.language_model.layers.22.linear_attn",
175
  "model.language_model.layers.22.linear_attn.norm",
 
 
176
  "model.language_model.layers.24.linear_attn",
177
  "model.language_model.layers.24.linear_attn.norm",
 
 
178
  "model.language_model.layers.25.linear_attn",
179
  "model.language_model.layers.25.linear_attn.norm",
 
 
180
  "model.language_model.layers.26.linear_attn",
181
  "model.language_model.layers.26.linear_attn.norm",
 
 
182
  "model.language_model.layers.28.linear_attn",
183
  "model.language_model.layers.28.linear_attn.norm",
 
 
184
  "model.language_model.layers.29.linear_attn",
185
  "model.language_model.layers.29.linear_attn.norm",
 
 
186
  "model.language_model.layers.30.linear_attn",
187
  "model.language_model.layers.30.linear_attn.norm",
 
 
188
  "model.language_model.layers.32.linear_attn",
189
  "model.language_model.layers.32.linear_attn.norm",
 
 
190
  "model.language_model.layers.33.linear_attn",
191
  "model.language_model.layers.33.linear_attn.norm",
 
 
192
  "model.language_model.layers.34.linear_attn",
193
  "model.language_model.layers.34.linear_attn.norm",
 
 
194
  "model.language_model.layers.36.linear_attn",
195
  "model.language_model.layers.36.linear_attn.norm",
 
 
196
  "model.language_model.layers.37.linear_attn",
197
  "model.language_model.layers.37.linear_attn.norm",
 
 
198
  "model.language_model.layers.38.linear_attn",
199
  "model.language_model.layers.38.linear_attn.norm",
 
 
200
  "model.language_model.layers.40.linear_attn",
201
  "model.language_model.layers.40.linear_attn.norm",
 
 
202
  "model.language_model.layers.41.linear_attn",
203
  "model.language_model.layers.41.linear_attn.norm",
 
 
204
  "model.language_model.layers.42.linear_attn",
205
  "model.language_model.layers.42.linear_attn.norm",
 
 
206
  "model.language_model.layers.44.linear_attn",
207
  "model.language_model.layers.44.linear_attn.norm",
 
 
208
  "model.language_model.layers.45.linear_attn",
209
  "model.language_model.layers.45.linear_attn.norm",
 
 
210
  "model.language_model.layers.46.linear_attn",
211
  "model.language_model.layers.46.linear_attn.norm",
 
 
212
  "model.language_model.layers.48.linear_attn",
213
  "model.language_model.layers.48.linear_attn.norm",
 
 
214
  "model.language_model.layers.49.linear_attn",
215
  "model.language_model.layers.49.linear_attn.norm",
 
 
216
  "model.language_model.layers.50.linear_attn",
217
  "model.language_model.layers.50.linear_attn.norm",
 
 
218
  "model.language_model.layers.52.linear_attn",
219
  "model.language_model.layers.52.linear_attn.norm",
 
 
220
  "model.language_model.layers.53.linear_attn",
221
  "model.language_model.layers.53.linear_attn.norm",
 
 
222
  "model.language_model.layers.54.linear_attn",
223
  "model.language_model.layers.54.linear_attn.norm",
 
 
224
  "model.language_model.layers.56.linear_attn",
225
  "model.language_model.layers.56.linear_attn.norm",
 
 
226
  "model.language_model.layers.57.linear_attn",
227
  "model.language_model.layers.57.linear_attn.norm",
 
 
228
  "model.language_model.layers.58.linear_attn",
229
  "model.language_model.layers.58.linear_attn.norm",
 
 
230
  "model.language_model.layers.60.linear_attn",
231
  "model.language_model.layers.60.linear_attn.norm",
 
 
232
  "model.language_model.layers.61.linear_attn",
233
  "model.language_model.layers.61.linear_attn.norm",
 
 
234
  "model.language_model.layers.62.linear_attn",
235
  "model.language_model.layers.62.linear_attn.norm",
 
 
236
  "lm_head"
237
  ],
238
  "kv_cache_scheme": null,
@@ -242,20 +361,137 @@
242
  "transform_config": {},
243
  "version": "0.18.0"
244
  },
245
- "rms_norm_eps": 1e-06,
246
- "rope_parameters": {
247
- "mrope_interleaved": true,
248
- "mrope_section": [
249
- 11,
250
- 11,
251
- 10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
  "partial_rotary_factor": 0.25,
254
- "rope_theta": 10000000,
255
- "rope_type": "default"
 
 
 
 
 
 
 
 
 
 
 
 
 
256
  },
257
  "tie_word_embeddings": false,
258
  "transformers_version": "5.14.1",
259
- "use_cache": true,
260
- "vocab_size": 248320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  }
 
1
  {
2
  "architectures": [
3
+ "Qwen3_5ForConditionalGeneration"
4
  ],
 
 
 
5
  "bos_token_id": null,
6
  "dtype": "bfloat16",
7
+ "eos_token_id": 248046,
8
+ "image_token_id": 248056,
9
+ "max_position_embeddings": 16384,
10
+ "model_type": "qwen3_5",
11
+ "pad_token_id": 248044,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  "quantization_config": {
13
  "config_groups": {
14
  "group_0": {
 
50
  "format": "float-quantized",
51
  "global_compression_ratio": null,
52
  "ignore": [
53
+ "model.visual.blocks.0.attn.qkv",
54
+ "model.visual.blocks.0.attn.proj",
55
+ "model.visual.blocks.0.mlp.linear_fc1",
56
+ "model.visual.blocks.0.mlp.linear_fc2",
57
+ "model.visual.blocks.1.attn.qkv",
58
+ "model.visual.blocks.1.attn.proj",
59
+ "model.visual.blocks.1.mlp.linear_fc1",
60
+ "model.visual.blocks.1.mlp.linear_fc2",
61
+ "model.visual.blocks.2.attn.qkv",
62
+ "model.visual.blocks.2.attn.proj",
63
+ "model.visual.blocks.2.mlp.linear_fc1",
64
+ "model.visual.blocks.2.mlp.linear_fc2",
65
+ "model.visual.blocks.3.attn.qkv",
66
+ "model.visual.blocks.3.attn.proj",
67
+ "model.visual.blocks.3.mlp.linear_fc1",
68
+ "model.visual.blocks.3.mlp.linear_fc2",
69
+ "model.visual.blocks.4.attn.qkv",
70
+ "model.visual.blocks.4.attn.proj",
71
+ "model.visual.blocks.4.mlp.linear_fc1",
72
+ "model.visual.blocks.4.mlp.linear_fc2",
73
+ "model.visual.blocks.5.attn.qkv",
74
+ "model.visual.blocks.5.attn.proj",
75
+ "model.visual.blocks.5.mlp.linear_fc1",
76
+ "model.visual.blocks.5.mlp.linear_fc2",
77
+ "model.visual.blocks.6.attn.qkv",
78
+ "model.visual.blocks.6.attn.proj",
79
+ "model.visual.blocks.6.mlp.linear_fc1",
80
+ "model.visual.blocks.6.mlp.linear_fc2",
81
+ "model.visual.blocks.7.attn.qkv",
82
+ "model.visual.blocks.7.attn.proj",
83
+ "model.visual.blocks.7.mlp.linear_fc1",
84
+ "model.visual.blocks.7.mlp.linear_fc2",
85
+ "model.visual.blocks.8.attn.qkv",
86
+ "model.visual.blocks.8.attn.proj",
87
+ "model.visual.blocks.8.mlp.linear_fc1",
88
+ "model.visual.blocks.8.mlp.linear_fc2",
89
+ "model.visual.blocks.9.attn.qkv",
90
+ "model.visual.blocks.9.attn.proj",
91
+ "model.visual.blocks.9.mlp.linear_fc1",
92
+ "model.visual.blocks.9.mlp.linear_fc2",
93
+ "model.visual.blocks.10.attn.qkv",
94
+ "model.visual.blocks.10.attn.proj",
95
+ "model.visual.blocks.10.mlp.linear_fc1",
96
+ "model.visual.blocks.10.mlp.linear_fc2",
97
+ "model.visual.blocks.11.attn.qkv",
98
+ "model.visual.blocks.11.attn.proj",
99
+ "model.visual.blocks.11.mlp.linear_fc1",
100
+ "model.visual.blocks.11.mlp.linear_fc2",
101
+ "model.visual.blocks.12.attn.qkv",
102
+ "model.visual.blocks.12.attn.proj",
103
+ "model.visual.blocks.12.mlp.linear_fc1",
104
+ "model.visual.blocks.12.mlp.linear_fc2",
105
+ "model.visual.blocks.13.attn.qkv",
106
+ "model.visual.blocks.13.attn.proj",
107
+ "model.visual.blocks.13.mlp.linear_fc1",
108
+ "model.visual.blocks.13.mlp.linear_fc2",
109
+ "model.visual.blocks.14.attn.qkv",
110
+ "model.visual.blocks.14.attn.proj",
111
+ "model.visual.blocks.14.mlp.linear_fc1",
112
+ "model.visual.blocks.14.mlp.linear_fc2",
113
+ "model.visual.blocks.15.attn.qkv",
114
+ "model.visual.blocks.15.attn.proj",
115
+ "model.visual.blocks.15.mlp.linear_fc1",
116
+ "model.visual.blocks.15.mlp.linear_fc2",
117
+ "model.visual.blocks.16.attn.qkv",
118
+ "model.visual.blocks.16.attn.proj",
119
+ "model.visual.blocks.16.mlp.linear_fc1",
120
+ "model.visual.blocks.16.mlp.linear_fc2",
121
+ "model.visual.blocks.17.attn.qkv",
122
+ "model.visual.blocks.17.attn.proj",
123
+ "model.visual.blocks.17.mlp.linear_fc1",
124
+ "model.visual.blocks.17.mlp.linear_fc2",
125
+ "model.visual.blocks.18.attn.qkv",
126
+ "model.visual.blocks.18.attn.proj",
127
+ "model.visual.blocks.18.mlp.linear_fc1",
128
+ "model.visual.blocks.18.mlp.linear_fc2",
129
+ "model.visual.blocks.19.attn.qkv",
130
+ "model.visual.blocks.19.attn.proj",
131
+ "model.visual.blocks.19.mlp.linear_fc1",
132
+ "model.visual.blocks.19.mlp.linear_fc2",
133
+ "model.visual.blocks.20.attn.qkv",
134
+ "model.visual.blocks.20.attn.proj",
135
+ "model.visual.blocks.20.mlp.linear_fc1",
136
+ "model.visual.blocks.20.mlp.linear_fc2",
137
+ "model.visual.blocks.21.attn.qkv",
138
+ "model.visual.blocks.21.attn.proj",
139
+ "model.visual.blocks.21.mlp.linear_fc1",
140
+ "model.visual.blocks.21.mlp.linear_fc2",
141
+ "model.visual.blocks.22.attn.qkv",
142
+ "model.visual.blocks.22.attn.proj",
143
+ "model.visual.blocks.22.mlp.linear_fc1",
144
+ "model.visual.blocks.22.mlp.linear_fc2",
145
+ "model.visual.blocks.23.attn.qkv",
146
+ "model.visual.blocks.23.attn.proj",
147
+ "model.visual.blocks.23.mlp.linear_fc1",
148
+ "model.visual.blocks.23.mlp.linear_fc2",
149
+ "model.visual.blocks.24.attn.qkv",
150
+ "model.visual.blocks.24.attn.proj",
151
+ "model.visual.blocks.24.mlp.linear_fc1",
152
+ "model.visual.blocks.24.mlp.linear_fc2",
153
+ "model.visual.blocks.25.attn.qkv",
154
+ "model.visual.blocks.25.attn.proj",
155
+ "model.visual.blocks.25.mlp.linear_fc1",
156
+ "model.visual.blocks.25.mlp.linear_fc2",
157
+ "model.visual.blocks.26.attn.qkv",
158
+ "model.visual.blocks.26.attn.proj",
159
+ "model.visual.blocks.26.mlp.linear_fc1",
160
+ "model.visual.blocks.26.mlp.linear_fc2",
161
+ "model.visual.merger.linear_fc1",
162
+ "model.visual.merger.linear_fc2",
163
  "model.language_model.layers.0.linear_attn",
164
  "model.language_model.layers.0.linear_attn.norm",
165
+ "model.language_model.layers.0.linear_attn.in_proj_b",
166
+ "model.language_model.layers.0.linear_attn.in_proj_a",
167
  "model.language_model.layers.1.linear_attn",
168
  "model.language_model.layers.1.linear_attn.norm",
169
+ "model.language_model.layers.1.linear_attn.in_proj_b",
170
+ "model.language_model.layers.1.linear_attn.in_proj_a",
171
  "model.language_model.layers.2.linear_attn",
172
  "model.language_model.layers.2.linear_attn.norm",
173
+ "model.language_model.layers.2.linear_attn.in_proj_b",
174
+ "model.language_model.layers.2.linear_attn.in_proj_a",
175
  "model.language_model.layers.4.linear_attn",
176
  "model.language_model.layers.4.linear_attn.norm",
177
+ "model.language_model.layers.4.linear_attn.in_proj_b",
178
+ "model.language_model.layers.4.linear_attn.in_proj_a",
179
  "model.language_model.layers.5.linear_attn",
180
  "model.language_model.layers.5.linear_attn.norm",
181
+ "model.language_model.layers.5.linear_attn.in_proj_b",
182
+ "model.language_model.layers.5.linear_attn.in_proj_a",
183
  "model.language_model.layers.6.linear_attn",
184
  "model.language_model.layers.6.linear_attn.norm",
185
+ "model.language_model.layers.6.linear_attn.in_proj_b",
186
+ "model.language_model.layers.6.linear_attn.in_proj_a",
187
  "model.language_model.layers.8.linear_attn",
188
  "model.language_model.layers.8.linear_attn.norm",
189
+ "model.language_model.layers.8.linear_attn.in_proj_b",
190
+ "model.language_model.layers.8.linear_attn.in_proj_a",
191
  "model.language_model.layers.9.linear_attn",
192
  "model.language_model.layers.9.linear_attn.norm",
193
+ "model.language_model.layers.9.linear_attn.in_proj_b",
194
+ "model.language_model.layers.9.linear_attn.in_proj_a",
195
  "model.language_model.layers.10.linear_attn",
196
  "model.language_model.layers.10.linear_attn.norm",
197
+ "model.language_model.layers.10.linear_attn.in_proj_b",
198
+ "model.language_model.layers.10.linear_attn.in_proj_a",
199
  "model.language_model.layers.12.linear_attn",
200
  "model.language_model.layers.12.linear_attn.norm",
201
+ "model.language_model.layers.12.linear_attn.in_proj_b",
202
+ "model.language_model.layers.12.linear_attn.in_proj_a",
203
  "model.language_model.layers.13.linear_attn",
204
  "model.language_model.layers.13.linear_attn.norm",
205
+ "model.language_model.layers.13.linear_attn.in_proj_b",
206
+ "model.language_model.layers.13.linear_attn.in_proj_a",
207
  "model.language_model.layers.14.linear_attn",
208
  "model.language_model.layers.14.linear_attn.norm",
209
+ "model.language_model.layers.14.linear_attn.in_proj_b",
210
+ "model.language_model.layers.14.linear_attn.in_proj_a",
211
  "model.language_model.layers.16.linear_attn",
212
  "model.language_model.layers.16.linear_attn.norm",
213
+ "model.language_model.layers.16.linear_attn.in_proj_b",
214
+ "model.language_model.layers.16.linear_attn.in_proj_a",
215
  "model.language_model.layers.17.linear_attn",
216
  "model.language_model.layers.17.linear_attn.norm",
217
+ "model.language_model.layers.17.linear_attn.in_proj_b",
218
+ "model.language_model.layers.17.linear_attn.in_proj_a",
219
  "model.language_model.layers.18.linear_attn",
220
  "model.language_model.layers.18.linear_attn.norm",
221
+ "model.language_model.layers.18.linear_attn.in_proj_b",
222
+ "model.language_model.layers.18.linear_attn.in_proj_a",
223
  "model.language_model.layers.20.linear_attn",
224
  "model.language_model.layers.20.linear_attn.norm",
225
+ "model.language_model.layers.20.linear_attn.in_proj_b",
226
+ "model.language_model.layers.20.linear_attn.in_proj_a",
227
  "model.language_model.layers.21.linear_attn",
228
  "model.language_model.layers.21.linear_attn.norm",
229
+ "model.language_model.layers.21.linear_attn.in_proj_b",
230
+ "model.language_model.layers.21.linear_attn.in_proj_a",
231
  "model.language_model.layers.22.linear_attn",
232
  "model.language_model.layers.22.linear_attn.norm",
233
+ "model.language_model.layers.22.linear_attn.in_proj_b",
234
+ "model.language_model.layers.22.linear_attn.in_proj_a",
235
  "model.language_model.layers.24.linear_attn",
236
  "model.language_model.layers.24.linear_attn.norm",
237
+ "model.language_model.layers.24.linear_attn.in_proj_b",
238
+ "model.language_model.layers.24.linear_attn.in_proj_a",
239
  "model.language_model.layers.25.linear_attn",
240
  "model.language_model.layers.25.linear_attn.norm",
241
+ "model.language_model.layers.25.linear_attn.in_proj_b",
242
+ "model.language_model.layers.25.linear_attn.in_proj_a",
243
  "model.language_model.layers.26.linear_attn",
244
  "model.language_model.layers.26.linear_attn.norm",
245
+ "model.language_model.layers.26.linear_attn.in_proj_b",
246
+ "model.language_model.layers.26.linear_attn.in_proj_a",
247
  "model.language_model.layers.28.linear_attn",
248
  "model.language_model.layers.28.linear_attn.norm",
249
+ "model.language_model.layers.28.linear_attn.in_proj_b",
250
+ "model.language_model.layers.28.linear_attn.in_proj_a",
251
  "model.language_model.layers.29.linear_attn",
252
  "model.language_model.layers.29.linear_attn.norm",
253
+ "model.language_model.layers.29.linear_attn.in_proj_b",
254
+ "model.language_model.layers.29.linear_attn.in_proj_a",
255
  "model.language_model.layers.30.linear_attn",
256
  "model.language_model.layers.30.linear_attn.norm",
257
+ "model.language_model.layers.30.linear_attn.in_proj_b",
258
+ "model.language_model.layers.30.linear_attn.in_proj_a",
259
  "model.language_model.layers.32.linear_attn",
260
  "model.language_model.layers.32.linear_attn.norm",
261
+ "model.language_model.layers.32.linear_attn.in_proj_b",
262
+ "model.language_model.layers.32.linear_attn.in_proj_a",
263
  "model.language_model.layers.33.linear_attn",
264
  "model.language_model.layers.33.linear_attn.norm",
265
+ "model.language_model.layers.33.linear_attn.in_proj_b",
266
+ "model.language_model.layers.33.linear_attn.in_proj_a",
267
  "model.language_model.layers.34.linear_attn",
268
  "model.language_model.layers.34.linear_attn.norm",
269
+ "model.language_model.layers.34.linear_attn.in_proj_b",
270
+ "model.language_model.layers.34.linear_attn.in_proj_a",
271
  "model.language_model.layers.36.linear_attn",
272
  "model.language_model.layers.36.linear_attn.norm",
273
+ "model.language_model.layers.36.linear_attn.in_proj_b",
274
+ "model.language_model.layers.36.linear_attn.in_proj_a",
275
  "model.language_model.layers.37.linear_attn",
276
  "model.language_model.layers.37.linear_attn.norm",
277
+ "model.language_model.layers.37.linear_attn.in_proj_b",
278
+ "model.language_model.layers.37.linear_attn.in_proj_a",
279
  "model.language_model.layers.38.linear_attn",
280
  "model.language_model.layers.38.linear_attn.norm",
281
+ "model.language_model.layers.38.linear_attn.in_proj_b",
282
+ "model.language_model.layers.38.linear_attn.in_proj_a",
283
  "model.language_model.layers.40.linear_attn",
284
  "model.language_model.layers.40.linear_attn.norm",
285
+ "model.language_model.layers.40.linear_attn.in_proj_b",
286
+ "model.language_model.layers.40.linear_attn.in_proj_a",
287
  "model.language_model.layers.41.linear_attn",
288
  "model.language_model.layers.41.linear_attn.norm",
289
+ "model.language_model.layers.41.linear_attn.in_proj_b",
290
+ "model.language_model.layers.41.linear_attn.in_proj_a",
291
  "model.language_model.layers.42.linear_attn",
292
  "model.language_model.layers.42.linear_attn.norm",
293
+ "model.language_model.layers.42.linear_attn.in_proj_b",
294
+ "model.language_model.layers.42.linear_attn.in_proj_a",
295
  "model.language_model.layers.44.linear_attn",
296
  "model.language_model.layers.44.linear_attn.norm",
297
+ "model.language_model.layers.44.linear_attn.in_proj_b",
298
+ "model.language_model.layers.44.linear_attn.in_proj_a",
299
  "model.language_model.layers.45.linear_attn",
300
  "model.language_model.layers.45.linear_attn.norm",
301
+ "model.language_model.layers.45.linear_attn.in_proj_b",
302
+ "model.language_model.layers.45.linear_attn.in_proj_a",
303
  "model.language_model.layers.46.linear_attn",
304
  "model.language_model.layers.46.linear_attn.norm",
305
+ "model.language_model.layers.46.linear_attn.in_proj_b",
306
+ "model.language_model.layers.46.linear_attn.in_proj_a",
307
  "model.language_model.layers.48.linear_attn",
308
  "model.language_model.layers.48.linear_attn.norm",
309
+ "model.language_model.layers.48.linear_attn.in_proj_b",
310
+ "model.language_model.layers.48.linear_attn.in_proj_a",
311
  "model.language_model.layers.49.linear_attn",
312
  "model.language_model.layers.49.linear_attn.norm",
313
+ "model.language_model.layers.49.linear_attn.in_proj_b",
314
+ "model.language_model.layers.49.linear_attn.in_proj_a",
315
  "model.language_model.layers.50.linear_attn",
316
  "model.language_model.layers.50.linear_attn.norm",
317
+ "model.language_model.layers.50.linear_attn.in_proj_b",
318
+ "model.language_model.layers.50.linear_attn.in_proj_a",
319
  "model.language_model.layers.52.linear_attn",
320
  "model.language_model.layers.52.linear_attn.norm",
321
+ "model.language_model.layers.52.linear_attn.in_proj_b",
322
+ "model.language_model.layers.52.linear_attn.in_proj_a",
323
  "model.language_model.layers.53.linear_attn",
324
  "model.language_model.layers.53.linear_attn.norm",
325
+ "model.language_model.layers.53.linear_attn.in_proj_b",
326
+ "model.language_model.layers.53.linear_attn.in_proj_a",
327
  "model.language_model.layers.54.linear_attn",
328
  "model.language_model.layers.54.linear_attn.norm",
329
+ "model.language_model.layers.54.linear_attn.in_proj_b",
330
+ "model.language_model.layers.54.linear_attn.in_proj_a",
331
  "model.language_model.layers.56.linear_attn",
332
  "model.language_model.layers.56.linear_attn.norm",
333
+ "model.language_model.layers.56.linear_attn.in_proj_b",
334
+ "model.language_model.layers.56.linear_attn.in_proj_a",
335
  "model.language_model.layers.57.linear_attn",
336
  "model.language_model.layers.57.linear_attn.norm",
337
+ "model.language_model.layers.57.linear_attn.in_proj_b",
338
+ "model.language_model.layers.57.linear_attn.in_proj_a",
339
  "model.language_model.layers.58.linear_attn",
340
  "model.language_model.layers.58.linear_attn.norm",
341
+ "model.language_model.layers.58.linear_attn.in_proj_b",
342
+ "model.language_model.layers.58.linear_attn.in_proj_a",
343
  "model.language_model.layers.60.linear_attn",
344
  "model.language_model.layers.60.linear_attn.norm",
345
+ "model.language_model.layers.60.linear_attn.in_proj_b",
346
+ "model.language_model.layers.60.linear_attn.in_proj_a",
347
  "model.language_model.layers.61.linear_attn",
348
  "model.language_model.layers.61.linear_attn.norm",
349
+ "model.language_model.layers.61.linear_attn.in_proj_b",
350
+ "model.language_model.layers.61.linear_attn.in_proj_a",
351
  "model.language_model.layers.62.linear_attn",
352
  "model.language_model.layers.62.linear_attn.norm",
353
+ "model.language_model.layers.62.linear_attn.in_proj_b",
354
+ "model.language_model.layers.62.linear_attn.in_proj_a",
355
  "lm_head"
356
  ],
357
  "kv_cache_scheme": null,
 
361
  "transform_config": {},
362
  "version": "0.18.0"
363
  },
364
+ "text_config": {
365
+ "attention_bias": false,
366
+ "attention_dropout": 0.0,
367
+ "attn_output_gate": true,
368
+ "bos_token_id": null,
369
+ "dtype": "bfloat16",
370
+ "eos_token_id": 248044,
371
+ "full_attention_interval": 4,
372
+ "head_dim": 256,
373
+ "hidden_act": "silu",
374
+ "hidden_size": 5120,
375
+ "initializer_range": 0.02,
376
+ "intermediate_size": 17408,
377
+ "layer_types": [
378
+ "linear_attention",
379
+ "linear_attention",
380
+ "linear_attention",
381
+ "full_attention",
382
+ "linear_attention",
383
+ "linear_attention",
384
+ "linear_attention",
385
+ "full_attention",
386
+ "linear_attention",
387
+ "linear_attention",
388
+ "linear_attention",
389
+ "full_attention",
390
+ "linear_attention",
391
+ "linear_attention",
392
+ "linear_attention",
393
+ "full_attention",
394
+ "linear_attention",
395
+ "linear_attention",
396
+ "linear_attention",
397
+ "full_attention",
398
+ "linear_attention",
399
+ "linear_attention",
400
+ "linear_attention",
401
+ "full_attention",
402
+ "linear_attention",
403
+ "linear_attention",
404
+ "linear_attention",
405
+ "full_attention",
406
+ "linear_attention",
407
+ "linear_attention",
408
+ "linear_attention",
409
+ "full_attention",
410
+ "linear_attention",
411
+ "linear_attention",
412
+ "linear_attention",
413
+ "full_attention",
414
+ "linear_attention",
415
+ "linear_attention",
416
+ "linear_attention",
417
+ "full_attention",
418
+ "linear_attention",
419
+ "linear_attention",
420
+ "linear_attention",
421
+ "full_attention",
422
+ "linear_attention",
423
+ "linear_attention",
424
+ "linear_attention",
425
+ "full_attention",
426
+ "linear_attention",
427
+ "linear_attention",
428
+ "linear_attention",
429
+ "full_attention",
430
+ "linear_attention",
431
+ "linear_attention",
432
+ "linear_attention",
433
+ "full_attention",
434
+ "linear_attention",
435
+ "linear_attention",
436
+ "linear_attention",
437
+ "full_attention",
438
+ "linear_attention",
439
+ "linear_attention",
440
+ "linear_attention",
441
+ "full_attention"
442
  ],
443
+ "linear_conv_kernel_dim": 4,
444
+ "linear_key_head_dim": 128,
445
+ "linear_num_key_heads": 16,
446
+ "linear_num_value_heads": 48,
447
+ "linear_value_head_dim": 128,
448
+ "mamba_ssm_dtype": "float32",
449
+ "max_position_embeddings": 262144,
450
+ "mlp_only_layers": [],
451
+ "model_type": "qwen3_5_text",
452
+ "mtp_num_hidden_layers": 0,
453
+ "mtp_use_dedicated_embeddings": false,
454
+ "num_attention_heads": 24,
455
+ "num_hidden_layers": 64,
456
+ "num_key_value_heads": 4,
457
+ "pad_token_id": null,
458
  "partial_rotary_factor": 0.25,
459
+ "rms_norm_eps": 1e-06,
460
+ "rope_parameters": {
461
+ "mrope_interleaved": true,
462
+ "mrope_section": [
463
+ 11,
464
+ 11,
465
+ 10
466
+ ],
467
+ "partial_rotary_factor": 0.25,
468
+ "rope_theta": 10000000,
469
+ "rope_type": "default"
470
+ },
471
+ "tie_word_embeddings": false,
472
+ "use_cache": true,
473
+ "vocab_size": 248320
474
  },
475
  "tie_word_embeddings": false,
476
  "transformers_version": "5.14.1",
477
+ "video_token_id": 248057,
478
+ "vision_config": {
479
+ "deepstack_visual_indexes": [],
480
+ "depth": 27,
481
+ "dtype": "bfloat16",
482
+ "hidden_act": "gelu_pytorch_tanh",
483
+ "hidden_size": 1152,
484
+ "in_channels": 3,
485
+ "initializer_range": 0.02,
486
+ "intermediate_size": 4304,
487
+ "model_type": "qwen3_5_vision",
488
+ "num_heads": 16,
489
+ "num_position_embeddings": 2304,
490
+ "out_hidden_size": 5120,
491
+ "patch_size": 16,
492
+ "spatial_merge_size": 2,
493
+ "temporal_patch_size": 2
494
+ },
495
+ "vision_end_token_id": 248054,
496
+ "vision_start_token_id": 248053
497
  }
model-00001-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dea629430271c23f8c0399fa6cadb2af73a9cff2ecce84b6a91ce703918278d1
3
+ size 4988600336
model-00002-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04ec2bbb9ece384a67daea2fb8e0aa80fd38f9f7bd8531c03173013415c393e3
3
+ size 4988055424
model-00003-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0035b786671e0628dc078e24763e4301b6863626d13b2c44a1e715138da153c3
3
+ size 4956577504
model-00004-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34cbc08d1b5a4c67006e7b5c0eecc36642c1b39d0da9411649ece343949745d3
3
+ size 4998446400
model-00005-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2225c6761f5f1f55b440316578d621e249dab549a65b1c61bf7b7793c417597
3
+ size 4988071120
model-00006-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2ae4b1458e8cef8b0e70b7f9dff4b12d691dd1080f1f8261ba14156c8f7fa24
3
+ size 2931945912
model-00007-of-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:334e041145b36f30972d481d5cc75189f8d38db4365eb9e97b658e74ef5a7112
3
+ size 2542796928
model.safetensors.index.json CHANGED
The diff for this file is too large to render. See raw diff
 
recipe.yaml CHANGED
@@ -2,7 +2,10 @@ default_stage:
2
  default_modifiers:
3
  QuantizationModifier:
4
  targets: [Linear]
5
- ignore: [lm_head]
 
 
 
6
  scheme: FP8_DYNAMIC
7
  bypass_divisibility_checks: false
8
  requires_calibration_data: false
 
2
  default_modifiers:
3
  QuantizationModifier:
4
  targets: [Linear]
5
+ ignore: [lm_head, 're:.*embed_tokens.*', 're:.*layernorm.*', 're:.*\.norm$', 're:.*norm_f$',
6
+ 're:.*linear_attn\.(A_log|conv1d|dt_bias|in_proj_a|in_proj_b|in_proj_ba|norm)$', 're:.*conv1d.*',
7
+ 're:.*\.(A_log|D|dt_bias)$', 're:.*mlp\.gate$', 're:.*shared_expert_gate$', 're:.*mixer\.gate.*',
8
+ 're:.*visual.*', 're:.*vision_tower.*', 're:.*audio_tower.*']
9
  scheme: FP8_DYNAMIC
10
  bypass_divisibility_checks: false
11
  requires_calibration_data: false