denisko Claude Opus 4.6 commited on
Commit
99684b3
·
1 Parent(s): 89275ae

Add preset placement configs with paper-matching names

Browse files

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

preset_configs/.DS_Store ADDED
Binary file (6.15 kB). View file
 
preset_configs/Idealized_All-18/config.json ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "sliding_window",
16
+ "sliding_window",
17
+ "sliding_window",
18
+ "attention",
19
+ "sliding_window",
20
+ "sliding_window",
21
+ "attention",
22
+ "sliding_window",
23
+ "sliding_window",
24
+ "attention",
25
+ "attention",
26
+ "sliding_window",
27
+ "sliding_window",
28
+ "attention",
29
+ "attention",
30
+ "sliding_window",
31
+ "attention",
32
+ "sliding_window",
33
+ "sliding_window",
34
+ "attention",
35
+ "sliding_window",
36
+ "gdn",
37
+ "attention",
38
+ "attention",
39
+ "sliding_window",
40
+ "sliding_window",
41
+ "sliding_window",
42
+ "kda",
43
+ "attention",
44
+ "sliding_window",
45
+ "sliding_window",
46
+ "sliding_window",
47
+ "sliding_window",
48
+ "sliding_window",
49
+ "sliding_window",
50
+ "sliding_window",
51
+ "sliding_window",
52
+ "sliding_window",
53
+ "attention",
54
+ "gdn",
55
+ "sliding_window",
56
+ "sliding_window",
57
+ "sliding_window",
58
+ "sliding_window",
59
+ "sliding_window",
60
+ "sliding_window",
61
+ "attention",
62
+ "sliding_window"
63
+ ],
64
+ "blocks": {
65
+ "sliding_window": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": 4096
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "attention": {
91
+ "mixer": {
92
+ "add_linear_biases": false,
93
+ "head_groups": 8,
94
+ "head_size": 128,
95
+ "heads": 32,
96
+ "rotary": {
97
+ "theta": 1000000000.0,
98
+ "type": "mistral_1d"
99
+ },
100
+ "type": "attention",
101
+ "window_size": null
102
+ },
103
+ "mlp": {
104
+ "activation": "silu",
105
+ "add_linear_biases": false,
106
+ "gated": true,
107
+ "intermediate_size": 14336,
108
+ "type": "mlp"
109
+ },
110
+ "normalization": {
111
+ "epsilon": 1e-05,
112
+ "type": "rms_norm"
113
+ }
114
+ },
115
+ "gdn": {
116
+ "mixer": {
117
+ "convolution_layer": {
118
+ "kernel_size": 4
119
+ },
120
+ "key_head_dim": 128,
121
+ "key_heads": 8,
122
+ "type": "gdn",
123
+ "value_head_dim": 128,
124
+ "value_heads": 32
125
+ },
126
+ "mlp": {
127
+ "activation": "silu",
128
+ "add_linear_biases": false,
129
+ "gated": true,
130
+ "intermediate_size": 14336,
131
+ "type": "mlp"
132
+ },
133
+ "normalization": {
134
+ "epsilon": 1e-05,
135
+ "type": "rms_norm"
136
+ }
137
+ },
138
+ "kda": {
139
+ "mixer": {
140
+ "convolution_layer": {
141
+ "kernel_size": 4
142
+ },
143
+ "head_dim": 128,
144
+ "heads": 32,
145
+ "normalization": {
146
+ "epsilon": 1e-05
147
+ },
148
+ "type": "kda"
149
+ },
150
+ "mlp": {
151
+ "activation": "silu",
152
+ "add_linear_biases": false,
153
+ "gated": true,
154
+ "intermediate_size": 14336,
155
+ "type": "mlp"
156
+ },
157
+ "normalization": {
158
+ "epsilon": 1e-05,
159
+ "type": "rms_norm"
160
+ }
161
+ }
162
+ }
163
+ },
164
+ "embeddings": {
165
+ "max_position_embeddings": 262144
166
+ },
167
+ "eos_token_id": 2,
168
+ "head": {
169
+ "normalization": {
170
+ "epsilon": 1e-05,
171
+ "type": "rms_norm"
172
+ }
173
+ },
174
+ "hidden_size": 5120,
175
+ "image_token_index": 10,
176
+ "model_type": "apriel2",
177
+ "tie_word_embeddings": false,
178
+ "transformers_version": "4.57.3",
179
+ "use_cache": true,
180
+ "vision_encoder": {
181
+ "adapter": {
182
+ "activation": "gelu_pytorch_tanh",
183
+ "add_linear_biases": true,
184
+ "gated": false,
185
+ "intermediate_size": 5120,
186
+ "type": "mlp"
187
+ },
188
+ "embeddings": {
189
+ "input_channels": 3,
190
+ "normalization": {
191
+ "epsilon": 1e-05,
192
+ "type": "rms_norm"
193
+ },
194
+ "patch_height": 16,
195
+ "patch_width": 16
196
+ },
197
+ "encoder": {
198
+ "block": {
199
+ "mixer": {
200
+ "add_linear_biases": false,
201
+ "causal": false,
202
+ "cross_document_attention": false,
203
+ "head_groups": 16,
204
+ "head_size": 64,
205
+ "heads": 16,
206
+ "rotary": {
207
+ "max_image_size": 1024,
208
+ "patch_size": 16,
209
+ "theta": 10000.0,
210
+ "type": "pixtral_2d"
211
+ },
212
+ "type": "attention"
213
+ },
214
+ "mlp": {
215
+ "activation": "silu",
216
+ "add_linear_biases": false,
217
+ "gated": true,
218
+ "intermediate_size": 4096,
219
+ "type": "mlp"
220
+ },
221
+ "normalization": {
222
+ "epsilon": 1e-05,
223
+ "type": "rms_norm"
224
+ }
225
+ },
226
+ "num_blocks": 24,
227
+ "type": "fixed"
228
+ },
229
+ "hidden_size": 1024,
230
+ "num_channels": 3,
231
+ "num_hidden_layers": 24,
232
+ "patch_size": 16
233
+ },
234
+ "vocab_size": 131072
235
+ }
preset_configs/Idealized_All-6/config.json ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "sliding_window",
16
+ "sliding_window",
17
+ "gdn",
18
+ "gdn",
19
+ "gdn",
20
+ "sliding_window",
21
+ "sliding_window",
22
+ "sliding_window",
23
+ "sliding_window",
24
+ "gdn",
25
+ "sliding_window",
26
+ "sliding_window",
27
+ "sliding_window",
28
+ "sliding_window",
29
+ "sliding_window",
30
+ "gdn",
31
+ "sliding_window",
32
+ "sliding_window",
33
+ "gdn",
34
+ "gdn",
35
+ "sliding_window",
36
+ "sliding_window",
37
+ "gdn",
38
+ "sliding_window",
39
+ "sliding_window",
40
+ "kda",
41
+ "sliding_window",
42
+ "sliding_window",
43
+ "sliding_window",
44
+ "gdn",
45
+ "gdn",
46
+ "sliding_window",
47
+ "kda",
48
+ "kda",
49
+ "sliding_window",
50
+ "gdn",
51
+ "sliding_window",
52
+ "sliding_window",
53
+ "gdn",
54
+ "sliding_window",
55
+ "gdn",
56
+ "sliding_window",
57
+ "sliding_window",
58
+ "kda",
59
+ "sliding_window",
60
+ "kda",
61
+ "sliding_window",
62
+ "sliding_window"
63
+ ],
64
+ "blocks": {
65
+ "sliding_window": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": 4096
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "gdn": {
91
+ "mixer": {
92
+ "convolution_layer": {
93
+ "kernel_size": 4
94
+ },
95
+ "key_head_dim": 128,
96
+ "key_heads": 8,
97
+ "type": "gdn",
98
+ "value_head_dim": 128,
99
+ "value_heads": 32
100
+ },
101
+ "mlp": {
102
+ "activation": "silu",
103
+ "add_linear_biases": false,
104
+ "gated": true,
105
+ "intermediate_size": 14336,
106
+ "type": "mlp"
107
+ },
108
+ "normalization": {
109
+ "epsilon": 1e-05,
110
+ "type": "rms_norm"
111
+ }
112
+ },
113
+ "kda": {
114
+ "mixer": {
115
+ "convolution_layer": {
116
+ "kernel_size": 4
117
+ },
118
+ "head_dim": 128,
119
+ "heads": 32,
120
+ "normalization": {
121
+ "epsilon": 1e-05
122
+ },
123
+ "type": "kda"
124
+ },
125
+ "mlp": {
126
+ "activation": "silu",
127
+ "add_linear_biases": false,
128
+ "gated": true,
129
+ "intermediate_size": 14336,
130
+ "type": "mlp"
131
+ },
132
+ "normalization": {
133
+ "epsilon": 1e-05,
134
+ "type": "rms_norm"
135
+ }
136
+ }
137
+ }
138
+ },
139
+ "embeddings": {
140
+ "max_position_embeddings": 262144
141
+ },
142
+ "eos_token_id": 2,
143
+ "head": {
144
+ "normalization": {
145
+ "epsilon": 1e-05,
146
+ "type": "rms_norm"
147
+ }
148
+ },
149
+ "hidden_size": 5120,
150
+ "image_token_index": 10,
151
+ "model_type": "apriel2",
152
+ "tie_word_embeddings": false,
153
+ "transformers_version": "4.57.3",
154
+ "use_cache": true,
155
+ "vision_encoder": {
156
+ "adapter": {
157
+ "activation": "gelu_pytorch_tanh",
158
+ "add_linear_biases": true,
159
+ "gated": false,
160
+ "intermediate_size": 5120,
161
+ "type": "mlp"
162
+ },
163
+ "embeddings": {
164
+ "input_channels": 3,
165
+ "normalization": {
166
+ "epsilon": 1e-05,
167
+ "type": "rms_norm"
168
+ },
169
+ "patch_height": 16,
170
+ "patch_width": 16
171
+ },
172
+ "encoder": {
173
+ "block": {
174
+ "mixer": {
175
+ "add_linear_biases": false,
176
+ "causal": false,
177
+ "cross_document_attention": false,
178
+ "head_groups": 16,
179
+ "head_size": 64,
180
+ "heads": 16,
181
+ "rotary": {
182
+ "max_image_size": 1024,
183
+ "patch_size": 16,
184
+ "theta": 10000.0,
185
+ "type": "pixtral_2d"
186
+ },
187
+ "type": "attention"
188
+ },
189
+ "mlp": {
190
+ "activation": "silu",
191
+ "add_linear_biases": false,
192
+ "gated": true,
193
+ "intermediate_size": 4096,
194
+ "type": "mlp"
195
+ },
196
+ "normalization": {
197
+ "epsilon": 1e-05,
198
+ "type": "rms_norm"
199
+ }
200
+ },
201
+ "num_blocks": 24,
202
+ "type": "fixed"
203
+ },
204
+ "hidden_size": 1024,
205
+ "num_channels": 3,
206
+ "num_hidden_layers": 24,
207
+ "patch_size": 16
208
+ },
209
+ "vocab_size": 131072
210
+ }
preset_configs/Idealized_Lklhd-6/config.json ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "sliding_window",
16
+ "gdn",
17
+ "gdn",
18
+ "gdn",
19
+ "gdn",
20
+ "sliding_window",
21
+ "kda",
22
+ "gdn",
23
+ "sliding_window",
24
+ "gdn",
25
+ "sliding_window",
26
+ "sliding_window",
27
+ "kda",
28
+ "gdn",
29
+ "sliding_window",
30
+ "sliding_window",
31
+ "sliding_window",
32
+ "sliding_window",
33
+ "sliding_window",
34
+ "sliding_window",
35
+ "sliding_window",
36
+ "sliding_window",
37
+ "sliding_window",
38
+ "sliding_window",
39
+ "sliding_window",
40
+ "sliding_window",
41
+ "sliding_window",
42
+ "sliding_window",
43
+ "sliding_window",
44
+ "sliding_window",
45
+ "gdn",
46
+ "sliding_window",
47
+ "kda",
48
+ "sliding_window",
49
+ "sliding_window",
50
+ "kda",
51
+ "sliding_window",
52
+ "gdn",
53
+ "gdn",
54
+ "sliding_window",
55
+ "gdn",
56
+ "gdn",
57
+ "sliding_window",
58
+ "sliding_window",
59
+ "gdn",
60
+ "sliding_window",
61
+ "sliding_window",
62
+ "kda"
63
+ ],
64
+ "blocks": {
65
+ "sliding_window": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": 4096
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "gdn": {
91
+ "mixer": {
92
+ "convolution_layer": {
93
+ "kernel_size": 4
94
+ },
95
+ "key_head_dim": 128,
96
+ "key_heads": 8,
97
+ "type": "gdn",
98
+ "value_head_dim": 128,
99
+ "value_heads": 32
100
+ },
101
+ "mlp": {
102
+ "activation": "silu",
103
+ "add_linear_biases": false,
104
+ "gated": true,
105
+ "intermediate_size": 14336,
106
+ "type": "mlp"
107
+ },
108
+ "normalization": {
109
+ "epsilon": 1e-05,
110
+ "type": "rms_norm"
111
+ }
112
+ },
113
+ "kda": {
114
+ "mixer": {
115
+ "convolution_layer": {
116
+ "kernel_size": 4
117
+ },
118
+ "head_dim": 128,
119
+ "heads": 32,
120
+ "normalization": {
121
+ "epsilon": 1e-05
122
+ },
123
+ "type": "kda"
124
+ },
125
+ "mlp": {
126
+ "activation": "silu",
127
+ "add_linear_biases": false,
128
+ "gated": true,
129
+ "intermediate_size": 14336,
130
+ "type": "mlp"
131
+ },
132
+ "normalization": {
133
+ "epsilon": 1e-05,
134
+ "type": "rms_norm"
135
+ }
136
+ }
137
+ }
138
+ },
139
+ "embeddings": {
140
+ "max_position_embeddings": 262144
141
+ },
142
+ "eos_token_id": 2,
143
+ "head": {
144
+ "normalization": {
145
+ "epsilon": 1e-05,
146
+ "type": "rms_norm"
147
+ }
148
+ },
149
+ "hidden_size": 5120,
150
+ "image_token_index": 10,
151
+ "model_type": "apriel2",
152
+ "tie_word_embeddings": false,
153
+ "transformers_version": "4.57.3",
154
+ "use_cache": true,
155
+ "vision_encoder": {
156
+ "adapter": {
157
+ "activation": "gelu_pytorch_tanh",
158
+ "add_linear_biases": true,
159
+ "gated": false,
160
+ "intermediate_size": 5120,
161
+ "type": "mlp"
162
+ },
163
+ "embeddings": {
164
+ "input_channels": 3,
165
+ "normalization": {
166
+ "epsilon": 1e-05,
167
+ "type": "rms_norm"
168
+ },
169
+ "patch_height": 16,
170
+ "patch_width": 16
171
+ },
172
+ "encoder": {
173
+ "block": {
174
+ "mixer": {
175
+ "add_linear_biases": false,
176
+ "causal": false,
177
+ "cross_document_attention": false,
178
+ "head_groups": 16,
179
+ "head_size": 64,
180
+ "heads": 16,
181
+ "rotary": {
182
+ "max_image_size": 1024,
183
+ "patch_size": 16,
184
+ "theta": 10000.0,
185
+ "type": "pixtral_2d"
186
+ },
187
+ "type": "attention"
188
+ },
189
+ "mlp": {
190
+ "activation": "silu",
191
+ "add_linear_biases": false,
192
+ "gated": true,
193
+ "intermediate_size": 4096,
194
+ "type": "mlp"
195
+ },
196
+ "normalization": {
197
+ "epsilon": 1e-05,
198
+ "type": "rms_norm"
199
+ }
200
+ },
201
+ "num_blocks": 24,
202
+ "type": "fixed"
203
+ },
204
+ "hidden_size": 1024,
205
+ "num_channels": 3,
206
+ "num_hidden_layers": 24,
207
+ "patch_size": 16
208
+ },
209
+ "vocab_size": 131072
210
+ }
preset_configs/Reg_Lklhd-10/config.json ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "gdn",
16
+ "gdn",
17
+ "gdn",
18
+ "gdn",
19
+ "gdn",
20
+ "gdn",
21
+ "sliding_window",
22
+ "gdn",
23
+ "gdn",
24
+ "gdn",
25
+ "gdn",
26
+ "sliding_window",
27
+ "kda",
28
+ "kda",
29
+ "sliding_window",
30
+ "gdn",
31
+ "gdn",
32
+ "gdn",
33
+ "kda",
34
+ "sliding_window",
35
+ "gdn",
36
+ "sliding_window",
37
+ "gdn",
38
+ "gdn",
39
+ "sliding_window",
40
+ "gdn",
41
+ "sliding_window",
42
+ "gdn",
43
+ "sliding_window",
44
+ "gdn",
45
+ "gdn",
46
+ "sliding_window",
47
+ "gdn",
48
+ "gdn",
49
+ "kda",
50
+ "gdn",
51
+ "gdn",
52
+ "gdn",
53
+ "gdn",
54
+ "gdn",
55
+ "kda",
56
+ "gdn",
57
+ "sliding_window",
58
+ "gdn",
59
+ "gdn",
60
+ "gdn",
61
+ "gdn",
62
+ "gdn"
63
+ ],
64
+ "blocks": {
65
+ "gdn": {
66
+ "mixer": {
67
+ "convolution_layer": {
68
+ "kernel_size": 4
69
+ },
70
+ "key_head_dim": 128,
71
+ "key_heads": 8,
72
+ "type": "gdn",
73
+ "value_head_dim": 128,
74
+ "value_heads": 32
75
+ },
76
+ "mlp": {
77
+ "activation": "silu",
78
+ "add_linear_biases": false,
79
+ "gated": true,
80
+ "intermediate_size": 14336,
81
+ "type": "mlp"
82
+ },
83
+ "normalization": {
84
+ "epsilon": 1e-05,
85
+ "type": "rms_norm"
86
+ }
87
+ },
88
+ "sliding_window": {
89
+ "mixer": {
90
+ "add_linear_biases": false,
91
+ "head_groups": 8,
92
+ "head_size": 128,
93
+ "heads": 32,
94
+ "rotary": {
95
+ "theta": 1000000000.0,
96
+ "type": "mistral_1d"
97
+ },
98
+ "type": "attention",
99
+ "window_size": 4096
100
+ },
101
+ "mlp": {
102
+ "activation": "silu",
103
+ "add_linear_biases": false,
104
+ "gated": true,
105
+ "intermediate_size": 14336,
106
+ "type": "mlp"
107
+ },
108
+ "normalization": {
109
+ "epsilon": 1e-05,
110
+ "type": "rms_norm"
111
+ }
112
+ },
113
+ "kda": {
114
+ "mixer": {
115
+ "convolution_layer": {
116
+ "kernel_size": 4
117
+ },
118
+ "head_dim": 128,
119
+ "heads": 32,
120
+ "normalization": {
121
+ "epsilon": 1e-05
122
+ },
123
+ "type": "kda"
124
+ },
125
+ "mlp": {
126
+ "activation": "silu",
127
+ "add_linear_biases": false,
128
+ "gated": true,
129
+ "intermediate_size": 14336,
130
+ "type": "mlp"
131
+ },
132
+ "normalization": {
133
+ "epsilon": 1e-05,
134
+ "type": "rms_norm"
135
+ }
136
+ }
137
+ }
138
+ },
139
+ "embeddings": {
140
+ "max_position_embeddings": 262144
141
+ },
142
+ "eos_token_id": 2,
143
+ "head": {
144
+ "normalization": {
145
+ "epsilon": 1e-05,
146
+ "type": "rms_norm"
147
+ }
148
+ },
149
+ "hidden_size": 5120,
150
+ "image_token_index": 10,
151
+ "model_type": "apriel2",
152
+ "tie_word_embeddings": false,
153
+ "transformers_version": "4.57.3",
154
+ "use_cache": true,
155
+ "vision_encoder": {
156
+ "adapter": {
157
+ "activation": "gelu_pytorch_tanh",
158
+ "add_linear_biases": true,
159
+ "gated": false,
160
+ "intermediate_size": 5120,
161
+ "type": "mlp"
162
+ },
163
+ "embeddings": {
164
+ "input_channels": 3,
165
+ "normalization": {
166
+ "epsilon": 1e-05,
167
+ "type": "rms_norm"
168
+ },
169
+ "patch_height": 16,
170
+ "patch_width": 16
171
+ },
172
+ "encoder": {
173
+ "block": {
174
+ "mixer": {
175
+ "add_linear_biases": false,
176
+ "causal": false,
177
+ "cross_document_attention": false,
178
+ "head_groups": 16,
179
+ "head_size": 64,
180
+ "heads": 16,
181
+ "rotary": {
182
+ "max_image_size": 1024,
183
+ "patch_size": 16,
184
+ "theta": 10000.0,
185
+ "type": "pixtral_2d"
186
+ },
187
+ "type": "attention"
188
+ },
189
+ "mlp": {
190
+ "activation": "silu",
191
+ "add_linear_biases": false,
192
+ "gated": true,
193
+ "intermediate_size": 4096,
194
+ "type": "mlp"
195
+ },
196
+ "normalization": {
197
+ "epsilon": 1e-05,
198
+ "type": "rms_norm"
199
+ }
200
+ },
201
+ "num_blocks": 24,
202
+ "type": "fixed"
203
+ },
204
+ "hidden_size": 1024,
205
+ "num_channels": 3,
206
+ "num_hidden_layers": 24,
207
+ "patch_size": 16
208
+ },
209
+ "vocab_size": 131072
210
+ }
preset_configs/Reg_Lklhd-13/config.json ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "gdn",
16
+ "gdn",
17
+ "gdn",
18
+ "gdn",
19
+ "gdn",
20
+ "gdn",
21
+ "sliding_window",
22
+ "gdn",
23
+ "kda",
24
+ "gdn",
25
+ "kda",
26
+ "sliding_window",
27
+ "kda",
28
+ "kda",
29
+ "sliding_window",
30
+ "kda",
31
+ "sliding_window",
32
+ "gdn",
33
+ "kda",
34
+ "sliding_window",
35
+ "gdn",
36
+ "sliding_window",
37
+ "sliding_window",
38
+ "kda",
39
+ "sliding_window",
40
+ "gdn",
41
+ "sliding_window",
42
+ "sliding_window",
43
+ "gdn",
44
+ "sliding_window",
45
+ "gdn",
46
+ "gdn",
47
+ "sliding_window",
48
+ "gdn",
49
+ "kda",
50
+ "kda",
51
+ "sliding_window",
52
+ "kda",
53
+ "kda",
54
+ "sliding_window",
55
+ "gdn",
56
+ "sliding_window",
57
+ "gdn",
58
+ "sliding_window",
59
+ "kda",
60
+ "gdn",
61
+ "gdn",
62
+ "kda"
63
+ ],
64
+ "blocks": {
65
+ "gdn": {
66
+ "mixer": {
67
+ "convolution_layer": {
68
+ "kernel_size": 4
69
+ },
70
+ "key_head_dim": 128,
71
+ "key_heads": 8,
72
+ "type": "gdn",
73
+ "value_head_dim": 128,
74
+ "value_heads": 32
75
+ },
76
+ "mlp": {
77
+ "activation": "silu",
78
+ "add_linear_biases": false,
79
+ "gated": true,
80
+ "intermediate_size": 14336,
81
+ "type": "mlp"
82
+ },
83
+ "normalization": {
84
+ "epsilon": 1e-05,
85
+ "type": "rms_norm"
86
+ }
87
+ },
88
+ "sliding_window": {
89
+ "mixer": {
90
+ "add_linear_biases": false,
91
+ "head_groups": 8,
92
+ "head_size": 128,
93
+ "heads": 32,
94
+ "rotary": {
95
+ "theta": 1000000000.0,
96
+ "type": "mistral_1d"
97
+ },
98
+ "type": "attention",
99
+ "window_size": 4096
100
+ },
101
+ "mlp": {
102
+ "activation": "silu",
103
+ "add_linear_biases": false,
104
+ "gated": true,
105
+ "intermediate_size": 14336,
106
+ "type": "mlp"
107
+ },
108
+ "normalization": {
109
+ "epsilon": 1e-05,
110
+ "type": "rms_norm"
111
+ }
112
+ },
113
+ "kda": {
114
+ "mixer": {
115
+ "convolution_layer": {
116
+ "kernel_size": 4
117
+ },
118
+ "head_dim": 128,
119
+ "heads": 32,
120
+ "normalization": {
121
+ "epsilon": 1e-05
122
+ },
123
+ "type": "kda"
124
+ },
125
+ "mlp": {
126
+ "activation": "silu",
127
+ "add_linear_biases": false,
128
+ "gated": true,
129
+ "intermediate_size": 14336,
130
+ "type": "mlp"
131
+ },
132
+ "normalization": {
133
+ "epsilon": 1e-05,
134
+ "type": "rms_norm"
135
+ }
136
+ }
137
+ }
138
+ },
139
+ "embeddings": {
140
+ "max_position_embeddings": 262144
141
+ },
142
+ "eos_token_id": 2,
143
+ "head": {
144
+ "normalization": {
145
+ "epsilon": 1e-05,
146
+ "type": "rms_norm"
147
+ }
148
+ },
149
+ "hidden_size": 5120,
150
+ "image_token_index": 10,
151
+ "model_type": "apriel2",
152
+ "tie_word_embeddings": false,
153
+ "transformers_version": "4.57.3",
154
+ "use_cache": true,
155
+ "vision_encoder": {
156
+ "adapter": {
157
+ "activation": "gelu_pytorch_tanh",
158
+ "add_linear_biases": true,
159
+ "gated": false,
160
+ "intermediate_size": 5120,
161
+ "type": "mlp"
162
+ },
163
+ "embeddings": {
164
+ "input_channels": 3,
165
+ "normalization": {
166
+ "epsilon": 1e-05,
167
+ "type": "rms_norm"
168
+ },
169
+ "patch_height": 16,
170
+ "patch_width": 16
171
+ },
172
+ "encoder": {
173
+ "block": {
174
+ "mixer": {
175
+ "add_linear_biases": false,
176
+ "causal": false,
177
+ "cross_document_attention": false,
178
+ "head_groups": 16,
179
+ "head_size": 64,
180
+ "heads": 16,
181
+ "rotary": {
182
+ "max_image_size": 1024,
183
+ "patch_size": 16,
184
+ "theta": 10000.0,
185
+ "type": "pixtral_2d"
186
+ },
187
+ "type": "attention"
188
+ },
189
+ "mlp": {
190
+ "activation": "silu",
191
+ "add_linear_biases": false,
192
+ "gated": true,
193
+ "intermediate_size": 4096,
194
+ "type": "mlp"
195
+ },
196
+ "normalization": {
197
+ "epsilon": 1e-05,
198
+ "type": "rms_norm"
199
+ }
200
+ },
201
+ "num_blocks": 24,
202
+ "type": "fixed"
203
+ },
204
+ "hidden_size": 1024,
205
+ "num_channels": 3,
206
+ "num_hidden_layers": 24,
207
+ "patch_size": 16
208
+ },
209
+ "vocab_size": 131072
210
+ }
preset_configs/Reg_Lklhd-18/config.json ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "attention",
16
+ "sliding_window",
17
+ "gdn",
18
+ "sliding_window",
19
+ "gdn",
20
+ "gdn",
21
+ "sliding_window",
22
+ "kda",
23
+ "sliding_window",
24
+ "gdn",
25
+ "sliding_window",
26
+ "sliding_window",
27
+ "sliding_window",
28
+ "kda",
29
+ "sliding_window",
30
+ "gdn",
31
+ "sliding_window",
32
+ "gdn",
33
+ "attention",
34
+ "sliding_window",
35
+ "kda",
36
+ "attention",
37
+ "sliding_window",
38
+ "sliding_window",
39
+ "sliding_window",
40
+ "gdn",
41
+ "sliding_window",
42
+ "sliding_window",
43
+ "gdn",
44
+ "sliding_window",
45
+ "gdn",
46
+ "sliding_window",
47
+ "sliding_window",
48
+ "gdn",
49
+ "sliding_window",
50
+ "gdn",
51
+ "sliding_window",
52
+ "gdn",
53
+ "sliding_window",
54
+ "gdn",
55
+ "sliding_window",
56
+ "gdn",
57
+ "sliding_window",
58
+ "kda",
59
+ "sliding_window",
60
+ "gdn",
61
+ "sliding_window",
62
+ "gdn"
63
+ ],
64
+ "blocks": {
65
+ "attention": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": null
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "sliding_window": {
91
+ "mixer": {
92
+ "add_linear_biases": false,
93
+ "head_groups": 8,
94
+ "head_size": 128,
95
+ "heads": 32,
96
+ "rotary": {
97
+ "theta": 1000000000.0,
98
+ "type": "mistral_1d"
99
+ },
100
+ "type": "attention",
101
+ "window_size": 4096
102
+ },
103
+ "mlp": {
104
+ "activation": "silu",
105
+ "add_linear_biases": false,
106
+ "gated": true,
107
+ "intermediate_size": 14336,
108
+ "type": "mlp"
109
+ },
110
+ "normalization": {
111
+ "epsilon": 1e-05,
112
+ "type": "rms_norm"
113
+ }
114
+ },
115
+ "gdn": {
116
+ "mixer": {
117
+ "convolution_layer": {
118
+ "kernel_size": 4
119
+ },
120
+ "key_head_dim": 128,
121
+ "key_heads": 8,
122
+ "type": "gdn",
123
+ "value_head_dim": 128,
124
+ "value_heads": 32
125
+ },
126
+ "mlp": {
127
+ "activation": "silu",
128
+ "add_linear_biases": false,
129
+ "gated": true,
130
+ "intermediate_size": 14336,
131
+ "type": "mlp"
132
+ },
133
+ "normalization": {
134
+ "epsilon": 1e-05,
135
+ "type": "rms_norm"
136
+ }
137
+ },
138
+ "kda": {
139
+ "mixer": {
140
+ "convolution_layer": {
141
+ "kernel_size": 4
142
+ },
143
+ "head_dim": 128,
144
+ "heads": 32,
145
+ "normalization": {
146
+ "epsilon": 1e-05
147
+ },
148
+ "type": "kda"
149
+ },
150
+ "mlp": {
151
+ "activation": "silu",
152
+ "add_linear_biases": false,
153
+ "gated": true,
154
+ "intermediate_size": 14336,
155
+ "type": "mlp"
156
+ },
157
+ "normalization": {
158
+ "epsilon": 1e-05,
159
+ "type": "rms_norm"
160
+ }
161
+ }
162
+ }
163
+ },
164
+ "embeddings": {
165
+ "max_position_embeddings": 262144
166
+ },
167
+ "eos_token_id": 2,
168
+ "head": {
169
+ "normalization": {
170
+ "epsilon": 1e-05,
171
+ "type": "rms_norm"
172
+ }
173
+ },
174
+ "hidden_size": 5120,
175
+ "image_token_index": 10,
176
+ "model_type": "apriel2",
177
+ "tie_word_embeddings": false,
178
+ "transformers_version": "4.57.3",
179
+ "use_cache": true,
180
+ "vision_encoder": {
181
+ "adapter": {
182
+ "activation": "gelu_pytorch_tanh",
183
+ "add_linear_biases": true,
184
+ "gated": false,
185
+ "intermediate_size": 5120,
186
+ "type": "mlp"
187
+ },
188
+ "embeddings": {
189
+ "input_channels": 3,
190
+ "normalization": {
191
+ "epsilon": 1e-05,
192
+ "type": "rms_norm"
193
+ },
194
+ "patch_height": 16,
195
+ "patch_width": 16
196
+ },
197
+ "encoder": {
198
+ "block": {
199
+ "mixer": {
200
+ "add_linear_biases": false,
201
+ "causal": false,
202
+ "cross_document_attention": false,
203
+ "head_groups": 16,
204
+ "head_size": 64,
205
+ "heads": 16,
206
+ "rotary": {
207
+ "max_image_size": 1024,
208
+ "patch_size": 16,
209
+ "theta": 10000.0,
210
+ "type": "pixtral_2d"
211
+ },
212
+ "type": "attention"
213
+ },
214
+ "mlp": {
215
+ "activation": "silu",
216
+ "add_linear_biases": false,
217
+ "gated": true,
218
+ "intermediate_size": 4096,
219
+ "type": "mlp"
220
+ },
221
+ "normalization": {
222
+ "epsilon": 1e-05,
223
+ "type": "rms_norm"
224
+ }
225
+ },
226
+ "num_blocks": 24,
227
+ "type": "fixed"
228
+ },
229
+ "hidden_size": 1024,
230
+ "num_channels": 3,
231
+ "num_hidden_layers": 24,
232
+ "patch_size": 16
233
+ },
234
+ "vocab_size": 131072
235
+ }
preset_configs/Reg_Lklhd-26/config.json ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "sliding_window",
16
+ "sliding_window",
17
+ "attention",
18
+ "attention",
19
+ "sliding_window",
20
+ "sliding_window",
21
+ "sliding_window",
22
+ "sliding_window",
23
+ "gdn",
24
+ "sliding_window",
25
+ "kda",
26
+ "attention",
27
+ "sliding_window",
28
+ "sliding_window",
29
+ "attention",
30
+ "sliding_window",
31
+ "kda",
32
+ "sliding_window",
33
+ "sliding_window",
34
+ "attention",
35
+ "attention",
36
+ "attention",
37
+ "sliding_window",
38
+ "sliding_window",
39
+ "sliding_window",
40
+ "sliding_window",
41
+ "attention",
42
+ "attention",
43
+ "sliding_window",
44
+ "attention",
45
+ "sliding_window",
46
+ "sliding_window",
47
+ "attention",
48
+ "kda",
49
+ "sliding_window",
50
+ "sliding_window",
51
+ "sliding_window",
52
+ "gdn",
53
+ "sliding_window",
54
+ "kda",
55
+ "kda",
56
+ "gdn",
57
+ "sliding_window",
58
+ "attention",
59
+ "sliding_window",
60
+ "sliding_window",
61
+ "gdn",
62
+ "kda"
63
+ ],
64
+ "blocks": {
65
+ "sliding_window": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": 4096
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "attention": {
91
+ "mixer": {
92
+ "add_linear_biases": false,
93
+ "head_groups": 8,
94
+ "head_size": 128,
95
+ "heads": 32,
96
+ "rotary": {
97
+ "theta": 1000000000.0,
98
+ "type": "mistral_1d"
99
+ },
100
+ "type": "attention",
101
+ "window_size": null
102
+ },
103
+ "mlp": {
104
+ "activation": "silu",
105
+ "add_linear_biases": false,
106
+ "gated": true,
107
+ "intermediate_size": 14336,
108
+ "type": "mlp"
109
+ },
110
+ "normalization": {
111
+ "epsilon": 1e-05,
112
+ "type": "rms_norm"
113
+ }
114
+ },
115
+ "gdn": {
116
+ "mixer": {
117
+ "convolution_layer": {
118
+ "kernel_size": 4
119
+ },
120
+ "key_head_dim": 128,
121
+ "key_heads": 8,
122
+ "type": "gdn",
123
+ "value_head_dim": 128,
124
+ "value_heads": 32
125
+ },
126
+ "mlp": {
127
+ "activation": "silu",
128
+ "add_linear_biases": false,
129
+ "gated": true,
130
+ "intermediate_size": 14336,
131
+ "type": "mlp"
132
+ },
133
+ "normalization": {
134
+ "epsilon": 1e-05,
135
+ "type": "rms_norm"
136
+ }
137
+ },
138
+ "kda": {
139
+ "mixer": {
140
+ "convolution_layer": {
141
+ "kernel_size": 4
142
+ },
143
+ "head_dim": 128,
144
+ "heads": 32,
145
+ "normalization": {
146
+ "epsilon": 1e-05
147
+ },
148
+ "type": "kda"
149
+ },
150
+ "mlp": {
151
+ "activation": "silu",
152
+ "add_linear_biases": false,
153
+ "gated": true,
154
+ "intermediate_size": 14336,
155
+ "type": "mlp"
156
+ },
157
+ "normalization": {
158
+ "epsilon": 1e-05,
159
+ "type": "rms_norm"
160
+ }
161
+ }
162
+ }
163
+ },
164
+ "embeddings": {
165
+ "max_position_embeddings": 262144
166
+ },
167
+ "eos_token_id": 2,
168
+ "head": {
169
+ "normalization": {
170
+ "epsilon": 1e-05,
171
+ "type": "rms_norm"
172
+ }
173
+ },
174
+ "hidden_size": 5120,
175
+ "image_token_index": 10,
176
+ "model_type": "apriel2",
177
+ "tie_word_embeddings": false,
178
+ "transformers_version": "4.57.3",
179
+ "use_cache": true,
180
+ "vision_encoder": {
181
+ "adapter": {
182
+ "activation": "gelu_pytorch_tanh",
183
+ "add_linear_biases": true,
184
+ "gated": false,
185
+ "intermediate_size": 5120,
186
+ "type": "mlp"
187
+ },
188
+ "embeddings": {
189
+ "input_channels": 3,
190
+ "normalization": {
191
+ "epsilon": 1e-05,
192
+ "type": "rms_norm"
193
+ },
194
+ "patch_height": 16,
195
+ "patch_width": 16
196
+ },
197
+ "encoder": {
198
+ "block": {
199
+ "mixer": {
200
+ "add_linear_biases": false,
201
+ "causal": false,
202
+ "cross_document_attention": false,
203
+ "head_groups": 16,
204
+ "head_size": 64,
205
+ "heads": 16,
206
+ "rotary": {
207
+ "max_image_size": 1024,
208
+ "patch_size": 16,
209
+ "theta": 10000.0,
210
+ "type": "pixtral_2d"
211
+ },
212
+ "type": "attention"
213
+ },
214
+ "mlp": {
215
+ "activation": "silu",
216
+ "add_linear_biases": false,
217
+ "gated": true,
218
+ "intermediate_size": 4096,
219
+ "type": "mlp"
220
+ },
221
+ "normalization": {
222
+ "epsilon": 1e-05,
223
+ "type": "rms_norm"
224
+ }
225
+ },
226
+ "num_blocks": 24,
227
+ "type": "fixed"
228
+ },
229
+ "hidden_size": 1024,
230
+ "num_channels": 3,
231
+ "num_hidden_layers": 24,
232
+ "patch_size": 16
233
+ },
234
+ "vocab_size": 131072
235
+ }
preset_configs/all-attention/config.json ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "attention",
16
+ "attention",
17
+ "attention",
18
+ "attention",
19
+ "attention",
20
+ "attention",
21
+ "attention",
22
+ "attention",
23
+ "attention",
24
+ "attention",
25
+ "attention",
26
+ "attention",
27
+ "attention",
28
+ "attention",
29
+ "attention",
30
+ "attention",
31
+ "attention",
32
+ "attention",
33
+ "attention",
34
+ "attention",
35
+ "attention",
36
+ "attention",
37
+ "attention",
38
+ "attention",
39
+ "attention",
40
+ "attention",
41
+ "attention",
42
+ "attention",
43
+ "attention",
44
+ "attention",
45
+ "attention",
46
+ "attention",
47
+ "attention",
48
+ "attention",
49
+ "attention",
50
+ "attention",
51
+ "attention",
52
+ "attention",
53
+ "attention",
54
+ "attention",
55
+ "attention",
56
+ "attention",
57
+ "attention",
58
+ "attention",
59
+ "attention",
60
+ "attention",
61
+ "attention",
62
+ "attention"
63
+ ],
64
+ "blocks": {
65
+ "attention": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": null
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ }
90
+ }
91
+ },
92
+ "embeddings": {
93
+ "max_position_embeddings": 262144
94
+ },
95
+ "eos_token_id": 2,
96
+ "head": {
97
+ "normalization": {
98
+ "epsilon": 1e-05,
99
+ "type": "rms_norm"
100
+ }
101
+ },
102
+ "hidden_size": 5120,
103
+ "image_token_index": 10,
104
+ "model_type": "apriel2",
105
+ "tie_word_embeddings": false,
106
+ "transformers_version": "4.57.3",
107
+ "use_cache": true,
108
+ "vision_encoder": {
109
+ "adapter": {
110
+ "activation": "gelu_pytorch_tanh",
111
+ "add_linear_biases": true,
112
+ "gated": false,
113
+ "intermediate_size": 5120,
114
+ "type": "mlp"
115
+ },
116
+ "embeddings": {
117
+ "input_channels": 3,
118
+ "normalization": {
119
+ "epsilon": 1e-05,
120
+ "type": "rms_norm"
121
+ },
122
+ "patch_height": 16,
123
+ "patch_width": 16
124
+ },
125
+ "encoder": {
126
+ "block": {
127
+ "mixer": {
128
+ "add_linear_biases": false,
129
+ "causal": false,
130
+ "cross_document_attention": false,
131
+ "head_groups": 16,
132
+ "head_size": 64,
133
+ "heads": 16,
134
+ "rotary": {
135
+ "max_image_size": 1024,
136
+ "patch_size": 16,
137
+ "theta": 10000.0,
138
+ "type": "pixtral_2d"
139
+ },
140
+ "type": "attention"
141
+ },
142
+ "mlp": {
143
+ "activation": "silu",
144
+ "add_linear_biases": false,
145
+ "gated": true,
146
+ "intermediate_size": 4096,
147
+ "type": "mlp"
148
+ },
149
+ "normalization": {
150
+ "epsilon": 1e-05,
151
+ "type": "rms_norm"
152
+ }
153
+ },
154
+ "num_blocks": 24,
155
+ "type": "fixed"
156
+ },
157
+ "hidden_size": 1024,
158
+ "num_channels": 3,
159
+ "num_hidden_layers": 24,
160
+ "patch_size": 16
161
+ },
162
+ "vocab_size": 131072
163
+ }
preset_configs/extra_bayesian-mix-7/config.json ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Apriel2ForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_apriel2.Apriel2Config",
7
+ "AutoModel": "modeling_apriel2.Apriel2Model",
8
+ "AutoModelForImageTextToText": "modeling_apriel2.Apriel2ForConditionalGeneration"
9
+ },
10
+ "bos_token_id": 1,
11
+ "decoder": {
12
+ "num_blocks": 48,
13
+ "type": "pattern",
14
+ "pattern": [
15
+ "sliding_window",
16
+ "sliding_window",
17
+ "sliding_window",
18
+ "sliding_window",
19
+ "sliding_window",
20
+ "sliding_window",
21
+ "sliding_window",
22
+ "sliding_window",
23
+ "sliding_window",
24
+ "sliding_window",
25
+ "sliding_window",
26
+ "kda",
27
+ "sliding_window",
28
+ "sliding_window",
29
+ "sliding_window",
30
+ "sliding_window",
31
+ "sliding_window",
32
+ "sliding_window",
33
+ "sliding_window",
34
+ "sliding_window",
35
+ "sliding_window",
36
+ "sliding_window",
37
+ "sliding_window",
38
+ "sliding_window",
39
+ "sliding_window",
40
+ "sliding_window",
41
+ "sliding_window",
42
+ "sliding_window",
43
+ "sliding_window",
44
+ "sliding_window",
45
+ "sliding_window",
46
+ "sliding_window",
47
+ "sliding_window",
48
+ "sliding_window",
49
+ "sliding_window",
50
+ "sliding_window",
51
+ "sliding_window",
52
+ "sliding_window",
53
+ "sliding_window",
54
+ "sliding_window",
55
+ "sliding_window",
56
+ "sliding_window",
57
+ "sliding_window",
58
+ "sliding_window",
59
+ "sliding_window",
60
+ "sliding_window",
61
+ "sliding_window",
62
+ "kda"
63
+ ],
64
+ "blocks": {
65
+ "sliding_window": {
66
+ "mixer": {
67
+ "add_linear_biases": false,
68
+ "head_groups": 8,
69
+ "head_size": 128,
70
+ "heads": 32,
71
+ "rotary": {
72
+ "theta": 1000000000.0,
73
+ "type": "mistral_1d"
74
+ },
75
+ "type": "attention",
76
+ "window_size": 4096
77
+ },
78
+ "mlp": {
79
+ "activation": "silu",
80
+ "add_linear_biases": false,
81
+ "gated": true,
82
+ "intermediate_size": 14336,
83
+ "type": "mlp"
84
+ },
85
+ "normalization": {
86
+ "epsilon": 1e-05,
87
+ "type": "rms_norm"
88
+ }
89
+ },
90
+ "kda": {
91
+ "mixer": {
92
+ "convolution_layer": {
93
+ "kernel_size": 4
94
+ },
95
+ "head_dim": 128,
96
+ "heads": 32,
97
+ "normalization": {
98
+ "epsilon": 1e-05
99
+ },
100
+ "type": "kda"
101
+ },
102
+ "mlp": {
103
+ "activation": "silu",
104
+ "add_linear_biases": false,
105
+ "gated": true,
106
+ "intermediate_size": 14336,
107
+ "type": "mlp"
108
+ },
109
+ "normalization": {
110
+ "epsilon": 1e-05,
111
+ "type": "rms_norm"
112
+ }
113
+ }
114
+ }
115
+ },
116
+ "embeddings": {
117
+ "max_position_embeddings": 262144
118
+ },
119
+ "eos_token_id": 2,
120
+ "head": {
121
+ "normalization": {
122
+ "epsilon": 1e-05,
123
+ "type": "rms_norm"
124
+ }
125
+ },
126
+ "hidden_size": 5120,
127
+ "image_token_index": 10,
128
+ "model_type": "apriel2",
129
+ "tie_word_embeddings": false,
130
+ "transformers_version": "4.57.3",
131
+ "use_cache": true,
132
+ "vision_encoder": {
133
+ "adapter": {
134
+ "activation": "gelu_pytorch_tanh",
135
+ "add_linear_biases": true,
136
+ "gated": false,
137
+ "intermediate_size": 5120,
138
+ "type": "mlp"
139
+ },
140
+ "embeddings": {
141
+ "input_channels": 3,
142
+ "normalization": {
143
+ "epsilon": 1e-05,
144
+ "type": "rms_norm"
145
+ },
146
+ "patch_height": 16,
147
+ "patch_width": 16
148
+ },
149
+ "encoder": {
150
+ "block": {
151
+ "mixer": {
152
+ "add_linear_biases": false,
153
+ "causal": false,
154
+ "cross_document_attention": false,
155
+ "head_groups": 16,
156
+ "head_size": 64,
157
+ "heads": 16,
158
+ "rotary": {
159
+ "max_image_size": 1024,
160
+ "patch_size": 16,
161
+ "theta": 10000.0,
162
+ "type": "pixtral_2d"
163
+ },
164
+ "type": "attention"
165
+ },
166
+ "mlp": {
167
+ "activation": "silu",
168
+ "add_linear_biases": false,
169
+ "gated": true,
170
+ "intermediate_size": 4096,
171
+ "type": "mlp"
172
+ },
173
+ "normalization": {
174
+ "epsilon": 1e-05,
175
+ "type": "rms_norm"
176
+ }
177
+ },
178
+ "num_blocks": 24,
179
+ "type": "fixed"
180
+ },
181
+ "hidden_size": 1024,
182
+ "num_channels": 3,
183
+ "num_hidden_layers": 24,
184
+ "patch_size": 16
185
+ },
186
+ "vocab_size": 131072
187
+ }