B4K2xx commited on
Commit
1c2639a
·
verified ·
1 Parent(s): f296a89

Add m23 weights and results

Browse files
m23/benchmark_results.json ADDED
@@ -0,0 +1,540 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m23",
3
+ "checkpoint": "results/m23/final.pt",
4
+ "trained_step": 15258,
5
+ "num_fewshot": 0,
6
+ "limit": null,
7
+ "results": {
8
+ "lambada_openai": {
9
+ "name": "lambada_openai",
10
+ "alias": "lambada_openai",
11
+ "sample_len": 5153,
12
+ "perplexity,none": 657.6825390310986,
13
+ "perplexity_stderr,none": 36.041339126600164,
14
+ "acc,none": 0.16922181253638657,
15
+ "acc_stderr,none": 0.005223754842464426
16
+ },
17
+ "hellaswag": {
18
+ "name": "hellaswag",
19
+ "alias": "hellaswag",
20
+ "sample_len": 10042,
21
+ "acc,none": 0.2672774347739494,
22
+ "acc_stderr,none": 0.004416339450436367,
23
+ "acc_norm,none": 0.2771360286795459,
24
+ "acc_norm_stderr,none": 0.004466695023677567
25
+ },
26
+ "piqa": {
27
+ "name": "piqa",
28
+ "alias": "piqa",
29
+ "sample_len": 1838,
30
+ "acc,none": 0.5952121871599565,
31
+ "acc_stderr,none": 0.011452361375057018,
32
+ "acc_norm,none": 0.5892274211099021,
33
+ "acc_norm_stderr,none": 0.011478565556775618
34
+ },
35
+ "sciq": {
36
+ "name": "sciq",
37
+ "alias": "sciq",
38
+ "sample_len": 1000,
39
+ "acc,none": 0.68,
40
+ "acc_stderr,none": 0.014758652303574938,
41
+ "acc_norm,none": 0.591,
42
+ "acc_norm_stderr,none": 0.015555094373257946
43
+ },
44
+ "arc_easy": {
45
+ "name": "arc_easy",
46
+ "alias": "arc_easy",
47
+ "sample_len": 2376,
48
+ "acc,none": 0.4494949494949495,
49
+ "acc_stderr,none": 0.010207308833916014,
50
+ "acc_norm,none": 0.406986531986532,
51
+ "acc_norm_stderr,none": 0.010080695355466778
52
+ },
53
+ "winogrande": {
54
+ "name": "winogrande",
55
+ "alias": "winogrande",
56
+ "sample_len": 1267,
57
+ "acc,none": 0.5074980268350434,
58
+ "acc_stderr,none": 0.014050905521228646
59
+ },
60
+ "blimp_adjunct_island": {
61
+ "name": "blimp_adjunct_island",
62
+ "alias": "blimp_adjunct_island",
63
+ "sample_len": 1000,
64
+ "acc,none": 0.787,
65
+ "acc_stderr,none": 0.012953717566737312
66
+ },
67
+ "blimp_anaphor_gender_agreement": {
68
+ "name": "blimp_anaphor_gender_agreement",
69
+ "alias": "blimp_anaphor_gender_agreement",
70
+ "sample_len": 1000,
71
+ "acc,none": 0.958,
72
+ "acc_stderr,none": 0.006346359293033784
73
+ },
74
+ "blimp_anaphor_number_agreement": {
75
+ "name": "blimp_anaphor_number_agreement",
76
+ "alias": "blimp_anaphor_number_agreement",
77
+ "sample_len": 1000,
78
+ "acc,none": 0.982,
79
+ "acc_stderr,none": 0.00420638724961144
80
+ },
81
+ "blimp_animate_subject_passive": {
82
+ "name": "blimp_animate_subject_passive",
83
+ "alias": "blimp_animate_subject_passive",
84
+ "sample_len": 1000,
85
+ "acc,none": 0.741,
86
+ "acc_stderr,none": 0.013860415257527861
87
+ },
88
+ "blimp_animate_subject_trans": {
89
+ "name": "blimp_animate_subject_trans",
90
+ "alias": "blimp_animate_subject_trans",
91
+ "sample_len": 1000,
92
+ "acc,none": 0.903,
93
+ "acc_stderr,none": 0.009363689373248062
94
+ },
95
+ "blimp_causative": {
96
+ "name": "blimp_causative",
97
+ "alias": "blimp_causative",
98
+ "sample_len": 1000,
99
+ "acc,none": 0.704,
100
+ "acc_stderr,none": 0.01444273494157504
101
+ },
102
+ "blimp_complex_NP_island": {
103
+ "name": "blimp_complex_NP_island",
104
+ "alias": "blimp_complex_NP_island",
105
+ "sample_len": 1000,
106
+ "acc,none": 0.555,
107
+ "acc_stderr,none": 0.015723301886761007
108
+ },
109
+ "blimp_coordinate_structure_constraint_complex_left_branch": {
110
+ "name": "blimp_coordinate_structure_constraint_complex_left_branch",
111
+ "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
112
+ "sample_len": 1000,
113
+ "acc,none": 0.447,
114
+ "acc_stderr,none": 0.015730176046009098
115
+ },
116
+ "blimp_coordinate_structure_constraint_object_extraction": {
117
+ "name": "blimp_coordinate_structure_constraint_object_extraction",
118
+ "alias": "blimp_coordinate_structure_constraint_object_extraction",
119
+ "sample_len": 1000,
120
+ "acc,none": 0.75,
121
+ "acc_stderr,none": 0.013699915608779773
122
+ },
123
+ "blimp_determiner_noun_agreement_1": {
124
+ "name": "blimp_determiner_noun_agreement_1",
125
+ "alias": "blimp_determiner_noun_agreement_1",
126
+ "sample_len": 1000,
127
+ "acc,none": 0.971,
128
+ "acc_stderr,none": 0.005309160685756948
129
+ },
130
+ "blimp_determiner_noun_agreement_2": {
131
+ "name": "blimp_determiner_noun_agreement_2",
132
+ "alias": "blimp_determiner_noun_agreement_2",
133
+ "sample_len": 1000,
134
+ "acc,none": 0.949,
135
+ "acc_stderr,none": 0.0069604200625714725
136
+ },
137
+ "blimp_determiner_noun_agreement_irregular_1": {
138
+ "name": "blimp_determiner_noun_agreement_irregular_1",
139
+ "alias": "blimp_determiner_noun_agreement_irregular_1",
140
+ "sample_len": 1000,
141
+ "acc,none": 0.857,
142
+ "acc_stderr,none": 0.011075814808567074
143
+ },
144
+ "blimp_determiner_noun_agreement_irregular_2": {
145
+ "name": "blimp_determiner_noun_agreement_irregular_2",
146
+ "alias": "blimp_determiner_noun_agreement_irregular_2",
147
+ "sample_len": 1000,
148
+ "acc,none": 0.937,
149
+ "acc_stderr,none": 0.007687007876286461
150
+ },
151
+ "blimp_determiner_noun_agreement_with_adj_2": {
152
+ "name": "blimp_determiner_noun_agreement_with_adj_2",
153
+ "alias": "blimp_determiner_noun_agreement_with_adj_2",
154
+ "sample_len": 1000,
155
+ "acc,none": 0.915,
156
+ "acc_stderr,none": 0.008823426366942316
157
+ },
158
+ "blimp_determiner_noun_agreement_with_adj_irregular_1": {
159
+ "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
160
+ "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
161
+ "sample_len": 1000,
162
+ "acc,none": 0.857,
163
+ "acc_stderr,none": 0.011075814808567074
164
+ },
165
+ "blimp_determiner_noun_agreement_with_adj_irregular_2": {
166
+ "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
167
+ "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
168
+ "sample_len": 1000,
169
+ "acc,none": 0.908,
170
+ "acc_stderr,none": 0.009144376393151129
171
+ },
172
+ "blimp_determiner_noun_agreement_with_adjective_1": {
173
+ "name": "blimp_determiner_noun_agreement_with_adjective_1",
174
+ "alias": "blimp_determiner_noun_agreement_with_adjective_1",
175
+ "sample_len": 1000,
176
+ "acc,none": 0.939,
177
+ "acc_stderr,none": 0.007572076091557476
178
+ },
179
+ "blimp_distractor_agreement_relational_noun": {
180
+ "name": "blimp_distractor_agreement_relational_noun",
181
+ "alias": "blimp_distractor_agreement_relational_noun",
182
+ "sample_len": 1000,
183
+ "acc,none": 0.844,
184
+ "acc_stderr,none": 0.011480235006122295
185
+ },
186
+ "blimp_distractor_agreement_relative_clause": {
187
+ "name": "blimp_distractor_agreement_relative_clause",
188
+ "alias": "blimp_distractor_agreement_relative_clause",
189
+ "sample_len": 1000,
190
+ "acc,none": 0.763,
191
+ "acc_stderr,none": 0.013454070462577935
192
+ },
193
+ "blimp_drop_argument": {
194
+ "name": "blimp_drop_argument",
195
+ "alias": "blimp_drop_argument",
196
+ "sample_len": 1000,
197
+ "acc,none": 0.751,
198
+ "acc_stderr,none": 0.013681600278702275
199
+ },
200
+ "blimp_ellipsis_n_bar_1": {
201
+ "name": "blimp_ellipsis_n_bar_1",
202
+ "alias": "blimp_ellipsis_n_bar_1",
203
+ "sample_len": 1000,
204
+ "acc,none": 0.772,
205
+ "acc_stderr,none": 0.013273740700804434
206
+ },
207
+ "blimp_ellipsis_n_bar_2": {
208
+ "name": "blimp_ellipsis_n_bar_2",
209
+ "alias": "blimp_ellipsis_n_bar_2",
210
+ "sample_len": 1000,
211
+ "acc,none": 0.843,
212
+ "acc_stderr,none": 0.011510146979230262
213
+ },
214
+ "blimp_existential_there_object_raising": {
215
+ "name": "blimp_existential_there_object_raising",
216
+ "alias": "blimp_existential_there_object_raising",
217
+ "sample_len": 1000,
218
+ "acc,none": 0.821,
219
+ "acc_stderr,none": 0.012128730605719056
220
+ },
221
+ "blimp_existential_there_quantifiers_1": {
222
+ "name": "blimp_existential_there_quantifiers_1",
223
+ "alias": "blimp_existential_there_quantifiers_1",
224
+ "sample_len": 1000,
225
+ "acc,none": 0.978,
226
+ "acc_stderr,none": 0.004640855259274689
227
+ },
228
+ "blimp_existential_there_quantifiers_2": {
229
+ "name": "blimp_existential_there_quantifiers_2",
230
+ "alias": "blimp_existential_there_quantifiers_2",
231
+ "sample_len": 1000,
232
+ "acc,none": 0.114,
233
+ "acc_stderr,none": 0.010055103435823278
234
+ },
235
+ "blimp_existential_there_subject_raising": {
236
+ "name": "blimp_existential_there_subject_raising",
237
+ "alias": "blimp_existential_there_subject_raising",
238
+ "sample_len": 1000,
239
+ "acc,none": 0.814,
240
+ "acc_stderr,none": 0.012310790208412926
241
+ },
242
+ "blimp_expletive_it_object_raising": {
243
+ "name": "blimp_expletive_it_object_raising",
244
+ "alias": "blimp_expletive_it_object_raising",
245
+ "sample_len": 1000,
246
+ "acc,none": 0.768,
247
+ "acc_stderr,none": 0.013354937452281623
248
+ },
249
+ "blimp_inchoative": {
250
+ "name": "blimp_inchoative",
251
+ "alias": "blimp_inchoative",
252
+ "sample_len": 1000,
253
+ "acc,none": 0.641,
254
+ "acc_stderr,none": 0.01517726422479853
255
+ },
256
+ "blimp_intransitive": {
257
+ "name": "blimp_intransitive",
258
+ "alias": "blimp_intransitive",
259
+ "sample_len": 1000,
260
+ "acc,none": 0.745,
261
+ "acc_stderr,none": 0.013790038620872863
262
+ },
263
+ "blimp_irregular_past_participle_adjectives": {
264
+ "name": "blimp_irregular_past_participle_adjectives",
265
+ "alias": "blimp_irregular_past_participle_adjectives",
266
+ "sample_len": 1000,
267
+ "acc,none": 0.751,
268
+ "acc_stderr,none": 0.013681600278702275
269
+ },
270
+ "blimp_irregular_past_participle_verbs": {
271
+ "name": "blimp_irregular_past_participle_verbs",
272
+ "alias": "blimp_irregular_past_participle_verbs",
273
+ "sample_len": 1000,
274
+ "acc,none": 0.862,
275
+ "acc_stderr,none": 0.010912152632504508
276
+ },
277
+ "blimp_irregular_plural_subject_verb_agreement_1": {
278
+ "name": "blimp_irregular_plural_subject_verb_agreement_1",
279
+ "alias": "blimp_irregular_plural_subject_verb_agreement_1",
280
+ "sample_len": 1000,
281
+ "acc,none": 0.877,
282
+ "acc_stderr,none": 0.010391293421849803
283
+ },
284
+ "blimp_irregular_plural_subject_verb_agreement_2": {
285
+ "name": "blimp_irregular_plural_subject_verb_agreement_2",
286
+ "alias": "blimp_irregular_plural_subject_verb_agreement_2",
287
+ "sample_len": 1000,
288
+ "acc,none": 0.861,
289
+ "acc_stderr,none": 0.010945263761042892
290
+ },
291
+ "blimp_left_branch_island_echo_question": {
292
+ "name": "blimp_left_branch_island_echo_question",
293
+ "alias": "blimp_left_branch_island_echo_question",
294
+ "sample_len": 1000,
295
+ "acc,none": 0.505,
296
+ "acc_stderr,none": 0.015818508944436743
297
+ },
298
+ "blimp_left_branch_island_simple_question": {
299
+ "name": "blimp_left_branch_island_simple_question",
300
+ "alias": "blimp_left_branch_island_simple_question",
301
+ "sample_len": 1000,
302
+ "acc,none": 0.449,
303
+ "acc_stderr,none": 0.015736792768752044
304
+ },
305
+ "blimp_matrix_question_npi_licensor_present": {
306
+ "name": "blimp_matrix_question_npi_licensor_present",
307
+ "alias": "blimp_matrix_question_npi_licensor_present",
308
+ "sample_len": 1000,
309
+ "acc,none": 0.46,
310
+ "acc_stderr,none": 0.015768596914394403
311
+ },
312
+ "blimp_npi_present_1": {
313
+ "name": "blimp_npi_present_1",
314
+ "alias": "blimp_npi_present_1",
315
+ "sample_len": 1000,
316
+ "acc,none": 0.603,
317
+ "acc_stderr,none": 0.015480007449307982
318
+ },
319
+ "blimp_npi_present_2": {
320
+ "name": "blimp_npi_present_2",
321
+ "alias": "blimp_npi_present_2",
322
+ "sample_len": 1000,
323
+ "acc,none": 0.595,
324
+ "acc_stderr,none": 0.01553113699045296
325
+ },
326
+ "blimp_only_npi_licensor_present": {
327
+ "name": "blimp_only_npi_licensor_present",
328
+ "alias": "blimp_only_npi_licensor_present",
329
+ "sample_len": 1000,
330
+ "acc,none": 0.905,
331
+ "acc_stderr,none": 0.009276910103103353
332
+ },
333
+ "blimp_only_npi_scope": {
334
+ "name": "blimp_only_npi_scope",
335
+ "alias": "blimp_only_npi_scope",
336
+ "sample_len": 1000,
337
+ "acc,none": 0.673,
338
+ "acc_stderr,none": 0.014842213153411162
339
+ },
340
+ "blimp_passive_1": {
341
+ "name": "blimp_passive_1",
342
+ "alias": "blimp_passive_1",
343
+ "sample_len": 1000,
344
+ "acc,none": 0.879,
345
+ "acc_stderr,none": 0.010318210380946179
346
+ },
347
+ "blimp_passive_2": {
348
+ "name": "blimp_passive_2",
349
+ "alias": "blimp_passive_2",
350
+ "sample_len": 1000,
351
+ "acc,none": 0.869,
352
+ "acc_stderr,none": 0.0106748748448379
353
+ },
354
+ "blimp_principle_A_c_command": {
355
+ "name": "blimp_principle_A_c_command",
356
+ "alias": "blimp_principle_A_c_command",
357
+ "sample_len": 1000,
358
+ "acc,none": 0.628,
359
+ "acc_stderr,none": 0.01529214994204052
360
+ },
361
+ "blimp_principle_A_case_1": {
362
+ "name": "blimp_principle_A_case_1",
363
+ "alias": "blimp_principle_A_case_1",
364
+ "sample_len": 1000,
365
+ "acc,none": 1.0,
366
+ "acc_stderr,none": 0.0
367
+ },
368
+ "blimp_principle_A_case_2": {
369
+ "name": "blimp_principle_A_case_2",
370
+ "alias": "blimp_principle_A_case_2",
371
+ "sample_len": 1000,
372
+ "acc,none": 0.884,
373
+ "acc_stderr,none": 0.010131468138756922
374
+ },
375
+ "blimp_principle_A_domain_1": {
376
+ "name": "blimp_principle_A_domain_1",
377
+ "alias": "blimp_principle_A_domain_1",
378
+ "sample_len": 1000,
379
+ "acc,none": 0.951,
380
+ "acc_stderr,none": 0.006829761756140959
381
+ },
382
+ "blimp_principle_A_domain_2": {
383
+ "name": "blimp_principle_A_domain_2",
384
+ "alias": "blimp_principle_A_domain_2",
385
+ "sample_len": 1000,
386
+ "acc,none": 0.729,
387
+ "acc_stderr,none": 0.014062601350986125
388
+ },
389
+ "blimp_principle_A_domain_3": {
390
+ "name": "blimp_principle_A_domain_3",
391
+ "alias": "blimp_principle_A_domain_3",
392
+ "sample_len": 1000,
393
+ "acc,none": 0.625,
394
+ "acc_stderr,none": 0.015316971293620996
395
+ },
396
+ "blimp_principle_A_reconstruction": {
397
+ "name": "blimp_principle_A_reconstruction",
398
+ "alias": "blimp_principle_A_reconstruction",
399
+ "sample_len": 1000,
400
+ "acc,none": 0.467,
401
+ "acc_stderr,none": 0.015784807891138876
402
+ },
403
+ "blimp_regular_plural_subject_verb_agreement_1": {
404
+ "name": "blimp_regular_plural_subject_verb_agreement_1",
405
+ "alias": "blimp_regular_plural_subject_verb_agreement_1",
406
+ "sample_len": 1000,
407
+ "acc,none": 0.848,
408
+ "acc_stderr,none": 0.01135891830347525
409
+ },
410
+ "blimp_regular_plural_subject_verb_agreement_2": {
411
+ "name": "blimp_regular_plural_subject_verb_agreement_2",
412
+ "alias": "blimp_regular_plural_subject_verb_agreement_2",
413
+ "sample_len": 1000,
414
+ "acc,none": 0.871,
415
+ "acc_stderr,none": 0.010605256784796466
416
+ },
417
+ "blimp_sentential_negation_npi_licensor_present": {
418
+ "name": "blimp_sentential_negation_npi_licensor_present",
419
+ "alias": "blimp_sentential_negation_npi_licensor_present",
420
+ "sample_len": 1000,
421
+ "acc,none": 0.995,
422
+ "acc_stderr,none": 0.0022315868748448895
423
+ },
424
+ "blimp_sentential_negation_npi_scope": {
425
+ "name": "blimp_sentential_negation_npi_scope",
426
+ "alias": "blimp_sentential_negation_npi_scope",
427
+ "sample_len": 1000,
428
+ "acc,none": 0.652,
429
+ "acc_stderr,none": 0.015070604603768328
430
+ },
431
+ "blimp_sentential_subject_island": {
432
+ "name": "blimp_sentential_subject_island",
433
+ "alias": "blimp_sentential_subject_island",
434
+ "sample_len": 1000,
435
+ "acc,none": 0.493,
436
+ "acc_stderr,none": 0.01581774956184353
437
+ },
438
+ "blimp_superlative_quantifiers_1": {
439
+ "name": "blimp_superlative_quantifiers_1",
440
+ "alias": "blimp_superlative_quantifiers_1",
441
+ "sample_len": 1000,
442
+ "acc,none": 0.822,
443
+ "acc_stderr,none": 0.012102167676183571
444
+ },
445
+ "blimp_superlative_quantifiers_2": {
446
+ "name": "blimp_superlative_quantifiers_2",
447
+ "alias": "blimp_superlative_quantifiers_2",
448
+ "sample_len": 1000,
449
+ "acc,none": 0.731,
450
+ "acc_stderr,none": 0.014029819522568203
451
+ },
452
+ "blimp_tough_vs_raising_1": {
453
+ "name": "blimp_tough_vs_raising_1",
454
+ "alias": "blimp_tough_vs_raising_1",
455
+ "sample_len": 1000,
456
+ "acc,none": 0.543,
457
+ "acc_stderr,none": 0.0157606915901365
458
+ },
459
+ "blimp_tough_vs_raising_2": {
460
+ "name": "blimp_tough_vs_raising_2",
461
+ "alias": "blimp_tough_vs_raising_2",
462
+ "sample_len": 1000,
463
+ "acc,none": 0.898,
464
+ "acc_stderr,none": 0.009575368801653944
465
+ },
466
+ "blimp_transitive": {
467
+ "name": "blimp_transitive",
468
+ "alias": "blimp_transitive",
469
+ "sample_len": 1000,
470
+ "acc,none": 0.812,
471
+ "acc_stderr,none": 0.012361586015103758
472
+ },
473
+ "blimp_wh_island": {
474
+ "name": "blimp_wh_island",
475
+ "alias": "blimp_wh_island",
476
+ "sample_len": 1000,
477
+ "acc,none": 0.719,
478
+ "acc_stderr,none": 0.01422115470843494
479
+ },
480
+ "blimp_wh_questions_object_gap": {
481
+ "name": "blimp_wh_questions_object_gap",
482
+ "alias": "blimp_wh_questions_object_gap",
483
+ "sample_len": 1000,
484
+ "acc,none": 0.684,
485
+ "acc_stderr,none": 0.014709193056057166
486
+ },
487
+ "blimp_wh_questions_subject_gap": {
488
+ "name": "blimp_wh_questions_subject_gap",
489
+ "alias": "blimp_wh_questions_subject_gap",
490
+ "sample_len": 1000,
491
+ "acc,none": 0.877,
492
+ "acc_stderr,none": 0.010391293421849803
493
+ },
494
+ "blimp_wh_questions_subject_gap_long_distance": {
495
+ "name": "blimp_wh_questions_subject_gap_long_distance",
496
+ "alias": "blimp_wh_questions_subject_gap_long_distance",
497
+ "sample_len": 1000,
498
+ "acc,none": 0.849,
499
+ "acc_stderr,none": 0.011328165223341657
500
+ },
501
+ "blimp_wh_vs_that_no_gap": {
502
+ "name": "blimp_wh_vs_that_no_gap",
503
+ "alias": "blimp_wh_vs_that_no_gap",
504
+ "sample_len": 1000,
505
+ "acc,none": 0.948,
506
+ "acc_stderr,none": 0.007024624213817088
507
+ },
508
+ "blimp_wh_vs_that_no_gap_long_distance": {
509
+ "name": "blimp_wh_vs_that_no_gap_long_distance",
510
+ "alias": "blimp_wh_vs_that_no_gap_long_distance",
511
+ "sample_len": 1000,
512
+ "acc,none": 0.966,
513
+ "acc_stderr,none": 0.0057338361396953854
514
+ },
515
+ "blimp_wh_vs_that_with_gap": {
516
+ "name": "blimp_wh_vs_that_with_gap",
517
+ "alias": "blimp_wh_vs_that_with_gap",
518
+ "sample_len": 1000,
519
+ "acc,none": 0.526,
520
+ "acc_stderr,none": 0.015797897758042797
521
+ },
522
+ "blimp_wh_vs_that_with_gap_long_distance": {
523
+ "name": "blimp_wh_vs_that_with_gap_long_distance",
524
+ "alias": "blimp_wh_vs_that_with_gap_long_distance",
525
+ "sample_len": 1000,
526
+ "acc,none": 0.23,
527
+ "acc_stderr,none": 0.01331455133593608
528
+ },
529
+ "blimp": {
530
+ "alias": "blimp",
531
+ "name": "blimp",
532
+ "sample_len": 67000,
533
+ "acc,none": 0.761955223880597,
534
+ "acc_stderr,none": 0.0014852092525077722,
535
+ "sample_count": {
536
+ "acc,none": 67000
537
+ }
538
+ }
539
+ }
540
+ }
m23/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m23",
3
+ "run_name": "m23-gqa-rope-dense-adamw-seed1338",
4
+ "data_dir": "data/fineweb-edu",
5
+ "vocab_size": 50257,
6
+ "block_size": 1024,
7
+ "d_model": 512,
8
+ "num_layers": 12,
9
+ "num_heads": 8,
10
+ "ffn_mult": 4,
11
+ "dropout": 0.0,
12
+ "bias": false,
13
+ "norm_type": "rmsnorm",
14
+ "tie_embeddings": true,
15
+ "attention_type": "gqa",
16
+ "gqa_num_kv_heads": 2,
17
+ "mla_latent_dim": 64,
18
+ "pos_type": "rope",
19
+ "rope_base": 10000,
20
+ "ffn_type": "dense",
21
+ "moe_num_experts": 8,
22
+ "moe_top_k": 2,
23
+ "optimizer": "adamw",
24
+ "lr": 0.0006,
25
+ "min_lr": 6e-05,
26
+ "weight_decay": 0.1,
27
+ "beta1": 0.9,
28
+ "beta2": 0.95,
29
+ "grad_clip": 1.0,
30
+ "muon_lr": 0.02,
31
+ "muon_momentum": 0.95,
32
+ "training_tokens": 1000000000,
33
+ "warmup_tokens": 6000000,
34
+ "lr_schedule": "cosine",
35
+ "batch_size": 4,
36
+ "grad_accum_steps": 16,
37
+ "device": "cuda",
38
+ "dtype": "bfloat16",
39
+ "compile": true,
40
+ "seed": 1338,
41
+ "eval_interval": 250,
42
+ "eval_iters": 100,
43
+ "log_interval": 10,
44
+ "checkpoint_interval": 1000,
45
+ "results_dir": "results",
46
+ "mlflow_experiment": "slm-ablation",
47
+ "mlflow_tracking_uri": "sqlite:///mlflow.db",
48
+ "mlflow_artifact_location": "./mlartifacts",
49
+ "seed_group": "m08"
50
+ }
m23/eval_results.json ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m23",
3
+ "checkpoint": "/home/akshat/codemindv2/results/m23/final.pt",
4
+ "trained_step": 15258,
5
+ "total_params": 58774528,
6
+ "active_params": 58774528,
7
+ "flops_per_forward": 117549056.0,
8
+ "val_loss": 3.4765728563070297,
9
+ "val_perplexity": 32.3486683513085,
10
+ "bits_per_token": 5.015634419083331,
11
+ "ppl_tokens_evaluated": 655360,
12
+ "prefill_tok_per_sec": 189550.8386998898,
13
+ "inference_tok_per_sec": 201.23427577598775,
14
+ "inference_tok_per_sec_kvcache": 210.77698908250312,
15
+ "kvcache_speedup": 1.047420914104803,
16
+ "gen_tokens": 128,
17
+ "peak_infer_vram_gb": 0.366,
18
+ "peak_train_vram_gb": 3.993,
19
+ "closeness_answer_ppl": 24.479,
20
+ "closeness_hit_rate": 0.4,
21
+ "closeness_token_f1": 0.051,
22
+ "closeness_n": 20,
23
+ "closeness_items": [
24
+ {
25
+ "prompt": "The capital of France is",
26
+ "answer": " Paris",
27
+ "generated": " the French capital of the province",
28
+ "answer_nll": 2.3204,
29
+ "answer_ppl": 10.179,
30
+ "hit": false,
31
+ "token_f1": 0.0
32
+ },
33
+ {
34
+ "prompt": "The capital of Japan is",
35
+ "answer": " Tokyo",
36
+ "generated": " the K\u014dsh\u014d,",
37
+ "answer_nll": 2.3939,
38
+ "answer_ppl": 10.956,
39
+ "hit": false,
40
+ "token_f1": 0.0
41
+ },
42
+ {
43
+ "prompt": "The largest planet in the solar system is",
44
+ "answer": " Jupiter",
45
+ "generated": " the Sun.\nThe Sun",
46
+ "answer_nll": 4.4133,
47
+ "answer_ppl": 82.538,
48
+ "hit": false,
49
+ "token_f1": 0.0
50
+ },
51
+ {
52
+ "prompt": "Water is made of hydrogen and",
53
+ "answer": " oxygen",
54
+ "generated": " oxygen.\nThe hydrogen is",
55
+ "answer_nll": 1.1928,
56
+ "answer_ppl": 3.296,
57
+ "hit": true,
58
+ "token_f1": 0.0
59
+ },
60
+ {
61
+ "prompt": "The opposite of hot is",
62
+ "answer": " cold",
63
+ "generated": " the opposite of hot.\n",
64
+ "answer_nll": 3.8481,
65
+ "answer_ppl": 46.903,
66
+ "hit": false,
67
+ "token_f1": 0.0
68
+ },
69
+ {
70
+ "prompt": "The opposite of up is",
71
+ "answer": " down",
72
+ "generated": " the same as up.\n",
73
+ "answer_nll": 4.1841,
74
+ "answer_ppl": 65.637,
75
+ "hit": false,
76
+ "token_f1": 0.0
77
+ },
78
+ {
79
+ "prompt": "Two plus two equals",
80
+ "answer": " four",
81
+ "generated": " two.\n- The first",
82
+ "answer_nll": 2.8029,
83
+ "answer_ppl": 16.493,
84
+ "hit": false,
85
+ "token_f1": 0.0
86
+ },
87
+ {
88
+ "prompt": "The sun rises in the",
89
+ "answer": " east",
90
+ "generated": " middle of the night sky and",
91
+ "answer_nll": 3.0683,
92
+ "answer_ppl": 21.506,
93
+ "hit": false,
94
+ "token_f1": 0.0
95
+ },
96
+ {
97
+ "prompt": "The first president of the United States was George",
98
+ "answer": " Washington",
99
+ "generated": " Washington, who was the first",
100
+ "answer_nll": 0.5824,
101
+ "answer_ppl": 1.79,
102
+ "hit": true,
103
+ "token_f1": 0.0
104
+ },
105
+ {
106
+ "prompt": "The speed of light is approximately 300,000 kilometers per",
107
+ "answer": " second",
108
+ "generated": " second.\nThe speed of",
109
+ "answer_nll": 0.1998,
110
+ "answer_ppl": 1.221,
111
+ "hit": true,
112
+ "token_f1": 0.0
113
+ },
114
+ {
115
+ "prompt": "A triangle has three",
116
+ "answer": " sides",
117
+ "generated": " sides and a triangle has four",
118
+ "answer_nll": 1.2028,
119
+ "answer_ppl": 3.329,
120
+ "hit": true,
121
+ "token_f1": 0.286
122
+ },
123
+ {
124
+ "prompt": "The chemical symbol for gold is",
125
+ "answer": " Au",
126
+ "generated": " the gold-silver-silver",
127
+ "answer_nll": 8.5195,
128
+ "answer_ppl": 5011.537,
129
+ "hit": false,
130
+ "token_f1": 0.0
131
+ },
132
+ {
133
+ "prompt": "The freezing point of water in Celsius is",
134
+ "answer": " zero",
135
+ "generated": " about 1.5 degrees Celsius",
136
+ "answer_nll": 5.5563,
137
+ "answer_ppl": 258.868,
138
+ "hit": false,
139
+ "token_f1": 0.0
140
+ },
141
+ {
142
+ "prompt": "The author of Romeo and Juliet is William",
143
+ "answer": " Shakespeare",
144
+ "generated": " Shakespeare.\nThe author of",
145
+ "answer_nll": 0.3232,
146
+ "answer_ppl": 1.382,
147
+ "hit": true,
148
+ "token_f1": 0.0
149
+ },
150
+ {
151
+ "prompt": "The Earth revolves around the",
152
+ "answer": " Sun",
153
+ "generated": " sun.\nThe sun is",
154
+ "answer_nll": 1.018,
155
+ "answer_ppl": 2.768,
156
+ "hit": true,
157
+ "token_f1": 0.4
158
+ },
159
+ {
160
+ "prompt": "The plural of mouse is",
161
+ "answer": " mice",
162
+ "generated": " the plural of mouse.\n",
163
+ "answer_nll": 9.8359,
164
+ "answer_ppl": 18693.672,
165
+ "hit": false,
166
+ "token_f1": 0.0
167
+ },
168
+ {
169
+ "prompt": "Photosynthesis occurs in the",
170
+ "answer": " leaves",
171
+ "generated": " same way as the production of",
172
+ "answer_nll": 5.8299,
173
+ "answer_ppl": 340.327,
174
+ "hit": false,
175
+ "token_f1": 0.0
176
+ },
177
+ {
178
+ "prompt": "The human heart pumps",
179
+ "answer": " blood",
180
+ "generated": " blood to the brain, which",
181
+ "answer_nll": 1.6457,
182
+ "answer_ppl": 5.185,
183
+ "hit": true,
184
+ "token_f1": 0.333
185
+ },
186
+ {
187
+ "prompt": "The currency used in the United States is the",
188
+ "answer": " dollar",
189
+ "generated": " US dollar.\nThe US",
190
+ "answer_nll": 2.3366,
191
+ "answer_ppl": 10.346,
192
+ "hit": true,
193
+ "token_f1": 0.0
194
+ },
195
+ {
196
+ "prompt": "The tallest animal in the world is the",
197
+ "answer": " giraffe",
198
+ "generated": " giant tortoise.\nThe",
199
+ "answer_nll": 2.6826,
200
+ "answer_ppl": 14.623,
201
+ "hit": false,
202
+ "token_f1": 0.0
203
+ }
204
+ ],
205
+ "generation_samples": {
206
+ "The most important thing about sleep is": "The most important thing about sleep is knowing what\u2019s going on. However, some people may notice:\n- Sleep difficulties:\n- Dizziness or lightheadedness\n- Sleep apnea\n- Trouble concentrating\nMost people will also notice:\n- Swelling on their head, neck, or spine\n- Increased",
207
+ "Scientists have discovered that": "Scientists have discovered that, in a recent study published in the British journal Nature, an Australian geneticist has uncovered a new genetic signature of a rare parasite called Listeria monocytogenes. Using a computer simulation of the gene, the scientists found that one of the parasite\u2019s genes, E2,",
208
+ "In the history of mathematics,": "In the history of mathematics, the term \"curntage\" is often used to describe a series of fields known as fields or sets of fields. The term \"curntage\" is often used to describe a collection of fields or sets of fields. In the history of mathematics, the term \"curntage\" is often",
209
+ "Python code for prime number": "Python code for prime number numbers, decimal numbers, and expressions. In addition, for example, \u201c2x4\u201d or \u201c2x4.9\u201d is used in the following form:\n- (x1) 1 = (2x3)\n- (x1) 1 = (",
210
+ "The best way to learn a new language is": "The best way to learn a new language is to learn about it. So if you have an understanding of a language, you will feel confident to learn it!\nBefore you begin learning a language, you should be aware of the terms and terms of the language, and you should understand them carefully. We\u2019ve discussed everything with the language"
211
+ }
212
+ }
m23/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9110d16c9c6575242f9b79fe26577a073e0774188236fc2c66dd19a757d3b017
3
+ size 235107624