B4K2xx commited on
Commit
ae1dbd2
·
verified ·
1 Parent(s): 9d66492

Upload folder using huggingface_hub

Browse files
m18/benchmark_results.json ADDED
@@ -0,0 +1,540 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m18",
3
+ "checkpoint": "results/m18/final.pt",
4
+ "trained_step": 15258,
5
+ "num_fewshot": 0,
6
+ "limit": null,
7
+ "results": {
8
+ "lambada_openai": {
9
+ "name": "lambada_openai",
10
+ "alias": "lambada_openai",
11
+ "sample_len": 5153,
12
+ "perplexity,none": 1004.0061038384438,
13
+ "perplexity_stderr,none": 54.23896500033554,
14
+ "acc,none": 0.1568018629924316,
15
+ "acc_stderr,none": 0.005065852040981164
16
+ },
17
+ "hellaswag": {
18
+ "name": "hellaswag",
19
+ "alias": "hellaswag",
20
+ "sample_len": 10042,
21
+ "acc,none": 0.27016530571599284,
22
+ "acc_stderr,none": 0.004431375549911545,
23
+ "acc_norm,none": 0.26976697868950406,
24
+ "acc_norm_stderr,none": 0.004429315788310707
25
+ },
26
+ "piqa": {
27
+ "name": "piqa",
28
+ "alias": "piqa",
29
+ "sample_len": 1838,
30
+ "acc,none": 0.5859630032644179,
31
+ "acc_stderr,none": 0.011492118481417717,
32
+ "acc_norm,none": 0.588139281828074,
33
+ "acc_norm_stderr,none": 0.011483141106304523
34
+ },
35
+ "sciq": {
36
+ "name": "sciq",
37
+ "alias": "sciq",
38
+ "sample_len": 1000,
39
+ "acc,none": 0.69,
40
+ "acc_stderr,none": 0.014632638658632766,
41
+ "acc_norm,none": 0.622,
42
+ "acc_norm_stderr,none": 0.015341165254026547
43
+ },
44
+ "arc_easy": {
45
+ "name": "arc_easy",
46
+ "alias": "arc_easy",
47
+ "sample_len": 2376,
48
+ "acc,none": 0.4414983164983165,
49
+ "acc_stderr,none": 0.010189314382749861,
50
+ "acc_norm,none": 0.3968855218855219,
51
+ "acc_norm_stderr,none": 0.010039236800583027
52
+ },
53
+ "winogrande": {
54
+ "name": "winogrande",
55
+ "alias": "winogrande",
56
+ "sample_len": 1267,
57
+ "acc,none": 0.500394632991318,
58
+ "acc_stderr,none": 0.01405248130604961
59
+ },
60
+ "blimp_adjunct_island": {
61
+ "name": "blimp_adjunct_island",
62
+ "alias": "blimp_adjunct_island",
63
+ "sample_len": 1000,
64
+ "acc,none": 0.712,
65
+ "acc_stderr,none": 0.01432694179723158
66
+ },
67
+ "blimp_anaphor_gender_agreement": {
68
+ "name": "blimp_anaphor_gender_agreement",
69
+ "alias": "blimp_anaphor_gender_agreement",
70
+ "sample_len": 1000,
71
+ "acc,none": 0.826,
72
+ "acc_stderr,none": 0.011994493230973447
73
+ },
74
+ "blimp_anaphor_number_agreement": {
75
+ "name": "blimp_anaphor_number_agreement",
76
+ "alias": "blimp_anaphor_number_agreement",
77
+ "sample_len": 1000,
78
+ "acc,none": 0.977,
79
+ "acc_stderr,none": 0.004742730594656784
80
+ },
81
+ "blimp_animate_subject_passive": {
82
+ "name": "blimp_animate_subject_passive",
83
+ "alias": "blimp_animate_subject_passive",
84
+ "sample_len": 1000,
85
+ "acc,none": 0.708,
86
+ "acc_stderr,none": 0.014385511563477433
87
+ },
88
+ "blimp_animate_subject_trans": {
89
+ "name": "blimp_animate_subject_trans",
90
+ "alias": "blimp_animate_subject_trans",
91
+ "sample_len": 1000,
92
+ "acc,none": 0.901,
93
+ "acc_stderr,none": 0.009449248027662746
94
+ },
95
+ "blimp_causative": {
96
+ "name": "blimp_causative",
97
+ "alias": "blimp_causative",
98
+ "sample_len": 1000,
99
+ "acc,none": 0.679,
100
+ "acc_stderr,none": 0.01477082181793475
101
+ },
102
+ "blimp_complex_NP_island": {
103
+ "name": "blimp_complex_NP_island",
104
+ "alias": "blimp_complex_NP_island",
105
+ "sample_len": 1000,
106
+ "acc,none": 0.528,
107
+ "acc_stderr,none": 0.015794475789511517
108
+ },
109
+ "blimp_coordinate_structure_constraint_complex_left_branch": {
110
+ "name": "blimp_coordinate_structure_constraint_complex_left_branch",
111
+ "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
112
+ "sample_len": 1000,
113
+ "acc,none": 0.314,
114
+ "acc_stderr,none": 0.014683991951088084
115
+ },
116
+ "blimp_coordinate_structure_constraint_object_extraction": {
117
+ "name": "blimp_coordinate_structure_constraint_object_extraction",
118
+ "alias": "blimp_coordinate_structure_constraint_object_extraction",
119
+ "sample_len": 1000,
120
+ "acc,none": 0.745,
121
+ "acc_stderr,none": 0.013790038620872863
122
+ },
123
+ "blimp_determiner_noun_agreement_1": {
124
+ "name": "blimp_determiner_noun_agreement_1",
125
+ "alias": "blimp_determiner_noun_agreement_1",
126
+ "sample_len": 1000,
127
+ "acc,none": 0.963,
128
+ "acc_stderr,none": 0.0059721576223896395
129
+ },
130
+ "blimp_determiner_noun_agreement_2": {
131
+ "name": "blimp_determiner_noun_agreement_2",
132
+ "alias": "blimp_determiner_noun_agreement_2",
133
+ "sample_len": 1000,
134
+ "acc,none": 0.938,
135
+ "acc_stderr,none": 0.007629823996280269
136
+ },
137
+ "blimp_determiner_noun_agreement_irregular_1": {
138
+ "name": "blimp_determiner_noun_agreement_irregular_1",
139
+ "alias": "blimp_determiner_noun_agreement_irregular_1",
140
+ "sample_len": 1000,
141
+ "acc,none": 0.864,
142
+ "acc_stderr,none": 0.01084535023047304
143
+ },
144
+ "blimp_determiner_noun_agreement_irregular_2": {
145
+ "name": "blimp_determiner_noun_agreement_irregular_2",
146
+ "alias": "blimp_determiner_noun_agreement_irregular_2",
147
+ "sample_len": 1000,
148
+ "acc,none": 0.911,
149
+ "acc_stderr,none": 0.009008893392651516
150
+ },
151
+ "blimp_determiner_noun_agreement_with_adj_2": {
152
+ "name": "blimp_determiner_noun_agreement_with_adj_2",
153
+ "alias": "blimp_determiner_noun_agreement_with_adj_2",
154
+ "sample_len": 1000,
155
+ "acc,none": 0.896,
156
+ "acc_stderr,none": 0.009658016218524242
157
+ },
158
+ "blimp_determiner_noun_agreement_with_adj_irregular_1": {
159
+ "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
160
+ "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
161
+ "sample_len": 1000,
162
+ "acc,none": 0.815,
163
+ "acc_stderr,none": 0.012285191326386656
164
+ },
165
+ "blimp_determiner_noun_agreement_with_adj_irregular_2": {
166
+ "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
167
+ "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
168
+ "sample_len": 1000,
169
+ "acc,none": 0.874,
170
+ "acc_stderr,none": 0.010499249222407964
171
+ },
172
+ "blimp_determiner_noun_agreement_with_adjective_1": {
173
+ "name": "blimp_determiner_noun_agreement_with_adjective_1",
174
+ "alias": "blimp_determiner_noun_agreement_with_adjective_1",
175
+ "sample_len": 1000,
176
+ "acc,none": 0.91,
177
+ "acc_stderr,none": 0.009054390204866477
178
+ },
179
+ "blimp_distractor_agreement_relational_noun": {
180
+ "name": "blimp_distractor_agreement_relational_noun",
181
+ "alias": "blimp_distractor_agreement_relational_noun",
182
+ "sample_len": 1000,
183
+ "acc,none": 0.776,
184
+ "acc_stderr,none": 0.013190830072364589
185
+ },
186
+ "blimp_distractor_agreement_relative_clause": {
187
+ "name": "blimp_distractor_agreement_relative_clause",
188
+ "alias": "blimp_distractor_agreement_relative_clause",
189
+ "sample_len": 1000,
190
+ "acc,none": 0.632,
191
+ "acc_stderr,none": 0.015258073561521743
192
+ },
193
+ "blimp_drop_argument": {
194
+ "name": "blimp_drop_argument",
195
+ "alias": "blimp_drop_argument",
196
+ "sample_len": 1000,
197
+ "acc,none": 0.756,
198
+ "acc_stderr,none": 0.013588548437881386
199
+ },
200
+ "blimp_ellipsis_n_bar_1": {
201
+ "name": "blimp_ellipsis_n_bar_1",
202
+ "alias": "blimp_ellipsis_n_bar_1",
203
+ "sample_len": 1000,
204
+ "acc,none": 0.741,
205
+ "acc_stderr,none": 0.013860415257527861
206
+ },
207
+ "blimp_ellipsis_n_bar_2": {
208
+ "name": "blimp_ellipsis_n_bar_2",
209
+ "alias": "blimp_ellipsis_n_bar_2",
210
+ "sample_len": 1000,
211
+ "acc,none": 0.825,
212
+ "acc_stderr,none": 0.012021627157731998
213
+ },
214
+ "blimp_existential_there_object_raising": {
215
+ "name": "blimp_existential_there_object_raising",
216
+ "alias": "blimp_existential_there_object_raising",
217
+ "sample_len": 1000,
218
+ "acc,none": 0.767,
219
+ "acc_stderr,none": 0.01337497251921999
220
+ },
221
+ "blimp_existential_there_quantifiers_1": {
222
+ "name": "blimp_existential_there_quantifiers_1",
223
+ "alias": "blimp_existential_there_quantifiers_1",
224
+ "sample_len": 1000,
225
+ "acc,none": 0.979,
226
+ "acc_stderr,none": 0.0045364721513065165
227
+ },
228
+ "blimp_existential_there_quantifiers_2": {
229
+ "name": "blimp_existential_there_quantifiers_2",
230
+ "alias": "blimp_existential_there_quantifiers_2",
231
+ "sample_len": 1000,
232
+ "acc,none": 0.412,
233
+ "acc_stderr,none": 0.015572363292015074
234
+ },
235
+ "blimp_existential_there_subject_raising": {
236
+ "name": "blimp_existential_there_subject_raising",
237
+ "alias": "blimp_existential_there_subject_raising",
238
+ "sample_len": 1000,
239
+ "acc,none": 0.848,
240
+ "acc_stderr,none": 0.01135891830347525
241
+ },
242
+ "blimp_expletive_it_object_raising": {
243
+ "name": "blimp_expletive_it_object_raising",
244
+ "alias": "blimp_expletive_it_object_raising",
245
+ "sample_len": 1000,
246
+ "acc,none": 0.746,
247
+ "acc_stderr,none": 0.013772206565168622
248
+ },
249
+ "blimp_inchoative": {
250
+ "name": "blimp_inchoative",
251
+ "alias": "blimp_inchoative",
252
+ "sample_len": 1000,
253
+ "acc,none": 0.607,
254
+ "acc_stderr,none": 0.015452824654081525
255
+ },
256
+ "blimp_intransitive": {
257
+ "name": "blimp_intransitive",
258
+ "alias": "blimp_intransitive",
259
+ "sample_len": 1000,
260
+ "acc,none": 0.723,
261
+ "acc_stderr,none": 0.014158794845306273
262
+ },
263
+ "blimp_irregular_past_participle_adjectives": {
264
+ "name": "blimp_irregular_past_participle_adjectives",
265
+ "alias": "blimp_irregular_past_participle_adjectives",
266
+ "sample_len": 1000,
267
+ "acc,none": 0.864,
268
+ "acc_stderr,none": 0.01084535023047304
269
+ },
270
+ "blimp_irregular_past_participle_verbs": {
271
+ "name": "blimp_irregular_past_participle_verbs",
272
+ "alias": "blimp_irregular_past_participle_verbs",
273
+ "sample_len": 1000,
274
+ "acc,none": 0.858,
275
+ "acc_stderr,none": 0.011043457699378305
276
+ },
277
+ "blimp_irregular_plural_subject_verb_agreement_1": {
278
+ "name": "blimp_irregular_plural_subject_verb_agreement_1",
279
+ "alias": "blimp_irregular_plural_subject_verb_agreement_1",
280
+ "sample_len": 1000,
281
+ "acc,none": 0.87,
282
+ "acc_stderr,none": 0.010640169792499236
283
+ },
284
+ "blimp_irregular_plural_subject_verb_agreement_2": {
285
+ "name": "blimp_irregular_plural_subject_verb_agreement_2",
286
+ "alias": "blimp_irregular_plural_subject_verb_agreement_2",
287
+ "sample_len": 1000,
288
+ "acc,none": 0.872,
289
+ "acc_stderr,none": 0.010570133761108595
290
+ },
291
+ "blimp_left_branch_island_echo_question": {
292
+ "name": "blimp_left_branch_island_echo_question",
293
+ "alias": "blimp_left_branch_island_echo_question",
294
+ "sample_len": 1000,
295
+ "acc,none": 0.435,
296
+ "acc_stderr,none": 0.015685057252717346
297
+ },
298
+ "blimp_left_branch_island_simple_question": {
299
+ "name": "blimp_left_branch_island_simple_question",
300
+ "alias": "blimp_left_branch_island_simple_question",
301
+ "sample_len": 1000,
302
+ "acc,none": 0.373,
303
+ "acc_stderr,none": 0.015300493622922927
304
+ },
305
+ "blimp_matrix_question_npi_licensor_present": {
306
+ "name": "blimp_matrix_question_npi_licensor_present",
307
+ "alias": "blimp_matrix_question_npi_licensor_present",
308
+ "sample_len": 1000,
309
+ "acc,none": 0.503,
310
+ "acc_stderr,none": 0.01581901517924682
311
+ },
312
+ "blimp_npi_present_1": {
313
+ "name": "blimp_npi_present_1",
314
+ "alias": "blimp_npi_present_1",
315
+ "sample_len": 1000,
316
+ "acc,none": 0.516,
317
+ "acc_stderr,none": 0.015811198373114985
318
+ },
319
+ "blimp_npi_present_2": {
320
+ "name": "blimp_npi_present_2",
321
+ "alias": "blimp_npi_present_2",
322
+ "sample_len": 1000,
323
+ "acc,none": 0.607,
324
+ "acc_stderr,none": 0.015452824654081525
325
+ },
326
+ "blimp_only_npi_licensor_present": {
327
+ "name": "blimp_only_npi_licensor_present",
328
+ "alias": "blimp_only_npi_licensor_present",
329
+ "sample_len": 1000,
330
+ "acc,none": 0.765,
331
+ "acc_stderr,none": 0.013414729030247124
332
+ },
333
+ "blimp_only_npi_scope": {
334
+ "name": "blimp_only_npi_scope",
335
+ "alias": "blimp_only_npi_scope",
336
+ "sample_len": 1000,
337
+ "acc,none": 0.42,
338
+ "acc_stderr,none": 0.015615500115072968
339
+ },
340
+ "blimp_passive_1": {
341
+ "name": "blimp_passive_1",
342
+ "alias": "blimp_passive_1",
343
+ "sample_len": 1000,
344
+ "acc,none": 0.85,
345
+ "acc_stderr,none": 0.011297239823409418
346
+ },
347
+ "blimp_passive_2": {
348
+ "name": "blimp_passive_2",
349
+ "alias": "blimp_passive_2",
350
+ "sample_len": 1000,
351
+ "acc,none": 0.87,
352
+ "acc_stderr,none": 0.010640169792499236
353
+ },
354
+ "blimp_principle_A_c_command": {
355
+ "name": "blimp_principle_A_c_command",
356
+ "alias": "blimp_principle_A_c_command",
357
+ "sample_len": 1000,
358
+ "acc,none": 0.498,
359
+ "acc_stderr,none": 0.01581917337430266
360
+ },
361
+ "blimp_principle_A_case_1": {
362
+ "name": "blimp_principle_A_case_1",
363
+ "alias": "blimp_principle_A_case_1",
364
+ "sample_len": 1000,
365
+ "acc,none": 1.0,
366
+ "acc_stderr,none": 0.0
367
+ },
368
+ "blimp_principle_A_case_2": {
369
+ "name": "blimp_principle_A_case_2",
370
+ "alias": "blimp_principle_A_case_2",
371
+ "sample_len": 1000,
372
+ "acc,none": 0.892,
373
+ "acc_stderr,none": 0.009820001651345708
374
+ },
375
+ "blimp_principle_A_domain_1": {
376
+ "name": "blimp_principle_A_domain_1",
377
+ "alias": "blimp_principle_A_domain_1",
378
+ "sample_len": 1000,
379
+ "acc,none": 0.931,
380
+ "acc_stderr,none": 0.008018934050315124
381
+ },
382
+ "blimp_principle_A_domain_2": {
383
+ "name": "blimp_principle_A_domain_2",
384
+ "alias": "blimp_principle_A_domain_2",
385
+ "sample_len": 1000,
386
+ "acc,none": 0.754,
387
+ "acc_stderr,none": 0.01362606581775054
388
+ },
389
+ "blimp_principle_A_domain_3": {
390
+ "name": "blimp_principle_A_domain_3",
391
+ "alias": "blimp_principle_A_domain_3",
392
+ "sample_len": 1000,
393
+ "acc,none": 0.656,
394
+ "acc_stderr,none": 0.015029633724408919
395
+ },
396
+ "blimp_principle_A_reconstruction": {
397
+ "name": "blimp_principle_A_reconstruction",
398
+ "alias": "blimp_principle_A_reconstruction",
399
+ "sample_len": 1000,
400
+ "acc,none": 0.49,
401
+ "acc_stderr,none": 0.015816135752773183
402
+ },
403
+ "blimp_regular_plural_subject_verb_agreement_1": {
404
+ "name": "blimp_regular_plural_subject_verb_agreement_1",
405
+ "alias": "blimp_regular_plural_subject_verb_agreement_1",
406
+ "sample_len": 1000,
407
+ "acc,none": 0.893,
408
+ "acc_stderr,none": 0.009779910359847203
409
+ },
410
+ "blimp_regular_plural_subject_verb_agreement_2": {
411
+ "name": "blimp_regular_plural_subject_verb_agreement_2",
412
+ "alias": "blimp_regular_plural_subject_verb_agreement_2",
413
+ "sample_len": 1000,
414
+ "acc,none": 0.893,
415
+ "acc_stderr,none": 0.009779910359847203
416
+ },
417
+ "blimp_sentential_negation_npi_licensor_present": {
418
+ "name": "blimp_sentential_negation_npi_licensor_present",
419
+ "alias": "blimp_sentential_negation_npi_licensor_present",
420
+ "sample_len": 1000,
421
+ "acc,none": 0.995,
422
+ "acc_stderr,none": 0.0022315868748448895
423
+ },
424
+ "blimp_sentential_negation_npi_scope": {
425
+ "name": "blimp_sentential_negation_npi_scope",
426
+ "alias": "blimp_sentential_negation_npi_scope",
427
+ "sample_len": 1000,
428
+ "acc,none": 0.554,
429
+ "acc_stderr,none": 0.01572677116675039
430
+ },
431
+ "blimp_sentential_subject_island": {
432
+ "name": "blimp_sentential_subject_island",
433
+ "alias": "blimp_sentential_subject_island",
434
+ "sample_len": 1000,
435
+ "acc,none": 0.335,
436
+ "acc_stderr,none": 0.014933117490932629
437
+ },
438
+ "blimp_superlative_quantifiers_1": {
439
+ "name": "blimp_superlative_quantifiers_1",
440
+ "alias": "blimp_superlative_quantifiers_1",
441
+ "sample_len": 1000,
442
+ "acc,none": 0.821,
443
+ "acc_stderr,none": 0.012128730605719056
444
+ },
445
+ "blimp_superlative_quantifiers_2": {
446
+ "name": "blimp_superlative_quantifiers_2",
447
+ "alias": "blimp_superlative_quantifiers_2",
448
+ "sample_len": 1000,
449
+ "acc,none": 0.799,
450
+ "acc_stderr,none": 0.012679107214617274
451
+ },
452
+ "blimp_tough_vs_raising_1": {
453
+ "name": "blimp_tough_vs_raising_1",
454
+ "alias": "blimp_tough_vs_raising_1",
455
+ "sample_len": 1000,
456
+ "acc,none": 0.458,
457
+ "acc_stderr,none": 0.015763390640483554
458
+ },
459
+ "blimp_tough_vs_raising_2": {
460
+ "name": "blimp_tough_vs_raising_2",
461
+ "alias": "blimp_tough_vs_raising_2",
462
+ "sample_len": 1000,
463
+ "acc,none": 0.922,
464
+ "acc_stderr,none": 0.00848457353011853
465
+ },
466
+ "blimp_transitive": {
467
+ "name": "blimp_transitive",
468
+ "alias": "blimp_transitive",
469
+ "sample_len": 1000,
470
+ "acc,none": 0.808,
471
+ "acc_stderr,none": 0.012461592646660028
472
+ },
473
+ "blimp_wh_island": {
474
+ "name": "blimp_wh_island",
475
+ "alias": "blimp_wh_island",
476
+ "sample_len": 1000,
477
+ "acc,none": 0.794,
478
+ "acc_stderr,none": 0.012795613612786522
479
+ },
480
+ "blimp_wh_questions_object_gap": {
481
+ "name": "blimp_wh_questions_object_gap",
482
+ "alias": "blimp_wh_questions_object_gap",
483
+ "sample_len": 1000,
484
+ "acc,none": 0.642,
485
+ "acc_stderr,none": 0.015167928865407633
486
+ },
487
+ "blimp_wh_questions_subject_gap": {
488
+ "name": "blimp_wh_questions_subject_gap",
489
+ "alias": "blimp_wh_questions_subject_gap",
490
+ "sample_len": 1000,
491
+ "acc,none": 0.874,
492
+ "acc_stderr,none": 0.010499249222407964
493
+ },
494
+ "blimp_wh_questions_subject_gap_long_distance": {
495
+ "name": "blimp_wh_questions_subject_gap_long_distance",
496
+ "alias": "blimp_wh_questions_subject_gap_long_distance",
497
+ "sample_len": 1000,
498
+ "acc,none": 0.862,
499
+ "acc_stderr,none": 0.010912152632504508
500
+ },
501
+ "blimp_wh_vs_that_no_gap": {
502
+ "name": "blimp_wh_vs_that_no_gap",
503
+ "alias": "blimp_wh_vs_that_no_gap",
504
+ "sample_len": 1000,
505
+ "acc,none": 0.951,
506
+ "acc_stderr,none": 0.006829761756140959
507
+ },
508
+ "blimp_wh_vs_that_no_gap_long_distance": {
509
+ "name": "blimp_wh_vs_that_no_gap_long_distance",
510
+ "alias": "blimp_wh_vs_that_no_gap_long_distance",
511
+ "sample_len": 1000,
512
+ "acc,none": 0.987,
513
+ "acc_stderr,none": 0.003583830889403662
514
+ },
515
+ "blimp_wh_vs_that_with_gap": {
516
+ "name": "blimp_wh_vs_that_with_gap",
517
+ "alias": "blimp_wh_vs_that_with_gap",
518
+ "sample_len": 1000,
519
+ "acc,none": 0.452,
520
+ "acc_stderr,none": 0.01574623586588058
521
+ },
522
+ "blimp_wh_vs_that_with_gap_long_distance": {
523
+ "name": "blimp_wh_vs_that_with_gap_long_distance",
524
+ "alias": "blimp_wh_vs_that_with_gap_long_distance",
525
+ "sample_len": 1000,
526
+ "acc,none": 0.15,
527
+ "acc_stderr,none": 0.011297239823409416
528
+ },
529
+ "blimp": {
530
+ "alias": "blimp",
531
+ "name": "blimp",
532
+ "sample_len": 67000,
533
+ "acc,none": 0.7401044776119403,
534
+ "acc_stderr,none": 0.0015168010428542197,
535
+ "sample_count": {
536
+ "acc,none": 67000
537
+ }
538
+ }
539
+ }
540
+ }
m18/config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m18",
3
+ "run_name": "m18-gqa-nope-dense-adamw",
4
+ "data_dir": "data/fineweb-edu",
5
+ "vocab_size": 50257,
6
+ "block_size": 1024,
7
+ "d_model": 512,
8
+ "num_layers": 12,
9
+ "num_heads": 8,
10
+ "ffn_mult": 4,
11
+ "dropout": 0.0,
12
+ "bias": false,
13
+ "norm_type": "rmsnorm",
14
+ "tie_embeddings": true,
15
+ "attention_type": "gqa",
16
+ "gqa_num_kv_heads": 2,
17
+ "mla_latent_dim": 64,
18
+ "pos_type": "nope",
19
+ "rope_base": 10000,
20
+ "ffn_type": "dense",
21
+ "moe_num_experts": 8,
22
+ "moe_top_k": 2,
23
+ "optimizer": "adamw",
24
+ "lr": 0.0006,
25
+ "min_lr": 6e-05,
26
+ "weight_decay": 0.1,
27
+ "beta1": 0.9,
28
+ "beta2": 0.95,
29
+ "grad_clip": 1.0,
30
+ "muon_lr": 0.02,
31
+ "muon_momentum": 0.95,
32
+ "training_tokens": 1000000000,
33
+ "warmup_tokens": 6000000,
34
+ "lr_schedule": "cosine",
35
+ "batch_size": 4,
36
+ "grad_accum_steps": 16,
37
+ "device": "cuda",
38
+ "dtype": "bfloat16",
39
+ "compile": true,
40
+ "seed": 1337,
41
+ "eval_interval": 250,
42
+ "eval_iters": 100,
43
+ "log_interval": 10,
44
+ "checkpoint_interval": 1000,
45
+ "results_dir": "results",
46
+ "mlflow_experiment": "slm-ablation",
47
+ "mlflow_tracking_uri": "sqlite:///mlflow.db",
48
+ "mlflow_artifact_location": "./mlartifacts"
49
+ }
m18/eval_results.json ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "m18",
3
+ "checkpoint": "/home/akshat/codemindv2/results/m18/final.pt",
4
+ "trained_step": 15258,
5
+ "total_params": 58774528,
6
+ "active_params": 58774528,
7
+ "flops_per_forward": 117549056.0,
8
+ "val_loss": 3.5828016698360443,
9
+ "val_perplexity": 35.97418758800085,
10
+ "bits_per_token": 5.168890201561158,
11
+ "ppl_tokens_evaluated": 655360,
12
+ "prefill_tok_per_sec": 192856.71674094364,
13
+ "inference_tok_per_sec": 286.03230651282206,
14
+ "inference_tok_per_sec_kvcache": 283.4770307274281,
15
+ "kvcache_speedup": 0.9910664784109643,
16
+ "gen_tokens": 128,
17
+ "peak_infer_vram_gb": 0.363,
18
+ "peak_train_vram_gb": 3.987,
19
+ "closeness_answer_ppl": 36.553,
20
+ "closeness_hit_rate": 0.4,
21
+ "closeness_token_f1": 0.033,
22
+ "closeness_n": 20,
23
+ "closeness_items": [
24
+ {
25
+ "prompt": "The capital of France is",
26
+ "answer": " Paris",
27
+ "generated": " the French capital of France.",
28
+ "answer_nll": 3.9908,
29
+ "answer_ppl": 54.098,
30
+ "hit": false,
31
+ "token_f1": 0.0
32
+ },
33
+ {
34
+ "prompt": "The capital of Japan is",
35
+ "answer": " Tokyo",
36
+ "generated": " Japan.\nThe Japanese government",
37
+ "answer_nll": 3.3975,
38
+ "answer_ppl": 29.888,
39
+ "hit": false,
40
+ "token_f1": 0.0
41
+ },
42
+ {
43
+ "prompt": "The largest planet in the solar system is",
44
+ "answer": " Jupiter",
45
+ "generated": " the sun. The sun is",
46
+ "answer_nll": 4.4833,
47
+ "answer_ppl": 88.53,
48
+ "hit": false,
49
+ "token_f1": 0.0
50
+ },
51
+ {
52
+ "prompt": "Water is made of hydrogen and",
53
+ "answer": " oxygen",
54
+ "generated": " oxygen.\nThe hydrogen and",
55
+ "answer_nll": 1.0029,
56
+ "answer_ppl": 2.726,
57
+ "hit": true,
58
+ "token_f1": 0.0
59
+ },
60
+ {
61
+ "prompt": "The opposite of hot is",
62
+ "answer": " cold",
63
+ "generated": " the case with the hot water",
64
+ "answer_nll": 5.5526,
65
+ "answer_ppl": 257.904,
66
+ "hit": false,
67
+ "token_f1": 0.0
68
+ },
69
+ {
70
+ "prompt": "The opposite of up is",
71
+ "answer": " down",
72
+ "generated": " the fact that the two are",
73
+ "answer_nll": 6.6943,
74
+ "answer_ppl": 807.812,
75
+ "hit": false,
76
+ "token_f1": 0.0
77
+ },
78
+ {
79
+ "prompt": "Two plus two equals",
80
+ "answer": " four",
81
+ "generated": ", and the sum of the",
82
+ "answer_nll": 5.2257,
83
+ "answer_ppl": 185.998,
84
+ "hit": false,
85
+ "token_f1": 0.0
86
+ },
87
+ {
88
+ "prompt": "The sun rises in the",
89
+ "answer": " east",
90
+ "generated": " sky, and the sun rises",
91
+ "answer_nll": 4.1287,
92
+ "answer_ppl": 62.099,
93
+ "hit": false,
94
+ "token_f1": 0.0
95
+ },
96
+ {
97
+ "prompt": "The first president of the United States was George",
98
+ "answer": " Washington",
99
+ "generated": " Washington, who was a member",
100
+ "answer_nll": 0.4225,
101
+ "answer_ppl": 1.526,
102
+ "hit": true,
103
+ "token_f1": 0.0
104
+ },
105
+ {
106
+ "prompt": "The speed of light is approximately 300,000 kilometers per",
107
+ "answer": " second",
108
+ "generated": " second, and the speed of",
109
+ "answer_nll": 0.4998,
110
+ "answer_ppl": 1.648,
111
+ "hit": true,
112
+ "token_f1": 0.0
113
+ },
114
+ {
115
+ "prompt": "A triangle has three",
116
+ "answer": " sides",
117
+ "generated": " sides, one of which is",
118
+ "answer_nll": 1.2318,
119
+ "answer_ppl": 3.427,
120
+ "hit": true,
121
+ "token_f1": 0.0
122
+ },
123
+ {
124
+ "prompt": "The chemical symbol for gold is",
125
+ "answer": " Au",
126
+ "generated": " the gold symbol.\nThe",
127
+ "answer_nll": 8.984,
128
+ "answer_ppl": 7974.202,
129
+ "hit": false,
130
+ "token_f1": 0.0
131
+ },
132
+ {
133
+ "prompt": "The freezing point of water in Celsius is",
134
+ "answer": " zero",
135
+ "generated": " the melting point of water in",
136
+ "answer_nll": 5.2682,
137
+ "answer_ppl": 194.07,
138
+ "hit": false,
139
+ "token_f1": 0.0
140
+ },
141
+ {
142
+ "prompt": "The author of Romeo and Juliet is William",
143
+ "answer": " Shakespeare",
144
+ "generated": " Shakespeare.\nThe author of",
145
+ "answer_nll": 0.8406,
146
+ "answer_ppl": 2.318,
147
+ "hit": true,
148
+ "token_f1": 0.0
149
+ },
150
+ {
151
+ "prompt": "The Earth revolves around the",
152
+ "answer": " Sun",
153
+ "generated": " Sun, and the Sun revolves",
154
+ "answer_nll": 1.3375,
155
+ "answer_ppl": 3.809,
156
+ "hit": true,
157
+ "token_f1": 0.333
158
+ },
159
+ {
160
+ "prompt": "The plural of mouse is",
161
+ "answer": " mice",
162
+ "generated": " the word for the word \"",
163
+ "answer_nll": 7.2894,
164
+ "answer_ppl": 1464.741,
165
+ "hit": false,
166
+ "token_f1": 0.0
167
+ },
168
+ {
169
+ "prompt": "Photosynthesis occurs in the",
170
+ "answer": " leaves",
171
+ "generated": " plant and is a major source",
172
+ "answer_nll": 3.6544,
173
+ "answer_ppl": 38.645,
174
+ "hit": false,
175
+ "token_f1": 0.0
176
+ },
177
+ {
178
+ "prompt": "The human heart pumps",
179
+ "answer": " blood",
180
+ "generated": " blood to the heart, and",
181
+ "answer_nll": 1.629,
182
+ "answer_ppl": 5.099,
183
+ "hit": true,
184
+ "token_f1": 0.333
185
+ },
186
+ {
187
+ "prompt": "The currency used in the United States is the",
188
+ "answer": " dollar",
189
+ "generated": " United States dollar.\nThe",
190
+ "answer_nll": 3.4756,
191
+ "answer_ppl": 32.318,
192
+ "hit": true,
193
+ "token_f1": 0.0
194
+ },
195
+ {
196
+ "prompt": "The tallest animal in the world is the",
197
+ "answer": " giraffe",
198
+ "generated": " giant, the largest, and",
199
+ "answer_nll": 2.8665,
200
+ "answer_ppl": 17.575,
201
+ "hit": false,
202
+ "token_f1": 0.0
203
+ }
204
+ ],
205
+ "generation_samples": {
206
+ "The most important thing about sleep is": "The most important thing about sleep is to create a schedule that doesn\u2019t require you to eat,\u201d says Dr. Martin. \u201cIt\u2019s important to focus on eating before, and not on just those things that you can\u2019t burn.\u201d\nDr. Andrew Walker, director of the Sleep",
207
+ "Scientists have discovered that": "Scientists have discovered that, in short, the amount of CO2 released by the sun causes serious problems in the environment.\n\u201cWe do not know how damaging the global warming is; the warming is caused by the sun\u2019s rays, rather than by the sun\u2019s rays,\u201d said John",
208
+ "In the history of mathematics,": "In the history of mathematics, the term \"curved' is a synonym for the common expression \"curved\" which can be found in many different ways. For example, the common expression \"curved\" (equation) can be found in two different places: a curving place, a table or a table?",
209
+ "Python code for prime number": "Python code for prime number of 5 decimal numbers. However, the string is not a string itself, and as such it is not a string. It does not have a string of 6 decimal numbers with a string of 6 decimal numbers. The string is not an integer, and the length\nnumber of 5 decimal numbers is,",
210
+ "The best way to learn a new language is": "The best way to learn a new language is to learn about the language you\u2019re looking for. If you are learning something that you have noticed, such as the following sentence, \u201cI have heard someone say, \u2018I heard this,\u2019 and I heard that.\u201d\nIf you were to learn an old language"
211
+ }
212
+ }
m18/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26733fb1ea7c79e3229d0df16d3d7049c9e7fbfca69d2a5e83bd0d55a350a5a1
3
+ size 235107624