ZachW commited on
Commit
96989ab
·
verified ·
1 Parent(s): 747fb08

Upload extrapolation artifact: 050302 RL OP15-20 step200 summary JSON

Browse files
050302/evals/rl050302_op15_20_step200_merged_hf/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "rl050302_op15_20_step200_merged_hf",
4
+ "id_acc": 0.7198645833333334,
5
+ "ood_acc": 0.29179921875,
6
+ "total_acc": 0.4945670230263158,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 203.807328125,
11
+ "ood_avg_resp_len": 332.50551015625,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 400116,
19
+ "answer_accuracy": 0.49413630256085994,
20
+ "avg_response_len": 260.3990833959058,
21
+ "resp_tokens_sum": 210852429,
22
+ "pass_at_k": {
23
+ "pass@1": 0.49413630256085994,
24
+ "pass@2": 0.5783138453725597,
25
+ "pass@4": 0.6522844111813358,
26
+ "pass@8": 0.7126406720979045,
27
+ "pass@16": 0.7607958557092384,
28
+ "pass@32": 0.8002622415970753,
29
+ "pass@64": 0.8330792580630666,
30
+ "pass@128": 0.8612077141953841
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.5904775073746312,
35
+ "pass@2": 0.6759274179499686,
36
+ "pass@4": 0.739327368647503,
37
+ "pass@8": 0.7888088892182306,
38
+ "pass@16": 0.8301737990227551,
39
+ "pass@32": 0.8653796021525334,
40
+ "pass@64": 0.8950191494801747,
41
+ "pass@128": 0.9203539823008849
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9306853582554517,
45
+ "pass@2": 0.951853291264994,
46
+ "pass@4": 0.9672763547664015,
47
+ "pass@8": 0.9777387432602255,
48
+ "pass@16": 0.9849406600666408,
49
+ "pass@32": 0.9906709430385632,
50
+ "pass@64": 0.9946393301197776,
51
+ "pass@128": 0.9968847352024922
52
+ },
53
+ "3": {
54
+ "pass@1": 0.8729692278287462,
55
+ "pass@2": 0.8916826193744132,
56
+ "pass@4": 0.9012887589166031,
57
+ "pass@8": 0.9069931487274436,
58
+ "pass@16": 0.9113667010227782,
59
+ "pass@32": 0.9158878607999068,
60
+ "pass@64": 0.921283309008342,
61
+ "pass@128": 0.926605504587156
62
+ },
63
+ "4": {
64
+ "pass@1": 0.79365234375,
65
+ "pass@2": 0.8259854053887796,
66
+ "pass@4": 0.8509953552680912,
67
+ "pass@8": 0.8705874877072775,
68
+ "pass@16": 0.8861866677459165,
69
+ "pass@32": 0.9001222965392037,
70
+ "pass@64": 0.9150975754710673,
71
+ "pass@128": 0.934375
72
+ },
73
+ "5": {
74
+ "pass@1": 0.711954365079365,
75
+ "pass@2": 0.7870863798275217,
76
+ "pass@4": 0.8438272906362893,
77
+ "pass@8": 0.8820148031784064,
78
+ "pass@16": 0.9061775402294971,
79
+ "pass@32": 0.9233827284096477,
80
+ "pass@64": 0.939203360121733,
81
+ "pass@128": 0.9587301587301588
82
+ },
83
+ "6": {
84
+ "pass@1": 0.6886695359281437,
85
+ "pass@2": 0.7508096503842708,
86
+ "pass@4": 0.7981536092606685,
87
+ "pass@8": 0.8300540071307995,
88
+ "pass@16": 0.8515233346236275,
89
+ "pass@32": 0.867618886093707,
90
+ "pass@64": 0.8782963968718454,
91
+ "pass@128": 0.8832335329341318
92
+ },
93
+ "7": {
94
+ "pass@1": 0.6789478550295858,
95
+ "pass@2": 0.7444577587010199,
96
+ "pass@4": 0.7924073042090151,
97
+ "pass@8": 0.8294529774774302,
98
+ "pass@16": 0.85758601479046,
99
+ "pass@32": 0.8766141160571631,
100
+ "pass@64": 0.8892577006853214,
101
+ "pass@128": 0.8994082840236687
102
+ },
103
+ "8": {
104
+ "pass@1": 0.6259515224358975,
105
+ "pass@2": 0.6997999949525543,
106
+ "pass@4": 0.7484703330352931,
107
+ "pass@8": 0.7817076118380484,
108
+ "pass@16": 0.8062330519425672,
109
+ "pass@32": 0.8273824576064478,
110
+ "pass@64": 0.8497525386355358,
111
+ "pass@128": 0.875
112
+ },
113
+ "9": {
114
+ "pass@1": 0.6052596830985916,
115
+ "pass@2": 0.6952658866585334,
116
+ "pass@4": 0.7621365283564899,
117
+ "pass@8": 0.8126598351468055,
118
+ "pass@16": 0.8499978703599235,
119
+ "pass@32": 0.8788524080867729,
120
+ "pass@64": 0.8985118669285895,
121
+ "pass@128": 0.9098591549295775
122
+ },
123
+ "11": {
124
+ "pass@1": 0.5223309135977338,
125
+ "pass@2": 0.6289980008253216,
126
+ "pass@4": 0.7091189907918726,
127
+ "pass@8": 0.7677548796207677,
128
+ "pass@16": 0.810923847471511,
129
+ "pass@32": 0.8425591447235117,
130
+ "pass@64": 0.8667520481945041,
131
+ "pass@128": 0.8895184135977338
132
+ },
133
+ "12": {
134
+ "pass@1": 0.5407618290960452,
135
+ "pass@2": 0.6592635765603452,
136
+ "pass@4": 0.7460638715181781,
137
+ "pass@8": 0.8079141082622625,
138
+ "pass@16": 0.852190879714896,
139
+ "pass@32": 0.8835107730092441,
140
+ "pass@64": 0.9053088531949328,
141
+ "pass@128": 0.9209039548022598
142
+ },
143
+ "13": {
144
+ "pass@1": 0.41783405172413796,
145
+ "pass@2": 0.5587110927356649,
146
+ "pass@4": 0.6785095712957503,
147
+ "pass@8": 0.7668253027465036,
148
+ "pass@16": 0.8288992955389046,
149
+ "pass@32": 0.8726845941313368,
150
+ "pass@64": 0.9048733323087187,
151
+ "pass@128": 0.9310344827586207
152
+ },
153
+ "14": {
154
+ "pass@1": 0.3425330996884735,
155
+ "pass@2": 0.47973429796158645,
156
+ "pass@4": 0.6109350025756123,
157
+ "pass@8": 0.7166362098428933,
158
+ "pass@16": 0.7922397712359336,
159
+ "pass@32": 0.8449829905372319,
160
+ "pass@64": 0.8827357163562379,
161
+ "pass@128": 0.9127725856697819
162
+ },
163
+ "15": {
164
+ "pass@1": 0.24970646676300579,
165
+ "pass@2": 0.36728884097219044,
166
+ "pass@4": 0.4872996991705517,
167
+ "pass@8": 0.590706698102598,
168
+ "pass@16": 0.6738243586965353,
169
+ "pass@32": 0.7419453886625642,
170
+ "pass@64": 0.796950252939072,
171
+ "pass@128": 0.838150289017341
172
+ },
173
+ "16": {
174
+ "pass@1": 0.22497479838709677,
175
+ "pass@2": 0.33648915735331486,
176
+ "pass@4": 0.4564946569178856,
177
+ "pass@8": 0.5695374743650482,
178
+ "pass@16": 0.6659607330734459,
179
+ "pass@32": 0.7420235555671716,
180
+ "pass@64": 0.8008246573666877,
181
+ "pass@128": 0.8516129032258064
182
+ },
183
+ "17": {
184
+ "pass@1": 0.1771134393063584,
185
+ "pass@2": 0.27040438248145277,
186
+ "pass@4": 0.3734399091551416,
187
+ "pass@8": 0.4705561750269978,
188
+ "pass@16": 0.556979762253697,
189
+ "pass@32": 0.6330896125700303,
190
+ "pass@64": 0.6962373325698672,
191
+ "pass@128": 0.7456647398843931
192
+ },
193
+ "18": {
194
+ "pass@1": 0.16613839285714285,
195
+ "pass@2": 0.25231404668166474,
196
+ "pass@4": 0.3428713514382129,
197
+ "pass@8": 0.4269419637345666,
198
+ "pass@16": 0.5030587296667712,
199
+ "pass@32": 0.5724071144846937,
200
+ "pass@64": 0.6359387733746433,
201
+ "pass@128": 0.6971428571428572
202
+ },
203
+ "19": {
204
+ "pass@1": 0.15615615615615616,
205
+ "pass@2": 0.2335178436310326,
206
+ "pass@4": 0.3141585730837702,
207
+ "pass@8": 0.3884583356444021,
208
+ "pass@16": 0.45846532735472434,
209
+ "pass@32": 0.5290161554991337,
210
+ "pass@64": 0.5971115408381422,
211
+ "pass@128": 0.6606606606606606
212
+ },
213
+ "20": {
214
+ "pass@1": 0.14114114114114115,
215
+ "pass@2": 0.22127085254545897,
216
+ "pass@4": 0.3091605038671966,
217
+ "pass@8": 0.38948503249060373,
218
+ "pass@16": 0.4591147943528322,
219
+ "pass@32": 0.5236253667328297,
220
+ "pass@64": 0.5845144560920016,
221
+ "pass@128": 0.6336336336336337
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 410116,
228
+ "answer_accuracy": 0.49945927513639904,
229
+ "avg_response_len": 268.1280945537802,
230
+ "resp_tokens_sum": 220165341,
231
+ "pass_at_k": {
232
+ "pass@1": 0.49945927513639904,
233
+ "pass@2": 0.5797713589581576,
234
+ "pass@4": 0.6508590268139766,
235
+ "pass@8": 0.7092176995513156,
236
+ "pass@16": 0.7549870987709693,
237
+ "pass@32": 0.7915088244120045,
238
+ "pass@64": 0.8217816179895945,
239
+ "pass@128": 0.8470771628994544
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.6425421779141104,
244
+ "pass@2": 0.7180774781411522,
245
+ "pass@4": 0.7713729633489064,
246
+ "pass@8": 0.8115314019701301,
247
+ "pass@16": 0.8422782157793477,
248
+ "pass@32": 0.8665394346696862,
249
+ "pass@64": 0.8875838773290864,
250
+ "pass@128": 0.9079754601226994
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9346267097264438,
254
+ "pass@2": 0.9484187648086544,
255
+ "pass@4": 0.9577753805660308,
256
+ "pass@8": 0.9642355108332087,
257
+ "pass@16": 0.9687406308251917,
258
+ "pass@32": 0.9722132661616432,
259
+ "pass@64": 0.976268636177634,
260
+ "pass@128": 0.9817629179331308
261
+ },
262
+ "3": {
263
+ "pass@1": 0.8785274621212121,
264
+ "pass@2": 0.8940225334049151,
265
+ "pass@4": 0.9043032615241278,
266
+ "pass@8": 0.9116606528207251,
267
+ "pass@16": 0.9163981804252643,
268
+ "pass@32": 0.919534935837756,
269
+ "pass@64": 0.9226355633253187,
270
+ "pass@128": 0.9272727272727272
271
+ },
272
+ "4": {
273
+ "pass@1": 0.791350776627219,
274
+ "pass@2": 0.8298940327540418,
275
+ "pass@4": 0.8566257274423535,
276
+ "pass@8": 0.8740505626426824,
277
+ "pass@16": 0.8863419432111916,
278
+ "pass@32": 0.8973476695044501,
279
+ "pass@64": 0.9076491744197636,
280
+ "pass@128": 0.9171597633136095
281
+ },
282
+ "5": {
283
+ "pass@1": 0.7036231884057971,
284
+ "pass@2": 0.760960144927536,
285
+ "pass@4": 0.8089746806105759,
286
+ "pass@8": 0.848232289495532,
287
+ "pass@16": 0.8788940387283235,
288
+ "pass@32": 0.903333779760882,
289
+ "pass@64": 0.9245269760364313,
290
+ "pass@128": 0.9449275362318841
291
+ },
292
+ "6": {
293
+ "pass@1": 0.6609733166189111,
294
+ "pass@2": 0.7211168580646615,
295
+ "pass@4": 0.7662969808752413,
296
+ "pass@8": 0.7963462083773434,
297
+ "pass@16": 0.8159017311926985,
298
+ "pass@32": 0.8326346080497815,
299
+ "pass@64": 0.8482729333475885,
300
+ "pass@128": 0.8595988538681948
301
+ },
302
+ "7": {
303
+ "pass@1": 0.669549313583815,
304
+ "pass@2": 0.7348724597196304,
305
+ "pass@4": 0.7796674157601687,
306
+ "pass@8": 0.8099202435632497,
307
+ "pass@16": 0.8312183082099309,
308
+ "pass@32": 0.8471171620508142,
309
+ "pass@64": 0.8607780565256175,
310
+ "pass@128": 0.8728323699421965
311
+ },
312
+ "8": {
313
+ "pass@1": 0.6231073943661972,
314
+ "pass@2": 0.7033488826660749,
315
+ "pass@4": 0.7635277449473741,
316
+ "pass@8": 0.8050630470213901,
317
+ "pass@16": 0.8335760394037561,
318
+ "pass@32": 0.8552420543936154,
319
+ "pass@64": 0.8752369946603544,
320
+ "pass@128": 0.8985915492957747
321
+ },
322
+ "9": {
323
+ "pass@1": 0.6189651946107785,
324
+ "pass@2": 0.7066431850865202,
325
+ "pass@4": 0.7706885601687509,
326
+ "pass@8": 0.8163749170683283,
327
+ "pass@16": 0.8472702317498962,
328
+ "pass@32": 0.8686874199142173,
329
+ "pass@64": 0.883720682738893,
330
+ "pass@128": 0.8922155688622755
331
+ },
332
+ "11": {
333
+ "pass@1": 0.5852315512048193,
334
+ "pass@2": 0.6793565749217343,
335
+ "pass@4": 0.7481086757039406,
336
+ "pass@8": 0.79819997407497,
337
+ "pass@16": 0.8376718939360224,
338
+ "pass@32": 0.8684909661065356,
339
+ "pass@64": 0.8906779124327036,
340
+ "pass@128": 0.9036144578313253
341
+ },
342
+ "12": {
343
+ "pass@1": 0.48897923197492166,
344
+ "pass@2": 0.6015723348061118,
345
+ "pass@4": 0.6915262334958906,
346
+ "pass@8": 0.7628699479020743,
347
+ "pass@16": 0.8170783827895058,
348
+ "pass@32": 0.8559537327526784,
349
+ "pass@64": 0.8835455838809168,
350
+ "pass@128": 0.9028213166144201
351
+ },
352
+ "13": {
353
+ "pass@1": 0.3866421568627451,
354
+ "pass@2": 0.5096691727872251,
355
+ "pass@4": 0.6137185039370071,
356
+ "pass@8": 0.698094258047033,
357
+ "pass@16": 0.7655805416707796,
358
+ "pass@32": 0.8172702345082308,
359
+ "pass@64": 0.8567427013588318,
360
+ "pass@128": 0.8879551820728291
361
+ },
362
+ "14": {
363
+ "pass@1": 0.34528051900584794,
364
+ "pass@2": 0.4769977868720359,
365
+ "pass@4": 0.5996480609879901,
366
+ "pass@8": 0.6973310194018792,
367
+ "pass@16": 0.7674308590346286,
368
+ "pass@32": 0.8171571009241003,
369
+ "pass@64": 0.85288334433611,
370
+ "pass@128": 0.8771929824561403
371
+ },
372
+ "15": {
373
+ "pass@1": 0.28020833333333334,
374
+ "pass@2": 0.405685173586256,
375
+ "pass@4": 0.5307156491802157,
376
+ "pass@8": 0.6359535624159368,
377
+ "pass@16": 0.7152338549121703,
378
+ "pass@32": 0.775609713345843,
379
+ "pass@64": 0.822731933165433,
380
+ "pass@128": 0.8575757575757575
381
+ },
382
+ "16": {
383
+ "pass@1": 0.22115384615384615,
384
+ "pass@2": 0.3294303256163492,
385
+ "pass@4": 0.4457514172907874,
386
+ "pass@8": 0.5532684120879664,
387
+ "pass@16": 0.6416775711504988,
388
+ "pass@32": 0.7124958371733157,
389
+ "pass@64": 0.771953988942229,
390
+ "pass@128": 0.8205128205128205
391
+ },
392
+ "17": {
393
+ "pass@1": 0.2061820652173913,
394
+ "pass@2": 0.3083883791020687,
395
+ "pass@4": 0.41528318888322185,
396
+ "pass@8": 0.5125658863922051,
397
+ "pass@16": 0.5953014311232198,
398
+ "pass@32": 0.6644794210402157,
399
+ "pass@64": 0.7232154857023478,
400
+ "pass@128": 0.7732919254658385
401
+ },
402
+ "18": {
403
+ "pass@1": 0.15284926470588237,
404
+ "pass@2": 0.23485265169059735,
405
+ "pass@4": 0.32879796367365843,
406
+ "pass@8": 0.42114827587863835,
407
+ "pass@16": 0.5031462412182746,
408
+ "pass@32": 0.5751373282485435,
409
+ "pass@64": 0.6395195992057614,
410
+ "pass@128": 0.6941176470588235
411
+ },
412
+ "19": {
413
+ "pass@1": 0.15843563988095238,
414
+ "pass@2": 0.23614211309523814,
415
+ "pass@4": 0.31818266717776345,
416
+ "pass@8": 0.39120863355325985,
417
+ "pass@16": 0.451844936121816,
418
+ "pass@32": 0.5046091647650751,
419
+ "pass@64": 0.5519295462717415,
420
+ "pass@128": 0.5982142857142857
421
+ },
422
+ "20": {
423
+ "pass@1": 0.14539670658682635,
424
+ "pass@2": 0.21817388844358518,
425
+ "pass@4": 0.2958124524292249,
426
+ "pass@8": 0.36673161731226755,
427
+ "pass@16": 0.42847335860140157,
428
+ "pass@32": 0.4838859785436273,
429
+ "pass@64": 0.532640616772042,
430
+ "pass@128": 0.5748502994011976
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 392555,
437
+ "answer_accuracy": 0.4899881670394632,
438
+ "avg_response_len": 286.30687435093466,
439
+ "resp_tokens_sum": 229375325,
440
+ "pass_at_k": {
441
+ "pass@1": 0.4899881670394632,
442
+ "pass@2": 0.5702338435802126,
443
+ "pass@4": 0.6417787718868243,
444
+ "pass@8": 0.7014595538946482,
445
+ "pass@16": 0.74919940638925,
446
+ "pass@32": 0.7875393111935014,
447
+ "pass@64": 0.8187212701325537,
448
+ "pass@128": 0.8448633967087394
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.6091651119402985,
453
+ "pass@2": 0.6854712657186507,
454
+ "pass@4": 0.7441988026683222,
455
+ "pass@8": 0.7897822970589528,
456
+ "pass@16": 0.8274429729171232,
457
+ "pass@32": 0.8595856001759086,
458
+ "pass@64": 0.8842101870456582,
459
+ "pass@128": 0.9014925373134328
460
+ },
461
+ "2": {
462
+ "pass@1": 0.91796875,
463
+ "pass@2": 0.9367565382452198,
464
+ "pass@4": 0.9504637439070117,
465
+ "pass@8": 0.9611798930422266,
466
+ "pass@16": 0.9683818048349383,
467
+ "pass@32": 0.9728251775938187,
468
+ "pass@64": 0.9764262211115565,
469
+ "pass@128": 0.98
470
+ },
471
+ "3": {
472
+ "pass@1": 0.874271137026239,
473
+ "pass@2": 0.8941968590023187,
474
+ "pass@4": 0.9057715524262092,
475
+ "pass@8": 0.915092597334439,
476
+ "pass@16": 0.9240054302470929,
477
+ "pass@32": 0.9324364578760008,
478
+ "pass@64": 0.9403403611202984,
479
+ "pass@128": 0.9475218658892128
480
+ },
481
+ "4": {
482
+ "pass@1": 0.784265350877193,
483
+ "pass@2": 0.8285923038403095,
484
+ "pass@4": 0.8585307519125894,
485
+ "pass@8": 0.8774520393462032,
486
+ "pass@16": 0.8905927333865701,
487
+ "pass@32": 0.9017483686467666,
488
+ "pass@64": 0.9111029706931209,
489
+ "pass@128": 0.9181286549707602
490
+ },
491
+ "5": {
492
+ "pass@1": 0.68125,
493
+ "pass@2": 0.7476084848309403,
494
+ "pass@4": 0.8026900538168023,
495
+ "pass@8": 0.8443404260011751,
496
+ "pass@16": 0.8745786447776861,
497
+ "pass@32": 0.8952482473128105,
498
+ "pass@64": 0.9085697858530525,
499
+ "pass@128": 0.9176470588235294
500
+ },
501
+ "6": {
502
+ "pass@1": 0.6679071372239748,
503
+ "pass@2": 0.7301391459052637,
504
+ "pass@4": 0.7808355910361361,
505
+ "pass@8": 0.818929674856254,
506
+ "pass@16": 0.8450057327852958,
507
+ "pass@32": 0.8632534413631587,
508
+ "pass@64": 0.8793223643701782,
509
+ "pass@128": 0.8958990536277602
510
+ },
511
+ "7": {
512
+ "pass@1": 0.6406497231012658,
513
+ "pass@2": 0.71024486382438,
514
+ "pass@4": 0.7654189415998627,
515
+ "pass@8": 0.8044126888367252,
516
+ "pass@16": 0.8300388065663724,
517
+ "pass@32": 0.8493341982733007,
518
+ "pass@64": 0.8669889661389816,
519
+ "pass@128": 0.8829113924050633
520
+ },
521
+ "8": {
522
+ "pass@1": 0.6278856981981982,
523
+ "pass@2": 0.7077998273864413,
524
+ "pass@4": 0.7638303280783592,
525
+ "pass@8": 0.8036424586250549,
526
+ "pass@16": 0.8327328899737727,
527
+ "pass@32": 0.8545952441143289,
528
+ "pass@64": 0.8720383615804834,
529
+ "pass@128": 0.8888888888888888
530
+ },
531
+ "9": {
532
+ "pass@1": 0.6134445337620579,
533
+ "pass@2": 0.7031613951692531,
534
+ "pass@4": 0.7679856327243657,
535
+ "pass@8": 0.8137176128360283,
536
+ "pass@16": 0.8453081708297483,
537
+ "pass@32": 0.8678143930156336,
538
+ "pass@64": 0.8861791073478019,
539
+ "pass@128": 0.9035369774919614
540
+ },
541
+ "11": {
542
+ "pass@1": 0.533779761904762,
543
+ "pass@2": 0.637331271091113,
544
+ "pass@4": 0.7189003565030557,
545
+ "pass@8": 0.7825376796716175,
546
+ "pass@16": 0.8307175370206826,
547
+ "pass@32": 0.8640638671817454,
548
+ "pass@64": 0.8861393273804471,
549
+ "pass@128": 0.9047619047619048
550
+ },
551
+ "12": {
552
+ "pass@1": 0.519447629969419,
553
+ "pass@2": 0.6325109411495584,
554
+ "pass@4": 0.716240447868328,
555
+ "pass@8": 0.7778574011864423,
556
+ "pass@16": 0.8239268041149975,
557
+ "pass@32": 0.8611130957104958,
558
+ "pass@64": 0.8914035041714263,
559
+ "pass@128": 0.9113149847094801
560
+ },
561
+ "13": {
562
+ "pass@1": 0.4123022762345679,
563
+ "pass@2": 0.5496697895401961,
564
+ "pass@4": 0.6635212849551213,
565
+ "pass@8": 0.7482771208219928,
566
+ "pass@16": 0.8095717661613954,
567
+ "pass@32": 0.8546775092104458,
568
+ "pass@64": 0.8861848341058806,
569
+ "pass@128": 0.9074074074074074
570
+ },
571
+ "14": {
572
+ "pass@1": 0.3145168768545994,
573
+ "pass@2": 0.4401428041543027,
574
+ "pass@4": 0.5588902007612545,
575
+ "pass@8": 0.6583031703543211,
576
+ "pass@16": 0.7376897670615584,
577
+ "pass@32": 0.7999922261319541,
578
+ "pass@64": 0.8459428217294271,
579
+ "pass@128": 0.8813056379821959
580
+ },
581
+ "15": {
582
+ "pass@1": 0.23210841049382716,
583
+ "pass@2": 0.3398600782541072,
584
+ "pass@4": 0.4533102031227575,
585
+ "pass@8": 0.5593127461017794,
586
+ "pass@16": 0.6496853742117522,
587
+ "pass@32": 0.7226145759565928,
588
+ "pass@64": 0.778493587814901,
589
+ "pass@128": 0.8179012345679012
590
+ },
591
+ "16": {
592
+ "pass@1": 0.21082227138643067,
593
+ "pass@2": 0.3154901371565279,
594
+ "pass@4": 0.4299715937056539,
595
+ "pass@8": 0.5375134403941474,
596
+ "pass@16": 0.6273091818412996,
597
+ "pass@32": 0.6999977456346922,
598
+ "pass@64": 0.760873083209445,
599
+ "pass@128": 0.8112094395280236
600
+ },
601
+ "17": {
602
+ "pass@1": 0.16036803463855423,
603
+ "pass@2": 0.24516397282041513,
604
+ "pass@4": 0.34268070369215886,
605
+ "pass@8": 0.43866697297296425,
606
+ "pass@16": 0.5218814653519248,
607
+ "pass@32": 0.591200169404961,
608
+ "pass@64": 0.6489296836444958,
609
+ "pass@128": 0.6987951807228916
610
+ },
611
+ "18": {
612
+ "pass@1": 0.16050907258064517,
613
+ "pass@2": 0.23742379984759968,
614
+ "pass@4": 0.32174134160649276,
615
+ "pass@8": 0.4042682761697166,
616
+ "pass@16": 0.48119832780048005,
617
+ "pass@32": 0.5537932896201498,
618
+ "pass@64": 0.6174600257015876,
619
+ "pass@128": 0.6709677419354839
620
+ },
621
+ "19": {
622
+ "pass@1": 0.1531344410876133,
623
+ "pass@2": 0.22887798249161445,
624
+ "pass@4": 0.3099796786164572,
625
+ "pass@8": 0.38504632696140717,
626
+ "pass@16": 0.44934495744307373,
627
+ "pass@32": 0.503410042848671,
628
+ "pass@64": 0.5509170785946527,
629
+ "pass@128": 0.5921450151057401
630
+ },
631
+ "20": {
632
+ "pass@1": 0.14949324324324326,
633
+ "pass@2": 0.2258773291007543,
634
+ "pass@4": 0.30744521067749414,
635
+ "pass@8": 0.3811890808222759,
636
+ "pass@16": 0.4431254983049504,
637
+ "pass@32": 0.49655440542341023,
638
+ "pass@64": 0.5479183432668487,
639
+ "pass@128": 0.6066066066066066
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]