ZachW commited on
Commit
9f113f4
·
verified ·
1 Parent(s): 547faad

Upload extrapolation artifact: 050302 RL OP13-20 step200 summary JSON

Browse files
050302/evals/rl050302_op13_20_step200_merged_hf/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "rl050302_op13_20_step200_merged_hf",
4
+ "id_acc": 0.7096996527777778,
5
+ "ood_acc": 0.34355625,
6
+ "total_acc": 0.5169925986842105,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 206.4914982638889,
11
+ "ood_avg_resp_len": 346.27614765625,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 420142,
19
+ "answer_accuracy": 0.5188680643376541,
20
+ "avg_response_len": 268.15166697952895,
21
+ "resp_tokens_sum": 217129913,
22
+ "pass_at_k": {
23
+ "pass@1": 0.5188680643376541,
24
+ "pass@2": 0.5992609202802675,
25
+ "pass@4": 0.6722506560642548,
26
+ "pass@8": 0.7345034465487572,
27
+ "pass@16": 0.7846585288237924,
28
+ "pass@32": 0.8237347339594457,
29
+ "pass@64": 0.8543135182380192,
30
+ "pass@128": 0.879386658235852
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.6432291666666666,
35
+ "pass@2": 0.7086545217522591,
36
+ "pass@4": 0.7614043908228281,
37
+ "pass@8": 0.8048343235263006,
38
+ "pass@16": 0.8406351544122402,
39
+ "pass@32": 0.8704496732524306,
40
+ "pass@64": 0.8947665326243408,
41
+ "pass@128": 0.9144542772861357
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9332408489096573,
45
+ "pass@2": 0.9600331610125838,
46
+ "pass@4": 0.9771553707655704,
47
+ "pass@8": 0.9854254731466165,
48
+ "pass@16": 0.988133712621007,
49
+ "pass@32": 0.9893193713664016,
50
+ "pass@64": 0.9902739585246382,
51
+ "pass@128": 0.9906542056074766
52
+ },
53
+ "3": {
54
+ "pass@1": 0.8757645259938838,
55
+ "pass@2": 0.9027283645163623,
56
+ "pass@4": 0.9244454405928614,
57
+ "pass@8": 0.9431411848874264,
58
+ "pass@16": 0.9582971577237228,
59
+ "pass@32": 0.9709724633793967,
60
+ "pass@64": 0.9820870411980793,
61
+ "pass@128": 0.9908256880733946
62
+ },
63
+ "4": {
64
+ "pass@1": 0.786328125,
65
+ "pass@2": 0.811225086122047,
66
+ "pass@4": 0.8296586022450316,
67
+ "pass@8": 0.8480276124139998,
68
+ "pass@16": 0.8670405071229179,
69
+ "pass@32": 0.886580837586342,
70
+ "pass@64": 0.9031976667014938,
71
+ "pass@128": 0.915625
72
+ },
73
+ "5": {
74
+ "pass@1": 0.6481150793650794,
75
+ "pass@2": 0.7089566929133857,
76
+ "pass@4": 0.7631540902625267,
77
+ "pass@8": 0.8083630689659272,
78
+ "pass@16": 0.8431150398714341,
79
+ "pass@32": 0.8683170445108382,
80
+ "pass@64": 0.8869570995314593,
81
+ "pass@128": 0.9015873015873016
82
+ },
83
+ "6": {
84
+ "pass@1": 0.6656296781437125,
85
+ "pass@2": 0.7128238601537084,
86
+ "pass@4": 0.7472944426108409,
87
+ "pass@8": 0.777625115201791,
88
+ "pass@16": 0.8070151119011811,
89
+ "pass@32": 0.8349043396850152,
90
+ "pass@64": 0.8587159112471352,
91
+ "pass@128": 0.8772455089820359
92
+ },
93
+ "7": {
94
+ "pass@1": 0.6652875369822485,
95
+ "pass@2": 0.7235667922704184,
96
+ "pass@4": 0.77103220131877,
97
+ "pass@8": 0.8127494418381218,
98
+ "pass@16": 0.8505656587294267,
99
+ "pass@32": 0.8821587788982282,
100
+ "pass@64": 0.9041908255934645,
101
+ "pass@128": 0.9201183431952663
102
+ },
103
+ "8": {
104
+ "pass@1": 0.6062199519230769,
105
+ "pass@2": 0.6758885082273367,
106
+ "pass@4": 0.7324884194163228,
107
+ "pass@8": 0.7816419557403208,
108
+ "pass@16": 0.8223828369246351,
109
+ "pass@32": 0.852237918950051,
110
+ "pass@64": 0.8736887652975474,
111
+ "pass@128": 0.8910256410256411
112
+ },
113
+ "9": {
114
+ "pass@1": 0.6165272887323944,
115
+ "pass@2": 0.6952627675501827,
116
+ "pass@4": 0.7554617862027808,
117
+ "pass@8": 0.8041103178637404,
118
+ "pass@16": 0.8443959343789783,
119
+ "pass@32": 0.8765953737508427,
120
+ "pass@64": 0.9015005102566016,
121
+ "pass@128": 0.923943661971831
122
+ },
123
+ "11": {
124
+ "pass@1": 0.5303868626062322,
125
+ "pass@2": 0.6172469245611297,
126
+ "pass@4": 0.6922284186856246,
127
+ "pass@8": 0.7539144484826834,
128
+ "pass@16": 0.8025265039258181,
129
+ "pass@32": 0.8409687637230883,
130
+ "pass@64": 0.8707484010776608,
131
+ "pass@128": 0.8895184135977338
132
+ },
133
+ "12": {
134
+ "pass@1": 0.5872175141242938,
135
+ "pass@2": 0.6783650851906224,
136
+ "pass@4": 0.753178664107664,
137
+ "pass@8": 0.8143028042975027,
138
+ "pass@16": 0.859033496675691,
139
+ "pass@32": 0.8894844562598956,
140
+ "pass@64": 0.9126887445809961,
141
+ "pass@128": 0.9322033898305084
142
+ },
143
+ "13": {
144
+ "pass@1": 0.48278311128526646,
145
+ "pass@2": 0.6048583170834043,
146
+ "pass@4": 0.7072133163017632,
147
+ "pass@8": 0.7851313248049716,
148
+ "pass@16": 0.8411216588483615,
149
+ "pass@32": 0.8791363164554562,
150
+ "pass@64": 0.9047157885030569,
151
+ "pass@128": 0.9247648902821317
152
+ },
153
+ "14": {
154
+ "pass@1": 0.43460377725856697,
155
+ "pass@2": 0.555995045011897,
156
+ "pass@4": 0.6646590192581069,
157
+ "pass@8": 0.7541116803627453,
158
+ "pass@16": 0.8232941417018945,
159
+ "pass@32": 0.8719027706379187,
160
+ "pass@64": 0.9013330720969001,
161
+ "pass@128": 0.9190031152647975
162
+ },
163
+ "15": {
164
+ "pass@1": 0.3279895231213873,
165
+ "pass@2": 0.45296072095034384,
166
+ "pass@4": 0.5703828111623328,
167
+ "pass@8": 0.6669801186470863,
168
+ "pass@16": 0.7386815028490659,
169
+ "pass@32": 0.7855314858948964,
170
+ "pass@64": 0.8175070460293162,
171
+ "pass@128": 0.846820809248555
172
+ },
173
+ "16": {
174
+ "pass@1": 0.3288054435483871,
175
+ "pass@2": 0.4507123920497839,
176
+ "pass@4": 0.5656597097741817,
177
+ "pass@8": 0.6606423419172535,
178
+ "pass@16": 0.7316974325599377,
179
+ "pass@32": 0.785344071661488,
180
+ "pass@64": 0.828913082215784,
181
+ "pass@128": 0.864516129032258
182
+ },
183
+ "17": {
184
+ "pass@1": 0.2260883309248555,
185
+ "pass@2": 0.33683285637886257,
186
+ "pass@4": 0.4542962630935586,
187
+ "pass@8": 0.5596214750308827,
188
+ "pass@16": 0.645575443790684,
189
+ "pass@32": 0.7144334545854256,
190
+ "pass@64": 0.768551512230959,
191
+ "pass@128": 0.8063583815028902
192
+ },
193
+ "18": {
194
+ "pass@1": 0.20683035714285714,
195
+ "pass@2": 0.31012197694038246,
196
+ "pass@4": 0.4170737739032621,
197
+ "pass@8": 0.5116615581807312,
198
+ "pass@16": 0.5892902093361223,
199
+ "pass@32": 0.6520943406851805,
200
+ "pass@64": 0.7067899408793984,
201
+ "pass@128": 0.76
202
+ },
203
+ "19": {
204
+ "pass@1": 0.17354072822822822,
205
+ "pass@2": 0.26094906422170194,
206
+ "pass@4": 0.358604284543261,
207
+ "pass@8": 0.456174177410017,
208
+ "pass@16": 0.5451820364907284,
209
+ "pass@32": 0.620654648238403,
210
+ "pass@64": 0.6820607217017262,
211
+ "pass@128": 0.7357357357357357
212
+ },
213
+ "20": {
214
+ "pass@1": 0.16500093843843844,
215
+ "pass@2": 0.2588242770329385,
216
+ "pass@4": 0.36199158973371576,
217
+ "pass@8": 0.4565102354746186,
218
+ "pass@16": 0.5352755572347072,
219
+ "pass@32": 0.6006625757883709,
220
+ "pass@64": 0.6604650436060762,
221
+ "pass@128": 0.7177177177177178
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 426542,
228
+ "answer_accuracy": 0.5194636593920499,
229
+ "avg_response_len": 275.8639187938426,
230
+ "resp_tokens_sum": 226517381,
231
+ "pass_at_k": {
232
+ "pass@1": 0.5194636593920499,
233
+ "pass@2": 0.5985632069276694,
234
+ "pass@4": 0.6708152881951757,
235
+ "pass@8": 0.7321737071047528,
236
+ "pass@16": 0.7813725088235535,
237
+ "pass@32": 0.8202247612977291,
238
+ "pass@64": 0.8508530729343532,
239
+ "pass@128": 0.8756040530007794
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.6664589723926381,
244
+ "pass@2": 0.7259352658084149,
245
+ "pass@4": 0.7729965696389173,
246
+ "pass@8": 0.8135385076355918,
247
+ "pass@16": 0.8485919842569918,
248
+ "pass@32": 0.8769118448464734,
249
+ "pass@64": 0.8981314161379662,
250
+ "pass@128": 0.9141104294478528
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9396846504559271,
254
+ "pass@2": 0.9592216032836322,
255
+ "pass@4": 0.9723210259811746,
256
+ "pass@8": 0.9800715101221468,
257
+ "pass@16": 0.9846143152197563,
258
+ "pass@32": 0.9880899912285871,
259
+ "pass@64": 0.9903714652039789,
260
+ "pass@128": 0.9908814589665653
261
+ },
262
+ "3": {
263
+ "pass@1": 0.8826467803030303,
264
+ "pass@2": 0.9058510796945836,
265
+ "pass@4": 0.9239027323289134,
266
+ "pass@8": 0.9375980631196633,
267
+ "pass@16": 0.9490507863390486,
268
+ "pass@32": 0.959496338875189,
269
+ "pass@64": 0.9681067897370846,
270
+ "pass@128": 0.9727272727272728
271
+ },
272
+ "4": {
273
+ "pass@1": 0.7678439349112426,
274
+ "pass@2": 0.8053503412850018,
275
+ "pass@4": 0.8354628671600961,
276
+ "pass@8": 0.8625236702397913,
277
+ "pass@16": 0.8866633816803322,
278
+ "pass@32": 0.9069734335131655,
279
+ "pass@64": 0.9228128772044121,
280
+ "pass@128": 0.9349112426035503
281
+ },
282
+ "5": {
283
+ "pass@1": 0.6854166666666667,
284
+ "pass@2": 0.741926280954011,
285
+ "pass@4": 0.7908239029903872,
286
+ "pass@8": 0.8307448053320945,
287
+ "pass@16": 0.8603277390818179,
288
+ "pass@32": 0.881465096873712,
289
+ "pass@64": 0.897257332752116,
290
+ "pass@128": 0.9101449275362319
291
+ },
292
+ "6": {
293
+ "pass@1": 0.6161130014326648,
294
+ "pass@2": 0.6660646701486812,
295
+ "pass@4": 0.7085057325391633,
296
+ "pass@8": 0.7456869974381682,
297
+ "pass@16": 0.7804926677467746,
298
+ "pass@32": 0.8129104072803103,
299
+ "pass@64": 0.8413385287286547,
300
+ "pass@128": 0.8653295128939829
301
+ },
302
+ "7": {
303
+ "pass@1": 0.6342124277456648,
304
+ "pass@2": 0.6974431494925126,
305
+ "pass@4": 0.7495586211044426,
306
+ "pass@8": 0.7918144315944411,
307
+ "pass@16": 0.8253342442369711,
308
+ "pass@32": 0.8523804182147894,
309
+ "pass@64": 0.8755104389451583,
310
+ "pass@128": 0.8959537572254336
311
+ },
312
+ "8": {
313
+ "pass@1": 0.6068882042253522,
314
+ "pass@2": 0.678888834978374,
315
+ "pass@4": 0.7377912274169952,
316
+ "pass@8": 0.7862378604682415,
317
+ "pass@16": 0.8278968421686421,
318
+ "pass@32": 0.8634145607393767,
319
+ "pass@64": 0.8914266288458986,
320
+ "pass@128": 0.9098591549295775
321
+ },
322
+ "9": {
323
+ "pass@1": 0.6005800898203593,
324
+ "pass@2": 0.6798503141355087,
325
+ "pass@4": 0.7427577106753871,
326
+ "pass@8": 0.7911772594066232,
327
+ "pass@16": 0.8284008725461969,
328
+ "pass@32": 0.8587425143072703,
329
+ "pass@64": 0.8862981605259711,
330
+ "pass@128": 0.9131736526946108
331
+ },
332
+ "11": {
333
+ "pass@1": 0.6095161897590361,
334
+ "pass@2": 0.69583760731904,
335
+ "pass@4": 0.7645579268705863,
336
+ "pass@8": 0.8162260764737186,
337
+ "pass@16": 0.8516898769534356,
338
+ "pass@32": 0.8782974399669164,
339
+ "pass@64": 0.9013430821206966,
340
+ "pass@128": 0.9216867469879518
341
+ },
342
+ "12": {
343
+ "pass@1": 0.5153311128526645,
344
+ "pass@2": 0.6049512656431268,
345
+ "pass@4": 0.6858902189538215,
346
+ "pass@8": 0.7578828775191587,
347
+ "pass@16": 0.8176346019227054,
348
+ "pass@32": 0.8628753895579362,
349
+ "pass@64": 0.8937528903345976,
350
+ "pass@128": 0.9153605015673981
351
+ },
352
+ "13": {
353
+ "pass@1": 0.4445684523809524,
354
+ "pass@2": 0.5486098474822998,
355
+ "pass@4": 0.6393596874445313,
356
+ "pass@8": 0.716317434796044,
357
+ "pass@16": 0.7785482587992962,
358
+ "pass@32": 0.8284156321914673,
359
+ "pass@64": 0.8666372329432644,
360
+ "pass@128": 0.8935574229691877
361
+ },
362
+ "14": {
363
+ "pass@1": 0.43891630116959063,
364
+ "pass@2": 0.5642393487820595,
365
+ "pass@4": 0.6743917816194029,
366
+ "pass@8": 0.7619551785897168,
367
+ "pass@16": 0.8262880821557116,
368
+ "pass@32": 0.8700014401361008,
369
+ "pass@64": 0.8976582359273326,
370
+ "pass@128": 0.9122807017543859
371
+ },
372
+ "15": {
373
+ "pass@1": 0.34739583333333335,
374
+ "pass@2": 0.47785917740396083,
375
+ "pass@4": 0.6037705800979423,
376
+ "pass@8": 0.7069009982792729,
377
+ "pass@16": 0.780389449448767,
378
+ "pass@32": 0.8276691872442753,
379
+ "pass@64": 0.8579498255115579,
380
+ "pass@128": 0.8818181818181818
381
+ },
382
+ "16": {
383
+ "pass@1": 0.32979433760683763,
384
+ "pass@2": 0.4497512730780443,
385
+ "pass@4": 0.5584297905603677,
386
+ "pass@8": 0.6485196575901173,
387
+ "pass@16": 0.7218197491529739,
388
+ "pass@32": 0.7783995418342657,
389
+ "pass@64": 0.8163805628573818,
390
+ "pass@128": 0.8433048433048433
391
+ },
392
+ "17": {
393
+ "pass@1": 0.25642954192546585,
394
+ "pass@2": 0.3702109417029389,
395
+ "pass@4": 0.4881588191421722,
396
+ "pass@8": 0.5911789755276028,
397
+ "pass@16": 0.6696642459629725,
398
+ "pass@32": 0.7277160161846408,
399
+ "pass@64": 0.7728749228117372,
400
+ "pass@128": 0.8105590062111802
401
+ },
402
+ "18": {
403
+ "pass@1": 0.2017233455882353,
404
+ "pass@2": 0.3029668683418246,
405
+ "pass@4": 0.4153720160899007,
406
+ "pass@8": 0.522158553810786,
407
+ "pass@16": 0.6110499248219567,
408
+ "pass@32": 0.6838685275214142,
409
+ "pass@64": 0.7456635518468799,
410
+ "pass@128": 0.7941176470588235
411
+ },
412
+ "19": {
413
+ "pass@1": 0.17510695684523808,
414
+ "pass@2": 0.2658245600159356,
415
+ "pass@4": 0.36414872164416945,
416
+ "pass@8": 0.4539296143435633,
417
+ "pass@16": 0.5288852495897007,
418
+ "pass@32": 0.5929693177682022,
419
+ "pass@64": 0.6520420203037799,
420
+ "pass@128": 0.7142857142857143
421
+ },
422
+ "20": {
423
+ "pass@1": 0.15629678143712575,
424
+ "pass@2": 0.23557437290772787,
425
+ "pass@4": 0.32072419053905704,
426
+ "pass@8": 0.40036653384062204,
427
+ "pass@16": 0.4718154181796683,
428
+ "pass@32": 0.5358254127504247,
429
+ "pass@64": 0.5920170158378041,
430
+ "pass@128": 0.6437125748502994
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 410642,
437
+ "answer_accuracy": 0.5125644072535549,
438
+ "avg_response_len": 296.403654986819,
439
+ "resp_tokens_sum": 237464381,
440
+ "pass_at_k": {
441
+ "pass@1": 0.5125644072535549,
442
+ "pass@2": 0.5958648483192146,
443
+ "pass@4": 0.671119462934007,
444
+ "pass@8": 0.7329547886236961,
445
+ "pass@16": 0.7809510290160512,
446
+ "pass@32": 0.8188012860048482,
447
+ "pass@64": 0.8497488405137779,
448
+ "pass@128": 0.8742610640677424
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.602705223880597,
453
+ "pass@2": 0.6844697526148781,
454
+ "pass@4": 0.7563404168695326,
455
+ "pass@8": 0.8121689999230759,
456
+ "pass@16": 0.8517769048136232,
457
+ "pass@32": 0.8805199885099825,
458
+ "pass@64": 0.9018508650133183,
459
+ "pass@128": 0.9164179104477612
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9134375,
463
+ "pass@2": 0.9471175478065238,
464
+ "pass@4": 0.9656707480314957,
465
+ "pass@8": 0.9754882120243954,
466
+ "pass@16": 0.9815712620898884,
467
+ "pass@32": 0.985608910192567,
468
+ "pass@64": 0.9884398719007649,
469
+ "pass@128": 0.9914285714285714
470
+ },
471
+ "3": {
472
+ "pass@1": 0.8868212463556852,
473
+ "pass@2": 0.9115012568581995,
474
+ "pass@4": 0.9298003041165043,
475
+ "pass@8": 0.9434327678795174,
476
+ "pass@16": 0.9529248871190527,
477
+ "pass@32": 0.9596095328337845,
478
+ "pass@64": 0.966078330717463,
479
+ "pass@128": 0.9737609329446064
480
+ },
481
+ "4": {
482
+ "pass@1": 0.7722953216374269,
483
+ "pass@2": 0.8158991947552606,
484
+ "pass@4": 0.8457159930118383,
485
+ "pass@8": 0.8657981988853063,
486
+ "pass@16": 0.8802608633837845,
487
+ "pass@32": 0.8918749623106107,
488
+ "pass@64": 0.901267233819749,
489
+ "pass@128": 0.9093567251461988
490
+ },
491
+ "5": {
492
+ "pass@1": 0.6706571691176471,
493
+ "pass@2": 0.7326706519221858,
494
+ "pass@4": 0.7866706808707734,
495
+ "pass@8": 0.8289938806620649,
496
+ "pass@16": 0.8614489528583463,
497
+ "pass@32": 0.8862952031282322,
498
+ "pass@64": 0.9044693028604022,
499
+ "pass@128": 0.9176470588235294
500
+ },
501
+ "6": {
502
+ "pass@1": 0.6571618690851735,
503
+ "pass@2": 0.7129446210785165,
504
+ "pass@4": 0.7565953859474185,
505
+ "pass@8": 0.7934670700328363,
506
+ "pass@16": 0.8263155758567313,
507
+ "pass@32": 0.8549766691048055,
508
+ "pass@64": 0.8790474533075844,
509
+ "pass@128": 0.8990536277602523
510
+ },
511
+ "7": {
512
+ "pass@1": 0.604356210443038,
513
+ "pass@2": 0.6695510284810123,
514
+ "pass@4": 0.724756073410127,
515
+ "pass@8": 0.7701029642296547,
516
+ "pass@16": 0.8056596299611132,
517
+ "pass@32": 0.8332147991955038,
518
+ "pass@64": 0.8542664774845095,
519
+ "pass@128": 0.870253164556962
520
+ },
521
+ "8": {
522
+ "pass@1": 0.6112284159159159,
523
+ "pass@2": 0.6834994443262159,
524
+ "pass@4": 0.7442294913023253,
525
+ "pass@8": 0.7955200017165065,
526
+ "pass@16": 0.835547167704164,
527
+ "pass@32": 0.8642999697942094,
528
+ "pass@64": 0.885844631796542,
529
+ "pass@128": 0.9039039039039038
530
+ },
531
+ "9": {
532
+ "pass@1": 0.6024165996784566,
533
+ "pass@2": 0.6791437482593614,
534
+ "pass@4": 0.7391940044873812,
535
+ "pass@8": 0.7833514191290923,
536
+ "pass@16": 0.8169698497647327,
537
+ "pass@32": 0.8439938353276553,
538
+ "pass@64": 0.8672170102630214,
539
+ "pass@128": 0.887459807073955
540
+ },
541
+ "11": {
542
+ "pass@1": 0.5569940476190476,
543
+ "pass@2": 0.6512545306836639,
544
+ "pass@4": 0.7297863037953585,
545
+ "pass@8": 0.7881979214724417,
546
+ "pass@16": 0.8305159472405158,
547
+ "pass@32": 0.8664025710331352,
548
+ "pass@64": 0.8977197150773083,
549
+ "pass@128": 0.9174603174603174
550
+ },
551
+ "12": {
552
+ "pass@1": 0.5906441131498471,
553
+ "pass@2": 0.6903511702665612,
554
+ "pass@4": 0.7648428261260917,
555
+ "pass@8": 0.8219961349721057,
556
+ "pass@16": 0.8647988817303227,
557
+ "pass@32": 0.8948407161093591,
558
+ "pass@64": 0.9176319993654141,
559
+ "pass@128": 0.9357798165137615
560
+ },
561
+ "13": {
562
+ "pass@1": 0.47955246913580246,
563
+ "pass@2": 0.6025725733936037,
564
+ "pass@4": 0.7088902833905013,
565
+ "pass@8": 0.7899693991703378,
566
+ "pass@16": 0.8414343832139441,
567
+ "pass@32": 0.8704900063676707,
568
+ "pass@64": 0.8882142857644638,
569
+ "pass@128": 0.9012345679012346
570
+ },
571
+ "14": {
572
+ "pass@1": 0.3917145771513353,
573
+ "pass@2": 0.5191060246734736,
574
+ "pass@4": 0.6323926407381574,
575
+ "pass@8": 0.7208649834356172,
576
+ "pass@16": 0.7849817631994386,
577
+ "pass@32": 0.8320991882644259,
578
+ "pass@64": 0.8661309975696629,
579
+ "pass@128": 0.8902077151335311
580
+ },
581
+ "15": {
582
+ "pass@1": 0.33253761574074076,
583
+ "pass@2": 0.4562964785651788,
584
+ "pass@4": 0.5772275210390362,
585
+ "pass@8": 0.677586574182789,
586
+ "pass@16": 0.7481361489708472,
587
+ "pass@32": 0.7971674319182525,
588
+ "pass@64": 0.8354807477013398,
589
+ "pass@128": 0.8672839506172839
590
+ },
591
+ "16": {
592
+ "pass@1": 0.30212942477876104,
593
+ "pass@2": 0.4332392632336884,
594
+ "pass@4": 0.5611566145492871,
595
+ "pass@8": 0.6676743747616286,
596
+ "pass@16": 0.7497157527260427,
597
+ "pass@32": 0.8118684200882799,
598
+ "pass@64": 0.8597076780207225,
599
+ "pass@128": 0.8938053097345132
600
+ },
601
+ "17": {
602
+ "pass@1": 0.20277202560240964,
603
+ "pass@2": 0.30790929465895067,
604
+ "pass@4": 0.42129222909636277,
605
+ "pass@8": 0.5249290260578731,
606
+ "pass@16": 0.6136367096463329,
607
+ "pass@32": 0.6894706624655844,
608
+ "pass@64": 0.750886437329411,
609
+ "pass@128": 0.7951807228915663
610
+ },
611
+ "18": {
612
+ "pass@1": 0.19248991935483872,
613
+ "pass@2": 0.28834931419862814,
614
+ "pass@4": 0.3872474058686214,
615
+ "pass@8": 0.4755959602758333,
616
+ "pass@16": 0.5545506915364673,
617
+ "pass@32": 0.6301610647827953,
618
+ "pass@64": 0.6995870673914846,
619
+ "pass@128": 0.7516129032258064
620
+ },
621
+ "19": {
622
+ "pass@1": 0.16498300604229607,
623
+ "pass@2": 0.25099168589575854,
624
+ "pass@4": 0.3448424330266873,
625
+ "pass@8": 0.4322719262756728,
626
+ "pass@16": 0.5071666429847727,
627
+ "pass@32": 0.5715699397863766,
628
+ "pass@64": 0.634263631001721,
629
+ "pass@128": 0.7009063444108762
630
+ },
631
+ "20": {
632
+ "pass@1": 0.15697728978978978,
633
+ "pass@2": 0.24347046357380994,
634
+ "pass@4": 0.33896897037645063,
635
+ "pass@8": 0.4278276847043507,
636
+ "pass@16": 0.5039384810396262,
637
+ "pass@32": 0.5702084386681836,
638
+ "pass@64": 0.6286186409348357,
639
+ "pass@128": 0.6726726726726727
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]