{ "checkpoint": "checkpoint-38796", "id": { "answer_accuracy": 0.9899713541666667, "avg_response_len": 200.3366076388889, "avg_loss": 0.0, "count": 1152000, "validation_metrics": {}, "pass_at_k": { "pass@1": 0.9899713541666667, "pass@2": 0.9929163932633402, "pass@4": 0.9950443724430285, "pass@8": 0.9967556474644427, "pass@16": 0.9981256002928118, "pass@32": 0.9991009218100472, "pass@64": 0.9996657523498159, "pass@128": 0.9998888888888889 }, "per_op_pass_at_k": { "10": { "pass@1": 0.941265625, "pass@2": 0.9563416584645673, "pass@4": 0.9692801073303342, "pass@8": 0.9799107461407804, "pass@16": 0.9882794827881477, "pass@32": 0.9940941877802166, "pass@64": 0.9975055528695351, "pass@128": 0.999 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.998203125, "pass@2": 0.9989484498031497, "pass@4": 0.999429442632171, "pass@8": 0.9997332787706423, "pass@16": 0.9999360887968483, "pass@32": 0.9999974424774314, "pass@64": 0.9999999996028359, "pass@128": 1.0 }, "4": { "pass@1": 0.99871875, "pass@2": 0.9995188238188978, "pass@4": 0.9998979254780653, "pass@8": 0.9999947100672492, "pass@16": 0.9999999877346716, "pass@32": 0.9999999999999909, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9939296875, "pass@2": 0.9966125738188978, "pass@4": 0.9980609976565431, "pass@8": 0.9988772917617279, "pass@16": 0.9992186179280644, "pass@32": 0.9994389261114691, "pass@64": 0.9997519685039321, "pass@128": 1.0 }, "6": { "pass@1": 0.9972734375, "pass@2": 0.9982062007874017, "pass@4": 0.9987222359392577, "pass@8": 0.9992138876134402, "pass@16": 0.9996969605662573, "pass@32": 0.9999586920114116, "pass@64": 0.9999996921831551, "pass@128": 1.0 }, "7": { "pass@1": 0.9960859375, "pass@2": 0.9974864665354329, "pass@4": 0.9981825993625796, "pass@8": 0.9987325731094482, "pass@16": 0.9992691028653226, "pass@32": 0.9997610564276355, "pass@64": 0.9999847315644707, "pass@128": 1.0 }, "8": { "pass@1": 0.991609375, "pass@2": 0.994200910433071, "pass@4": 0.9958966556055493, "pass@8": 0.9973900285847406, "pass@16": 0.9986377579744303, "pass@32": 0.9995155655514678, "pass@64": 0.9999335572652138, "pass@128": 1.0 }, "9": { "pass@1": 0.99265625, "pass@2": 0.9949324557086615, "pass@4": 0.9959293879827522, "pass@8": 0.9969483111319597, "pass@16": 0.9980924039815611, "pass@32": 0.9991424259308008, "pass@64": 0.9998162691591999, "pass@128": 1.0 } }, "per_op_accuracy": { "10": 0.941265625, "2": 1.0, "3": 0.998203125, "4": 0.99871875, "5": 0.9939296875, "6": 0.9972734375, "7": 0.9960859375, "8": 0.991609375, "9": 0.99265625 }, "per_op_avg_response_len": { "10": 281.453609375, "2": 115.61171875, "3": 145.2217734375, "4": 160.020953125, "5": 173.8887578125, "6": 199.223328125, "7": 224.244828125, "8": 243.6493359375, "9": 259.7151640625 }, "per_op_avg_loss": {}, "per_op_length": { "10": 281.453609375, "2": 115.61171875, "3": 145.2217734375, "4": 160.020953125, "5": 173.8887578125, "6": 199.223328125, "7": 224.244828125, "8": 243.6493359375, "9": 259.7151640625 }, "per_op_loss": {}, "per_template": { "crazy_zootopia": { "count": 379008, "correct": 375204, "answer_accuracy": 0.9899632725430598, "avg_response_len": 190.6923653326579, "resp_tokens_sum": 72273932, "pass_at_k": { "pass@1": 0.9899632725430598, "pass@2": 0.9924113957164932, "pass@4": 0.994310804899388, "pass@8": 0.9960184386538377, "pass@16": 0.9974505520342333, "pass@32": 0.9985272006809436, "pass@64": 0.9992532853259626, "pass@128": 0.999662276258021 }, "per_op_pass_at_k": { "10": { "pass@1": 0.940058075221239, "pass@2": 0.952202445241911, "pass@4": 0.9639851915634436, "pass@8": 0.9747452943211968, "pass@16": 0.9837117746740343, "pass@32": 0.9903082598333327, "pass@64": 0.9945723951890973, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999641628440367, "pass@2": 0.9999793066291025, "pass@4": 0.999999905401733, "pass@8": 0.9999999999996472, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9995361328125, "pass@2": 0.9999784694881889, "pass@4": 0.9999999370195912, "pass@8": 0.9999999999999016, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.991170634920635, "pass@2": 0.9935695538057743, "pass@4": 0.9953833833270841, "pass@8": 0.996783521967192, "pass@16": 0.9975425302123753, "pass@32": 0.9982188768739435, "pass@64": 0.9992125984251853, "pass@128": 1.0 }, "6": { "pass@1": 0.9999298278443114, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9981046597633136, "pass@2": 0.9996745853328983, "pass@4": 0.9999718887764769, "pass@8": 0.9999998022755866, "pass@16": 0.9999999999967328, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9924879807692307, "pass@2": 0.995214390268524, "pass@4": 0.9969837258924363, "pass@8": 0.9983251613066434, "pass@16": 0.9993038302603201, "pass@32": 0.9998838431738969, "pass@64": 0.9999990205966608, "pass@128": 1.0 }, "9": { "pass@1": 0.9905809859154929, "pass@2": 0.992608059775979, "pass@4": 0.9937131769620348, "pass@8": 0.9952330962332059, "pass@16": 0.9970819743714158, "pass@32": 0.9986530587245823, "pass@64": 0.9996542900027615, "pass@128": 1.0 } } }, "teachers_in_school": { "count": 390656, "correct": 386226, "answer_accuracy": 0.9886600999344692, "avg_response_len": 198.37706319626474, "resp_tokens_sum": 77497190, "pass_at_k": { "pass@1": 0.9886600999344692, "pass@2": 0.9917850676850604, "pass@4": 0.9941407065986837, "pass@8": 0.9961116676864055, "pass@16": 0.9978235394203914, "pass@32": 0.9991057439094873, "pass@64": 0.999825416636273, "pass@128": 1.0 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9300469708588958, "pass@2": 0.9466099916670692, "pass@4": 0.9608256378803874, "pass@8": 0.9735479075977559, "pass@16": 0.9852696934862603, "pass@32": 0.9940972426092773, "pass@64": 0.9987733649475956, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9961647727272728, "pass@2": 0.9972918157957528, "pass@4": 0.9983365499767074, "pass@8": 0.9991928917832018, "pass@16": 0.9998063298472625, "pass@32": 0.9999922499316101, "pass@64": 0.9999999987964725, "pass@128": 1.0 }, "4": { "pass@1": 0.997965976331361, "pass@2": 0.9991187596095608, "pass@4": 0.9997857705841503, "pass@8": 0.9999865024173374, "pass@16": 0.9999999643806659, "pass@32": 0.9999999999999731, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.99375, "pass@2": 0.9967833504507589, "pass@4": 0.9986635988436225, "pass@8": 0.9996831882834758, "pass@16": 0.9999789012150662, "pass@32": 0.9999999417280491, "pass@64": 0.9999999999999963, "pass@128": 1.0 }, "6": { "pass@1": 0.9955676934097422, "pass@2": 0.9973236243034089, "pass@4": 0.9983310997300121, "pass@8": 0.9991142096348365, "pass@16": 0.9997496114237934, "pass@32": 0.9999854792844564, "pass@64": 0.999999993573057, "pass@128": 1.0 }, "7": { "pass@1": 0.9948067196531792, "pass@2": 0.9959908089071959, "pass@4": 0.9969885587820309, "pass@8": 0.9979235613025731, "pass@16": 0.9987206336864413, "pass@32": 0.9995059858135572, "pass@64": 0.9999600471131015, "pass@128": 1.0 }, "8": { "pass@1": 0.9922975352112676, "pass@2": 0.9943263419097261, "pass@4": 0.99571091852955, "pass@8": 0.9969972148105722, "pass@16": 0.99814919003857, "pass@32": 0.9991126715814442, "pass@64": 0.9998322253384277, "pass@128": 1.0 }, "9": { "pass@1": 0.995158121257485, "pass@2": 0.9969157403696542, "pass@4": 0.997358424601865, "pass@8": 0.9976945425380946, "pass@16": 0.9982569113741001, "pass@32": 0.9990676801627342, "pass@64": 0.9998216794756942, "pass@128": 1.0 } } }, "movie_festival_awards": { "count": 382336, "correct": 379017, "answer_accuracy": 0.9913191538332775, "avg_response_len": 211.89908875962504, "resp_tokens_sum": 81016650, "pass_at_k": { "pass@1": 0.9913191538332775, "pass@2": 0.9945729394304457, "pass@4": 0.9966948852162644, "pass@8": 0.9981444327241598, "pass@16": 0.9991034066792451, "pass@32": 0.9996647220162015, "pass@64": 0.9999114903663415, "pass@128": 1.0 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9534048507462687, "pass@2": 0.970000514161476, "pass@4": 0.9828655804218498, "pass@8": 0.9913297716090343, "pass@16": 0.9958306599914305, "pass@32": 0.9979223480778879, "pass@64": 0.9992399818731803, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9987928206997084, "pass@2": 0.99955952572255, "pass@4": 0.9999370614387487, "pass@8": 0.9999989051962135, "pass@16": 0.9999999998462147, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986979166666666, "pass@2": 0.9994841310954552, "pass@4": 0.9999133192232549, "pass@8": 0.9999978720766682, "pass@16": 0.9999999993392, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9966681985294118, "pass@2": 0.9992585543075498, "pass@4": 0.999930265610181, "pass@8": 0.9999993599537156, "pass@16": 0.9999999999763772, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.996352523659306, "pass@2": 0.9972878735189645, "pass@4": 0.9978065682444268, "pass@8": 0.9984953578892186, "pass@16": 0.9993197040358152, "pass@32": 0.9998856774168334, "pass@64": 0.999999036044663, "pass@128": 1.0 }, "7": { "pass@1": 0.9953273338607594, "pass@2": 0.9967836607943786, "pass@4": 0.9975761380302146, "pass@8": 0.998262935853195, "pass@16": 0.9990878595282221, "pass@32": 0.9997847637219769, "pass@64": 0.9999954280485364, "pass@128": 1.0 }, "8": { "pass@1": 0.9900525525525525, "pass@2": 0.9931176254995152, "pass@4": 0.9950761472946511, "pass@8": 0.996932633631575, "pass@16": 0.9985345359444987, "pass@32": 0.9996000242336316, "pass@64": 0.9999802487805222, "pass@128": 1.0 }, "9": { "pass@1": 0.9923382234726688, "pass@2": 0.9954557466389852, "pass@4": 0.9969244255440737, "pass@8": 0.9981047741525653, "pass@16": 0.9990691147291293, "pass@32": 0.9997813019589095, "pass@64": 0.9999953545444935, "pass@128": 1.0 } } } } }, "ood": { "answer_accuracy": 0.8540453125, "avg_response_len": 381.87561171875, "avg_loss": 0.0, "count": 1280000, "validation_metrics": {}, "pass_at_k": { "pass@1": 0.8540453125, "pass@2": 0.9031663385826517, "pass@4": 0.9325282339801222, "pass@8": 0.9512891821432544, "pass@16": 0.9638628298980004, "pass@32": 0.9727622294964876, "pass@64": 0.97954231019947, "pass@128": 0.9848 }, "per_op_pass_at_k": { "11": { "pass@1": 0.9624921875, "pass@2": 0.9759582923228339, "pass@4": 0.9853311848518935, "pass@8": 0.9919277954226844, "pass@16": 0.9960295972437585, "pass@32": 0.9978717406808197, "pass@64": 0.9984840527828055, "pass@128": 0.999 }, "12": { "pass@1": 0.951375, "pass@2": 0.9708375984251973, "pass@4": 0.9831443614548185, "pass@8": 0.9908391481052844, "pass@16": 0.9954637287723308, "pass@32": 0.9979827905163924, "pass@64": 0.9992459838187541, "pass@128": 1.0 }, "13": { "pass@1": 0.927671875, "pass@2": 0.9560797244094491, "pass@4": 0.9735508835770535, "pass@8": 0.9845974255568427, "pass@16": 0.9910754289987089, "pass@32": 0.9948196851214032, "pass@64": 0.9974767526798574, "pass@128": 1.0 }, "14": { "pass@1": 0.9342578125, "pass@2": 0.9600164862204726, "pass@4": 0.9764483093363321, "pass@8": 0.9877587695211665, "pass@16": 0.9944378788673935, "pass@32": 0.9976130640940937, "pass@64": 0.9991646203883312, "pass@128": 1.0 }, "15": { "pass@1": 0.930546875, "pass@2": 0.9632146899606311, "pass@4": 0.979843436070491, "pass@8": 0.9879005123945149, "pass@16": 0.9915401089295047, "pass@32": 0.9936956420221912, "pass@64": 0.9951344064115004, "pass@128": 0.996 }, "16": { "pass@1": 0.866078125, "pass@2": 0.9150172244094495, "pass@4": 0.9411895326209221, "pass@8": 0.9579022839054648, "pass@16": 0.969572659701787, "pass@32": 0.9778052014424473, "pass@64": 0.9839818197350004, "pass@128": 0.989 }, "17": { "pass@1": 0.8524375, "pass@2": 0.9029194143700803, "pass@4": 0.9286221111736035, "pass@8": 0.9443386208689557, "pass@16": 0.955734081883812, "pass@32": 0.9648144384726397, "pass@64": 0.9726182428749347, "pass@128": 0.98 }, "18": { "pass@1": 0.8109296875, "pass@2": 0.8804515255905518, "pass@4": 0.9190456437945261, "pass@8": 0.9411653142560411, "pass@16": 0.954709171238404, "pass@32": 0.9636073849642399, "pass@64": 0.9701930633795666, "pass@128": 0.975 }, "19": { "pass@1": 0.6708515625, "pass@2": 0.7712925688976383, "pass@4": 0.8335043385826767, "pass@8": 0.8737365678515328, "pass@16": 0.9019963268627569, "pass@32": 0.9241395020391616, "pass@64": 0.9433327950305856, "pass@128": 0.96 }, "20": { "pass@1": 0.6338125, "pass@2": 0.7358758612204728, "pass@4": 0.8046025383389569, "pass@8": 0.8527253835500371, "pass@16": 0.8880693164814899, "pass@32": 0.915272845611458, "pass@64": 0.9357913648933431, "pass@128": 0.949 } }, "per_op_accuracy": { "11": 0.9624921875, "12": 0.951375, "13": 0.927671875, "14": 0.9342578125, "15": 0.930546875, "16": 0.866078125, "17": 0.8524375, "18": 0.8109296875, "19": 0.6708515625, "20": 0.6338125 }, "per_op_avg_response_len": { "11": 300.8788359375, "12": 313.412328125, "13": 334.19246875, "14": 361.4050703125, "15": 381.8040859375, "16": 392.90459375, "17": 414.05021875, "18": 432.1644609375, "19": 434.3923984375, "20": 453.55165625 }, "per_op_avg_loss": {}, "per_op_length": { "11": 300.8788359375, "12": 313.412328125, "13": 334.19246875, "14": 361.4050703125, "15": 381.8040859375, "16": 392.90459375, "17": 414.05021875, "18": 432.1644609375, "19": 434.3923984375, "20": 453.55165625 }, "per_op_loss": {}, "per_template": { "crazy_zootopia": { "count": 430720, "correct": 366800, "answer_accuracy": 0.8515973254086181, "avg_response_len": 363.4109096396731, "resp_tokens_sum": 156528347, "pass_at_k": { "pass@1": 0.8515973254086181, "pass@2": 0.9000831422353838, "pass@4": 0.9288547789252976, "pass@8": 0.9476257278411426, "pass@16": 0.961010872591785, "pass@32": 0.9707065629086032, "pass@64": 0.9777220682162787, "pass@128": 0.9830609212481426 }, "per_op_pass_at_k": { "11": { "pass@1": 0.9567767351274787, "pass@2": 0.9729954579420491, "pass@4": 0.9837077224412106, "pass@8": 0.9915053244533025, "pass@16": 0.9967604348473688, "pass@32": 0.9993037377604537, "pass@64": 0.999959962274875, "pass@128": 1.0 }, "12": { "pass@1": 0.9463497528248588, "pass@2": 0.9650545370123225, "pass@4": 0.9766829093397222, "pass@8": 0.9848453467690281, "pass@16": 0.9914138441913528, "pass@32": 0.9958525342115812, "pass@64": 0.9982239261012225, "pass@128": 1.0 }, "13": { "pass@1": 0.9212627351097179, "pass@2": 0.953259601856194, "pass@4": 0.974031526035734, "pass@8": 0.9872172251386195, "pass@16": 0.9946975877158775, "pass@32": 0.9979212628228779, "pass@64": 0.9992200434190286, "pass@128": 1.0 }, "14": { "pass@1": 0.9226781542056075, "pass@2": 0.9488315457355219, "pass@4": 0.966660107907072, "pass@8": 0.9807327520370567, "pass@16": 0.9900033030501925, "pass@32": 0.994705328348, "pass@64": 0.997677205060592, "pass@128": 1.0 }, "15": { "pass@1": 0.9338872832369942, "pass@2": 0.9658665826544082, "pass@4": 0.9800264382595235, "pass@8": 0.9852710411507393, "pass@16": 0.9867086223947849, "pass@32": 0.9875391269726735, "pass@64": 0.9882671703608444, "pass@128": 0.9884393063583815 }, "16": { "pass@1": 0.8443296370967742, "pass@2": 0.8949314992379985, "pass@4": 0.9236545057875828, "pass@8": 0.9436194889710628, "pass@16": 0.9597137566119098, "pass@32": 0.9731177144426382, "pass@64": 0.9836290640887158, "pass@128": 0.9935483870967742 }, "17": { "pass@1": 0.8775063222543352, "pass@2": 0.9258678343043105, "pass@4": 0.9494584514544068, "pass@8": 0.9630582535369274, "pass@16": 0.9726016060789089, "pass@32": 0.9786961371782323, "pass@64": 0.9822631761264358, "pass@128": 0.9855491329479769 }, "18": { "pass@1": 0.8074330357142857, "pass@2": 0.8726863048368955, "pass@4": 0.9079920633135147, "pass@8": 0.9290310338171512, "pass@16": 0.9431022516433624, "pass@32": 0.9531360758782476, "pass@64": 0.9606210100169015, "pass@128": 0.9657142857142857 }, "19": { "pass@1": 0.6631240615615616, "pass@2": 0.7647959228913951, "pass@4": 0.8269798164306035, "pass@8": 0.8654212985928951, "pass@16": 0.8938579593227545, "pass@32": 0.9182400396910518, "pass@64": 0.9393446013378842, "pass@128": 0.9579579579579579 }, "20": { "pass@1": 0.6333521021021021, "pass@2": 0.7295261550920993, "pass@4": 0.7942091112484806, "pass@8": 0.8420309365275893, "pass@16": 0.8790149220263148, "pass@32": 0.9073538190959931, "pass@64": 0.9277009935323411, "pass@128": 0.93993993993994 } } }, "teachers_in_school": { "count": 430464, "correct": 371427, "answer_accuracy": 0.8628526427297056, "avg_response_len": 377.68319534269995, "resp_tokens_sum": 162579019, "pass_at_k": { "pass@1": 0.8628526427297056, "pass@2": 0.9105499562749843, "pass@4": 0.9382578365260149, "pass@8": 0.9553304746020284, "pass@16": 0.9664182673619955, "pass@32": 0.974095193480023, "pass@64": 0.9799923229829369, "pass@128": 0.9845376152245019 }, "per_op_pass_at_k": { "11": { "pass@1": 0.97265625, "pass@2": 0.9817142586092406, "pass@4": 0.9872070927128084, "pass@8": 0.9907918466392928, "pass@16": 0.9932876981463241, "pass@32": 0.9946033623345881, "pass@64": 0.9954810039531555, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9539086990595611, "pass@2": 0.9742443783476912, "pass@4": 0.9873180110910899, "pass@8": 0.9939946254019711, "pass@16": 0.9967163926023948, "pass@32": 0.9983956273931865, "pass@64": 0.9996077018477316, "pass@128": 1.0 }, "13": { "pass@1": 0.9435836834733894, "pass@2": 0.9670878686120119, "pass@4": 0.9795068925207874, "pass@8": 0.9867661154939795, "pass@16": 0.9912037146425846, "pass@32": 0.9943074150090554, "pass@64": 0.9971910964276763, "pass@128": 1.0 }, "14": { "pass@1": 0.9339820906432749, "pass@2": 0.961808433485288, "pass@4": 0.9791213154276766, "pass@8": 0.9903633423119838, "pass@16": 0.9965715597469695, "pass@32": 0.998978060543375, "pass@64": 0.9998257129593244, "pass@128": 1.0 }, "15": { "pass@1": 0.936032196969697, "pass@2": 0.9698241022429014, "pass@4": 0.986383716808126, "pass@8": 0.993887020431712, "pass@16": 0.997341288437486, "pass@32": 0.9992550901802849, "pass@64": 0.999953724719699, "pass@128": 1.0 }, "16": { "pass@1": 0.8875979344729344, "pass@2": 0.9302362344931245, "pass@4": 0.9518918415112638, "pass@8": 0.9653093003879017, "pass@16": 0.97478416069728, "pass@32": 0.9809651521660178, "pass@64": 0.986492391819359, "pass@128": 0.9914529914529915 }, "17": { "pass@1": 0.8626746894409938, "pass@2": 0.9124707322590117, "pass@4": 0.9368205890024611, "pass@8": 0.9508498521267209, "pass@16": 0.960456726487503, "pass@32": 0.9678057412486193, "pass@64": 0.9734389189717876, "pass@128": 0.9782608695652174 }, "18": { "pass@1": 0.8121553308823529, "pass@2": 0.8809753502779064, "pass@4": 0.9216372758552233, "pass@8": 0.9462221099752758, "pass@16": 0.9606949776103674, "pass@32": 0.9685754572068952, "pass@64": 0.9731564892213497, "pass@128": 0.9764705882352941 }, "19": { "pass@1": 0.66943359375, "pass@2": 0.7666238985751784, "pass@4": 0.8258763982627161, "pass@8": 0.8655574931646675, "pass@16": 0.8941625232220163, "pass@32": 0.9161714667742615, "pass@64": 0.9350029611318369, "pass@128": 0.9523809523809523 }, "20": { "pass@1": 0.6556652320359282, "pass@2": 0.7598498868404915, "pass@4": 0.8262946623625637, "pass@8": 0.8690020135890967, "pass@16": 0.8983220264807161, "pass@32": 0.9212925306509724, "pass@64": 0.9391524696479382, "pass@128": 0.9491017964071856 } } }, "movie_festival_awards": { "count": 418816, "correct": 354951, "answer_accuracy": 0.8475106013141809, "avg_response_len": 405.1741504622555, "resp_tokens_sum": 169693417, "pass_at_k": { "pass@1": 0.8475106013141809, "pass@2": 0.8987481996491417, "pass@4": 0.9304171468461809, "pass@8": 0.9509030749267878, "pass@16": 0.9641693397219542, "pass@32": 0.9735062882347418, "pass@64": 0.9809517604692125, "pass@128": 0.9868581907090465 }, "per_op_pass_at_k": { "11": { "pass@1": 0.9581845238095238, "pass@2": 0.973211942257218, "pass@4": 0.9851733461888689, "pass@8": 0.9935984850997558, "pass@16": 0.9981004697081209, "pass@32": 0.9997117617660194, "pass@64": 0.9999952151978636, "pass@128": 1.0 }, "12": { "pass@1": 0.9543434633027523, "pass@2": 0.9737747266970067, "pass@4": 0.9860677859648275, "pass@8": 0.994249571393947, "pass@16": 0.9986259898728691, "pass@32": 0.9998862026515789, "pass@64": 0.9999995629648154, "pass@128": 1.0 }, "13": { "pass@1": 0.9164496527777778, "pass@2": 0.9467269824778841, "pass@4": 0.966515018955963, "pass@8": 0.9796284799576312, "pass@16": 0.9873678160183381, "pass@32": 0.9923304170452849, "pass@64": 0.9960751154460078, "pass@128": 1.0 }, "14": { "pass@1": 0.9455675074183977, "pass@2": 0.9688518569359099, "pass@4": 0.9830591241005852, "pass@8": 0.9918079912361211, "pass@16": 0.9964965731596999, "pass@32": 0.9989974984823478, "pass@64": 0.999910515524606, "pass@128": 1.0 }, "15": { "pass@1": 0.9213927469135802, "pass@2": 0.9536509216729855, "pass@4": 0.9729866107593029, "pass@8": 0.9846111586848604, "pass@16": 0.9907910506066029, "pass@32": 0.9946077911424446, "pass@64": 0.9975593713245301, "pass@128": 1.0 }, "16": { "pass@1": 0.8636845501474927, "pass@2": 0.9176269656005392, "pass@4": 0.9461433612280757, "pass@8": 0.9632940350686785, "pass@16": 0.9731921968948374, "pass@32": 0.9788198865927932, "pass@64": 0.9817049567519268, "pass@128": 0.9823008849557522 }, "17": { "pass@1": 0.816382718373494, "pass@2": 0.8697396623897163, "pass@4": 0.8989555943120052, "pass@8": 0.918514496266187, "pass@16": 0.9335748802757035, "pass@32": 0.9474461636352892, "pass@64": 0.9617706386333493, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8135332661290322, "pass@2": 0.8886441929133857, "pass@4": 0.9286830575613528, "pass@8": 0.9493191452530526, "pass@16": 0.9612486799216202, "pass@32": 0.9699809772790603, "pass@64": 0.9777500114141688, "pass@128": 0.9838709677419355 }, "19": { "pass@1": 0.6800651435045317, "pass@2": 0.7825676636058709, "pass@4": 0.8478114498338757, "pass@8": 0.890404706092961, "pass@16": 0.9181360078721577, "pass@32": 0.9381630090208142, "pass@64": 0.9558007487757464, "pass@128": 0.9697885196374623 }, "20": { "pass@1": 0.6123545420420421, "pass@2": 0.7181795476578936, "pass@4": 0.7932386998922434, "pass@8": 0.8470943217645397, "pass@16": 0.8868402120185244, "pass@32": 0.9171541099551569, "pass@64": 0.9405105380920781, "pass@128": 0.9579579579579579 } } } } }, "total": { "answer_accuracy": 0.9184313322368421, "avg_loss": 0.0, "count": 2432000, "validation_metrics": { "id": {}, "ood": {} }, "pass_at_k": { "pass@1": 0.9184313322368421, "pass@2": 0.9456795223787686, "pass@4": 0.9621411416730752, "pass@8": 0.9728259288743495, "pass@16": 0.9800925632428983, "pass@32": 0.9852384521713364, "pass@64": 0.989074467007528, "pass@128": 0.9919473684210526 }, "per_op_pass_at_k": { "10": { "pass@1": 0.941265625, "pass@2": 0.9563416584645673, "pass@4": 0.9692801073303342, "pass@8": 0.9799107461407804, "pass@16": 0.9882794827881477, "pass@32": 0.9940941877802166, "pass@64": 0.9975055528695351, "pass@128": 0.999 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.998203125, "pass@2": 0.9989484498031497, "pass@4": 0.999429442632171, "pass@8": 0.9997332787706423, "pass@16": 0.9999360887968483, "pass@32": 0.9999974424774314, "pass@64": 0.9999999996028359, "pass@128": 1.0 }, "4": { "pass@1": 0.99871875, "pass@2": 0.9995188238188978, "pass@4": 0.9998979254780653, "pass@8": 0.9999947100672492, "pass@16": 0.9999999877346716, "pass@32": 0.9999999999999909, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9939296875, "pass@2": 0.9966125738188978, "pass@4": 0.9980609976565431, "pass@8": 0.9988772917617279, "pass@16": 0.9992186179280644, "pass@32": 0.9994389261114691, "pass@64": 0.9997519685039321, "pass@128": 1.0 }, "6": { "pass@1": 0.9972734375, "pass@2": 0.9982062007874017, "pass@4": 0.9987222359392577, "pass@8": 0.9992138876134402, "pass@16": 0.9996969605662573, "pass@32": 0.9999586920114116, "pass@64": 0.9999996921831551, "pass@128": 1.0 }, "7": { "pass@1": 0.9960859375, "pass@2": 0.9974864665354329, "pass@4": 0.9981825993625796, "pass@8": 0.9987325731094482, "pass@16": 0.9992691028653226, "pass@32": 0.9997610564276355, "pass@64": 0.9999847315644707, "pass@128": 1.0 }, "8": { "pass@1": 0.991609375, "pass@2": 0.994200910433071, "pass@4": 0.9958966556055493, "pass@8": 0.9973900285847406, "pass@16": 0.9986377579744303, "pass@32": 0.9995155655514678, "pass@64": 0.9999335572652138, "pass@128": 1.0 }, "9": { "pass@1": 0.99265625, "pass@2": 0.9949324557086615, "pass@4": 0.9959293879827522, "pass@8": 0.9969483111319597, "pass@16": 0.9980924039815611, "pass@32": 0.9991424259308008, "pass@64": 0.9998162691591999, "pass@128": 1.0 }, "11": { "pass@1": 0.9624921875, "pass@2": 0.9759582923228339, "pass@4": 0.9853311848518935, "pass@8": 0.9919277954226844, "pass@16": 0.9960295972437585, "pass@32": 0.9978717406808197, "pass@64": 0.9984840527828055, "pass@128": 0.999 }, "12": { "pass@1": 0.951375, "pass@2": 0.9708375984251973, "pass@4": 0.9831443614548185, "pass@8": 0.9908391481052844, "pass@16": 0.9954637287723308, "pass@32": 0.9979827905163924, "pass@64": 0.9992459838187541, "pass@128": 1.0 }, "13": { "pass@1": 0.927671875, "pass@2": 0.9560797244094491, "pass@4": 0.9735508835770535, "pass@8": 0.9845974255568427, "pass@16": 0.9910754289987089, "pass@32": 0.9948196851214032, "pass@64": 0.9974767526798574, "pass@128": 1.0 }, "14": { "pass@1": 0.9342578125, "pass@2": 0.9600164862204726, "pass@4": 0.9764483093363321, "pass@8": 0.9877587695211665, "pass@16": 0.9944378788673935, "pass@32": 0.9976130640940937, "pass@64": 0.9991646203883312, "pass@128": 1.0 }, "15": { "pass@1": 0.930546875, "pass@2": 0.9632146899606311, "pass@4": 0.979843436070491, "pass@8": 0.9879005123945149, "pass@16": 0.9915401089295047, "pass@32": 0.9936956420221912, "pass@64": 0.9951344064115004, "pass@128": 0.996 }, "16": { "pass@1": 0.866078125, "pass@2": 0.9150172244094495, "pass@4": 0.9411895326209221, "pass@8": 0.9579022839054648, "pass@16": 0.969572659701787, "pass@32": 0.9778052014424473, "pass@64": 0.9839818197350004, "pass@128": 0.989 }, "17": { "pass@1": 0.8524375, "pass@2": 0.9029194143700803, "pass@4": 0.9286221111736035, "pass@8": 0.9443386208689557, "pass@16": 0.955734081883812, "pass@32": 0.9648144384726397, "pass@64": 0.9726182428749347, "pass@128": 0.98 }, "18": { "pass@1": 0.8109296875, "pass@2": 0.8804515255905518, "pass@4": 0.9190456437945261, "pass@8": 0.9411653142560411, "pass@16": 0.954709171238404, "pass@32": 0.9636073849642399, "pass@64": 0.9701930633795666, "pass@128": 0.975 }, "19": { "pass@1": 0.6708515625, "pass@2": 0.7712925688976383, "pass@4": 0.8335043385826767, "pass@8": 0.8737365678515328, "pass@16": 0.9019963268627569, "pass@32": 0.9241395020391616, "pass@64": 0.9433327950305856, "pass@128": 0.96 }, "20": { "pass@1": 0.6338125, "pass@2": 0.7358758612204728, "pass@4": 0.8046025383389569, "pass@8": 0.8527253835500371, "pass@16": 0.8880693164814899, "pass@32": 0.915272845611458, "pass@64": 0.9357913648933431, "pass@128": 0.949 } }, "per_op_accuracy": { "10": 0.941265625, "2": 1.0, "3": 0.998203125, "4": 0.99871875, "5": 0.9939296875, "6": 0.9972734375, "7": 0.9960859375, "8": 0.991609375, "9": 0.99265625, "11": 0.9624921875, "12": 0.951375, "13": 0.927671875, "14": 0.9342578125, "15": 0.930546875, "16": 0.866078125, "17": 0.8524375, "18": 0.8109296875, "19": 0.6708515625, "20": 0.6338125 }, "per_op_avg_response_len": { "10": 281.453609375, "2": 115.61171875, "3": 145.2217734375, "4": 160.020953125, "5": 173.8887578125, "6": 199.223328125, "7": 224.244828125, "8": 243.6493359375, "9": 259.7151640625, "11": 300.8788359375, "12": 313.412328125, "13": 334.19246875, "14": 361.4050703125, "15": 381.8040859375, "16": 392.90459375, "17": 414.05021875, "18": 432.1644609375, "19": 434.3923984375, "20": 453.55165625 }, "per_op_avg_loss": {}, "per_op_length": { "10": 281.453609375, "2": 115.61171875, "3": 145.2217734375, "4": 160.020953125, "5": 173.8887578125, "6": 199.223328125, "7": 224.244828125, "8": 243.6493359375, "9": 259.7151640625, "11": 300.8788359375, "12": 313.412328125, "13": 334.19246875, "14": 361.4050703125, "15": 381.8040859375, "16": 392.90459375, "17": 414.05021875, "18": 432.1644609375, "19": 434.3923984375, "20": 453.55165625 }, "per_op_loss": {}, "per_template": { "crazy_zootopia": { "count": 809728, "correct": 742004, "answer_accuracy": 0.9163620376225102, "avg_response_len": 282.5668360239488, "resp_tokens_sum": 228802279, "pass_at_k": { "pass@1": 0.9163620376225102, "pass@2": 0.9432990699238839, "pass@4": 0.9594926690468993, "pass@8": 0.9702768212202704, "pass@16": 0.978067131021929, "pass@32": 0.9837285212462428, "pass@64": 0.9878001481817852, "pass@128": 0.9908314890926335 }, "per_op_pass_at_k": { "10": { "pass@1": 0.940058075221239, "pass@2": 0.952202445241911, "pass@4": 0.9639851915634436, "pass@8": 0.9747452943211968, "pass@16": 0.9837117746740343, "pass@32": 0.9903082598333327, "pass@64": 0.9945723951890973, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999641628440367, "pass@2": 0.9999793066291025, "pass@4": 0.999999905401733, "pass@8": 0.9999999999996472, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9995361328125, "pass@2": 0.9999784694881889, "pass@4": 0.9999999370195912, "pass@8": 0.9999999999999016, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.991170634920635, "pass@2": 0.9935695538057743, "pass@4": 0.9953833833270841, "pass@8": 0.996783521967192, "pass@16": 0.9975425302123753, "pass@32": 0.9982188768739435, "pass@64": 0.9992125984251853, "pass@128": 1.0 }, "6": { "pass@1": 0.9999298278443114, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9981046597633136, "pass@2": 0.9996745853328983, "pass@4": 0.9999718887764769, "pass@8": 0.9999998022755866, "pass@16": 0.9999999999967328, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9924879807692307, "pass@2": 0.995214390268524, "pass@4": 0.9969837258924363, "pass@8": 0.9983251613066434, "pass@16": 0.9993038302603201, "pass@32": 0.9998838431738969, "pass@64": 0.9999990205966608, "pass@128": 1.0 }, "9": { "pass@1": 0.9905809859154929, "pass@2": 0.992608059775979, "pass@4": 0.9937131769620348, "pass@8": 0.9952330962332059, "pass@16": 0.9970819743714158, "pass@32": 0.9986530587245823, "pass@64": 0.9996542900027615, "pass@128": 1.0 }, "11": { "pass@1": 0.9567767351274787, "pass@2": 0.9729954579420491, "pass@4": 0.9837077224412106, "pass@8": 0.9915053244533025, "pass@16": 0.9967604348473688, "pass@32": 0.9993037377604537, "pass@64": 0.999959962274875, "pass@128": 1.0 }, "12": { "pass@1": 0.9463497528248588, "pass@2": 0.9650545370123225, "pass@4": 0.9766829093397222, "pass@8": 0.9848453467690281, "pass@16": 0.9914138441913528, "pass@32": 0.9958525342115812, "pass@64": 0.9982239261012225, "pass@128": 1.0 }, "13": { "pass@1": 0.9212627351097179, "pass@2": 0.953259601856194, "pass@4": 0.974031526035734, "pass@8": 0.9872172251386195, "pass@16": 0.9946975877158775, "pass@32": 0.9979212628228779, "pass@64": 0.9992200434190286, "pass@128": 1.0 }, "14": { "pass@1": 0.9226781542056075, "pass@2": 0.9488315457355219, "pass@4": 0.966660107907072, "pass@8": 0.9807327520370567, "pass@16": 0.9900033030501925, "pass@32": 0.994705328348, "pass@64": 0.997677205060592, "pass@128": 1.0 }, "15": { "pass@1": 0.9338872832369942, "pass@2": 0.9658665826544082, "pass@4": 0.9800264382595235, "pass@8": 0.9852710411507393, "pass@16": 0.9867086223947849, "pass@32": 0.9875391269726735, "pass@64": 0.9882671703608444, "pass@128": 0.9884393063583815 }, "16": { "pass@1": 0.8443296370967742, "pass@2": 0.8949314992379985, "pass@4": 0.9236545057875828, "pass@8": 0.9436194889710628, "pass@16": 0.9597137566119098, "pass@32": 0.9731177144426382, "pass@64": 0.9836290640887158, "pass@128": 0.9935483870967742 }, "17": { "pass@1": 0.8775063222543352, "pass@2": 0.9258678343043105, "pass@4": 0.9494584514544068, "pass@8": 0.9630582535369274, "pass@16": 0.9726016060789089, "pass@32": 0.9786961371782323, "pass@64": 0.9822631761264358, "pass@128": 0.9855491329479769 }, "18": { "pass@1": 0.8074330357142857, "pass@2": 0.8726863048368955, "pass@4": 0.9079920633135147, "pass@8": 0.9290310338171512, "pass@16": 0.9431022516433624, "pass@32": 0.9531360758782476, "pass@64": 0.9606210100169015, "pass@128": 0.9657142857142857 }, "19": { "pass@1": 0.6631240615615616, "pass@2": 0.7647959228913951, "pass@4": 0.8269798164306035, "pass@8": 0.8654212985928951, "pass@16": 0.8938579593227545, "pass@32": 0.9182400396910518, "pass@64": 0.9393446013378842, "pass@128": 0.9579579579579579 }, "20": { "pass@1": 0.6333521021021021, "pass@2": 0.7295261550920993, "pass@4": 0.7942091112484806, "pass@8": 0.8420309365275893, "pass@16": 0.8790149220263148, "pass@32": 0.9073538190959931, "pass@64": 0.9277009935323411, "pass@128": 0.93993993993994 } } }, "teachers_in_school": { "count": 821120, "correct": 757653, "answer_accuracy": 0.9227067907248636, "avg_response_len": 292.3765210931411, "resp_tokens_sum": 240076209, "pass_at_k": { "pass@1": 0.9227067907248636, "pass@2": 0.9491983678141017, "pass@4": 0.9648446673072715, "pass@8": 0.974732532480983, "pass@16": 0.9813596376382586, "pass@32": 0.9859942113928409, "pass@64": 0.9894281143827801, "pass@128": 0.9918939984411536 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9300469708588958, "pass@2": 0.9466099916670692, "pass@4": 0.9608256378803874, "pass@8": 0.9735479075977559, "pass@16": 0.9852696934862603, "pass@32": 0.9940972426092773, "pass@64": 0.9987733649475956, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9961647727272728, "pass@2": 0.9972918157957528, "pass@4": 0.9983365499767074, "pass@8": 0.9991928917832018, "pass@16": 0.9998063298472625, "pass@32": 0.9999922499316101, "pass@64": 0.9999999987964725, "pass@128": 1.0 }, "4": { "pass@1": 0.997965976331361, "pass@2": 0.9991187596095608, "pass@4": 0.9997857705841503, "pass@8": 0.9999865024173374, "pass@16": 0.9999999643806659, "pass@32": 0.9999999999999731, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.99375, "pass@2": 0.9967833504507589, "pass@4": 0.9986635988436225, "pass@8": 0.9996831882834758, "pass@16": 0.9999789012150662, "pass@32": 0.9999999417280491, "pass@64": 0.9999999999999963, "pass@128": 1.0 }, "6": { "pass@1": 0.9955676934097422, "pass@2": 0.9973236243034089, "pass@4": 0.9983310997300121, "pass@8": 0.9991142096348365, "pass@16": 0.9997496114237934, "pass@32": 0.9999854792844564, "pass@64": 0.999999993573057, "pass@128": 1.0 }, "7": { "pass@1": 0.9948067196531792, "pass@2": 0.9959908089071959, "pass@4": 0.9969885587820309, "pass@8": 0.9979235613025731, "pass@16": 0.9987206336864413, "pass@32": 0.9995059858135572, "pass@64": 0.9999600471131015, "pass@128": 1.0 }, "8": { "pass@1": 0.9922975352112676, "pass@2": 0.9943263419097261, "pass@4": 0.99571091852955, "pass@8": 0.9969972148105722, "pass@16": 0.99814919003857, "pass@32": 0.9991126715814442, "pass@64": 0.9998322253384277, "pass@128": 1.0 }, "9": { "pass@1": 0.995158121257485, "pass@2": 0.9969157403696542, "pass@4": 0.997358424601865, "pass@8": 0.9976945425380946, "pass@16": 0.9982569113741001, "pass@32": 0.9990676801627342, "pass@64": 0.9998216794756942, "pass@128": 1.0 }, "11": { "pass@1": 0.97265625, "pass@2": 0.9817142586092406, "pass@4": 0.9872070927128084, "pass@8": 0.9907918466392928, "pass@16": 0.9932876981463241, "pass@32": 0.9946033623345881, "pass@64": 0.9954810039531555, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9539086990595611, "pass@2": 0.9742443783476912, "pass@4": 0.9873180110910899, "pass@8": 0.9939946254019711, "pass@16": 0.9967163926023948, "pass@32": 0.9983956273931865, "pass@64": 0.9996077018477316, "pass@128": 1.0 }, "13": { "pass@1": 0.9435836834733894, "pass@2": 0.9670878686120119, "pass@4": 0.9795068925207874, "pass@8": 0.9867661154939795, "pass@16": 0.9912037146425846, "pass@32": 0.9943074150090554, "pass@64": 0.9971910964276763, "pass@128": 1.0 }, "14": { "pass@1": 0.9339820906432749, "pass@2": 0.961808433485288, "pass@4": 0.9791213154276766, "pass@8": 0.9903633423119838, "pass@16": 0.9965715597469695, "pass@32": 0.998978060543375, "pass@64": 0.9998257129593244, "pass@128": 1.0 }, "15": { "pass@1": 0.936032196969697, "pass@2": 0.9698241022429014, "pass@4": 0.986383716808126, "pass@8": 0.993887020431712, "pass@16": 0.997341288437486, "pass@32": 0.9992550901802849, "pass@64": 0.999953724719699, "pass@128": 1.0 }, "16": { "pass@1": 0.8875979344729344, "pass@2": 0.9302362344931245, "pass@4": 0.9518918415112638, "pass@8": 0.9653093003879017, "pass@16": 0.97478416069728, "pass@32": 0.9809651521660178, "pass@64": 0.986492391819359, "pass@128": 0.9914529914529915 }, "17": { "pass@1": 0.8626746894409938, "pass@2": 0.9124707322590117, "pass@4": 0.9368205890024611, "pass@8": 0.9508498521267209, "pass@16": 0.960456726487503, "pass@32": 0.9678057412486193, "pass@64": 0.9734389189717876, "pass@128": 0.9782608695652174 }, "18": { "pass@1": 0.8121553308823529, "pass@2": 0.8809753502779064, "pass@4": 0.9216372758552233, "pass@8": 0.9462221099752758, "pass@16": 0.9606949776103674, "pass@32": 0.9685754572068952, "pass@64": 0.9731564892213497, "pass@128": 0.9764705882352941 }, "19": { "pass@1": 0.66943359375, "pass@2": 0.7666238985751784, "pass@4": 0.8258763982627161, "pass@8": 0.8655574931646675, "pass@16": 0.8941625232220163, "pass@32": 0.9161714667742615, "pass@64": 0.9350029611318369, "pass@128": 0.9523809523809523 }, "20": { "pass@1": 0.6556652320359282, "pass@2": 0.7598498868404915, "pass@4": 0.8262946623625637, "pass@8": 0.8690020135890967, "pass@16": 0.8983220264807161, "pass@32": 0.9212925306509724, "pass@64": 0.9391524696479382, "pass@128": 0.9491017964071856 } } }, "movie_festival_awards": { "count": 801152, "correct": 733968, "answer_accuracy": 0.9161407573094743, "avg_response_len": 312.93695453546894, "resp_tokens_sum": 250710067, "pass_at_k": { "pass@1": 0.9161407573094743, "pass@2": 0.9444789070667229, "pass@4": 0.9620470564981104, "pass@8": 0.9734481996656786, "pass@16": 0.9808410217800196, "pass@32": 0.985989950434011, "pass@64": 0.9899999651668862, "pass@128": 0.993129892954146 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9534048507462687, "pass@2": 0.970000514161476, "pass@4": 0.9828655804218498, "pass@8": 0.9913297716090343, "pass@16": 0.9958306599914305, "pass@32": 0.9979223480778879, "pass@64": 0.9992399818731803, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9987928206997084, "pass@2": 0.99955952572255, "pass@4": 0.9999370614387487, "pass@8": 0.9999989051962135, "pass@16": 0.9999999998462147, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986979166666666, "pass@2": 0.9994841310954552, "pass@4": 0.9999133192232549, "pass@8": 0.9999978720766682, "pass@16": 0.9999999993392, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9966681985294118, "pass@2": 0.9992585543075498, "pass@4": 0.999930265610181, "pass@8": 0.9999993599537156, "pass@16": 0.9999999999763772, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.996352523659306, "pass@2": 0.9972878735189645, "pass@4": 0.9978065682444268, "pass@8": 0.9984953578892186, "pass@16": 0.9993197040358152, "pass@32": 0.9998856774168334, "pass@64": 0.999999036044663, "pass@128": 1.0 }, "7": { "pass@1": 0.9953273338607594, "pass@2": 0.9967836607943786, "pass@4": 0.9975761380302146, "pass@8": 0.998262935853195, "pass@16": 0.9990878595282221, "pass@32": 0.9997847637219769, "pass@64": 0.9999954280485364, "pass@128": 1.0 }, "8": { "pass@1": 0.9900525525525525, "pass@2": 0.9931176254995152, "pass@4": 0.9950761472946511, "pass@8": 0.996932633631575, "pass@16": 0.9985345359444987, "pass@32": 0.9996000242336316, "pass@64": 0.9999802487805222, "pass@128": 1.0 }, "9": { "pass@1": 0.9923382234726688, "pass@2": 0.9954557466389852, "pass@4": 0.9969244255440737, "pass@8": 0.9981047741525653, "pass@16": 0.9990691147291293, "pass@32": 0.9997813019589095, "pass@64": 0.9999953545444935, "pass@128": 1.0 }, "11": { "pass@1": 0.9581845238095238, "pass@2": 0.973211942257218, "pass@4": 0.9851733461888689, "pass@8": 0.9935984850997558, "pass@16": 0.9981004697081209, "pass@32": 0.9997117617660194, "pass@64": 0.9999952151978636, "pass@128": 1.0 }, "12": { "pass@1": 0.9543434633027523, "pass@2": 0.9737747266970067, "pass@4": 0.9860677859648275, "pass@8": 0.994249571393947, "pass@16": 0.9986259898728691, "pass@32": 0.9998862026515789, "pass@64": 0.9999995629648154, "pass@128": 1.0 }, "13": { "pass@1": 0.9164496527777778, "pass@2": 0.9467269824778841, "pass@4": 0.966515018955963, "pass@8": 0.9796284799576312, "pass@16": 0.9873678160183381, "pass@32": 0.9923304170452849, "pass@64": 0.9960751154460078, "pass@128": 1.0 }, "14": { "pass@1": 0.9455675074183977, "pass@2": 0.9688518569359099, "pass@4": 0.9830591241005852, "pass@8": 0.9918079912361211, "pass@16": 0.9964965731596999, "pass@32": 0.9989974984823478, "pass@64": 0.999910515524606, "pass@128": 1.0 }, "15": { "pass@1": 0.9213927469135802, "pass@2": 0.9536509216729855, "pass@4": 0.9729866107593029, "pass@8": 0.9846111586848604, "pass@16": 0.9907910506066029, "pass@32": 0.9946077911424446, "pass@64": 0.9975593713245301, "pass@128": 1.0 }, "16": { "pass@1": 0.8636845501474927, "pass@2": 0.9176269656005392, "pass@4": 0.9461433612280757, "pass@8": 0.9632940350686785, "pass@16": 0.9731921968948374, "pass@32": 0.9788198865927932, "pass@64": 0.9817049567519268, "pass@128": 0.9823008849557522 }, "17": { "pass@1": 0.816382718373494, "pass@2": 0.8697396623897163, "pass@4": 0.8989555943120052, "pass@8": 0.918514496266187, "pass@16": 0.9335748802757035, "pass@32": 0.9474461636352892, "pass@64": 0.9617706386333493, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8135332661290322, "pass@2": 0.8886441929133857, "pass@4": 0.9286830575613528, "pass@8": 0.9493191452530526, "pass@16": 0.9612486799216202, "pass@32": 0.9699809772790603, "pass@64": 0.9777500114141688, "pass@128": 0.9838709677419355 }, "19": { "pass@1": 0.6800651435045317, "pass@2": 0.7825676636058709, "pass@4": 0.8478114498338757, "pass@8": 0.890404706092961, "pass@16": 0.9181360078721577, "pass@32": 0.9381630090208142, "pass@64": 0.9558007487757464, "pass@128": 0.9697885196374623 }, "20": { "pass@1": 0.6123545420420421, "pass@2": 0.7181795476578936, "pass@4": 0.7932386998922434, "pass@8": 0.8470943217645397, "pass@16": 0.8868402120185244, "pass@32": 0.9171541099551569, "pass@64": 0.9405105380920781, "pass@128": 0.9579579579579579 } } } } } }