[ { "checkpoint": "checkpoint-85656", "id_acc": 0.49327430555555557, "ood_acc": 0.12814140625, "total_acc": 0.30109909539473684, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.13701649305557, "ood_avg_resp_len": 306.88617109375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 239685, "answer_accuracy": 0.2960068072241543, "avg_response_len": 245.8270233955106, "resp_tokens_sum": 199053024, "pass_at_k": { "pass@1": 0.2960068072241543, "pass@2": 0.38263211474455283, "pass@4": 0.480826212706066, "pass@8": 0.5767558778808439, "pass@16": 0.6568090850720517, "pass@32": 0.7180689672909597, "pass@64": 0.7652942721942446, "pass@128": 0.8039835599114764 }, "per_op_pass_at_k": { "10": { "pass@1": 0.19035766961651918, "pass@2": 0.30144277111931805, "pass@4": 0.43816523764175513, "pass@8": 0.5809281738806453, "pass@16": 0.7052121639952813, "pass@32": 0.8019947591187581, "pass@64": 0.8754555701737227, "pass@128": 0.9233038348082596 }, "2": { "pass@1": 0.9800915109034268, "pass@2": 0.9870491144798491, "pass@4": 0.990911137860104, "pass@8": 0.9931530593847345, "pass@16": 0.9945585526843174, "pass@32": 0.9955696135172313, "pass@64": 0.9965045227606399, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.9717842125382263, "pass@2": 0.9858160111006766, "pass@4": 0.991448598168348, "pass@8": 0.994291079132131, "pass@16": 0.995887473442639, "pass@32": 0.9967283718547971, "pass@64": 0.9969374773681062, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.6986328125, "pass@2": 0.7886968657726381, "pass@4": 0.8542420943475817, "pass@8": 0.9004456167198726, "pass@16": 0.9312778575005979, "pass@32": 0.9517112440915444, "pass@64": 0.9667046571410877, "pass@128": 0.978125 }, "5": { "pass@1": 0.33509424603174603, "pass@2": 0.41271325459317576, "pass@4": 0.48771068527148376, "pass@8": 0.5572081248784712, "pass@16": 0.6188679542444174, "pass@32": 0.6719682822822658, "pass@64": 0.7166542430331456, "pass@128": 0.7523809523809524 }, "6": { "pass@1": 0.46206025449101795, "pass@2": 0.6031718551086802, "pass@4": 0.7298749273107324, "pass@8": 0.8273623466954299, "pass@16": 0.8904114372774126, "pass@32": 0.9291838638462578, "pass@64": 0.9545298379466033, "pass@128": 0.9730538922155688 }, "7": { "pass@1": 0.30755362426035504, "pass@2": 0.44404614918697294, "pass@4": 0.5866653091255597, "pass@8": 0.7100621776567952, "pass@16": 0.8009828251974904, "pass@32": 0.8641776511072881, "pass@64": 0.9077531903073257, "pass@128": 0.9408284023668639 }, "8": { "pass@1": 0.25122696314102566, "pass@2": 0.38249427430345234, "pass@4": 0.5251050217280533, "pass@8": 0.6536996226393975, "pass@16": 0.7536627127551332, "pass@32": 0.826661161406808, "pass@64": 0.8762717660769928, "pass@128": 0.9038461538461539 }, "9": { "pass@1": 0.2078125, "pass@2": 0.3245858517245202, "pass@4": 0.4576318187339257, "pass@8": 0.5851793502978136, "pass@16": 0.6946462264181785, "pass@32": 0.7833343799733238, "pass@64": 0.8508485859197652, "pass@128": 0.8985915492957747 }, "11": { "pass@1": 0.18229904390934845, "pass@2": 0.2881442946844819, "pass@4": 0.41277183172570875, "pass@8": 0.5372213699972903, "pass@16": 0.6464648060956772, "pass@32": 0.7364996012096983, "pass@64": 0.8090871748200775, "pass@128": 0.8725212464589235 }, "12": { "pass@1": 0.167041843220339, "pass@2": 0.267159418234797, "pass@4": 0.38463744918537746, "pass@8": 0.5005959279433829, "pass@16": 0.6027360469209624, "pass@32": 0.6906821588303728, "pass@64": 0.7683561968655983, "pass@128": 0.8389830508474576 }, "13": { "pass@1": 0.14863342476489028, "pass@2": 0.24290428381013504, "pass@4": 0.3593948659043012, "pass@8": 0.4767867703415537, "pass@16": 0.5776974942645328, "pass@32": 0.6625444544056447, "pass@64": 0.7355174465136388, "pass@128": 0.7962382445141066 }, "14": { "pass@1": 0.13086545950155765, "pass@2": 0.21606400397380243, "pass@4": 0.3265159209070825, "pass@8": 0.44258768544823535, "pass@16": 0.5412763209024235, "pass@32": 0.6182166903432107, "pass@64": 0.6805139060593539, "pass@128": 0.735202492211838 }, "15": { "pass@1": 0.11870032514450866, "pass@2": 0.19539072811888392, "pass@4": 0.2952038571948737, "pass@8": 0.4034823819223066, "pass@16": 0.4996538273439035, "pass@32": 0.577105706738467, "pass@64": 0.6430681278184355, "pass@128": 0.7052023121387283 }, "16": { "pass@1": 0.12497479838709677, "pass@2": 0.20588566802133618, "pass@4": 0.30794752833315187, "pass@8": 0.41409910598717037, "pass@16": 0.503439116374473, "pass@32": 0.5680757189322889, "pass@64": 0.6185146367153228, "pass@128": 0.667741935483871 }, "17": { "pass@1": 0.1048139450867052, "pass@2": 0.1729335686814438, "pass@4": 0.26166974377480257, "pass@8": 0.3569233003143246, "pass@16": 0.43881371763626015, "pass@32": 0.5017114929543303, "pass@64": 0.5540504002245086, "pass@128": 0.6011560693641619 }, "18": { "pass@1": 0.11404017857142858, "pass@2": 0.1839486079865017, "pass@4": 0.27364456023354217, "pass@8": 0.3720505437784955, "pass@16": 0.45863649753082086, "pass@32": 0.5215180865558517, "pass@64": 0.5696621746276215, "pass@128": 0.62 }, "19": { "pass@1": 0.10076482732732733, "pass@2": 0.1690570984370197, "pass@4": 0.2604623833619896, "pass@8": 0.3599502461168844, "pass@16": 0.44260688970777834, "pass@32": 0.4944624925735973, "pass@64": 0.521548379452954, "pass@128": 0.5405405405405406 }, "20": { "pass@1": 0.10095251501501501, "pass@2": 0.16684992078692867, "pass@4": 0.2509089626747106, "pass@8": 0.3406114996432136, "pass@16": 0.42006647369881606, "pass@32": 0.47932235146072466, "pass@64": 0.5187436798897297, "pass@128": 0.5465465465465466 } } }, "teachers_in_school": { "count": 821120, "correct": 248622, "answer_accuracy": 0.3027840023382697, "avg_response_len": 254.0808444563523, "resp_tokens_sum": 208630863, "pass_at_k": { "pass@1": 0.3027840023382697, "pass@2": 0.39144283820524556, "pass@4": 0.4912621577164558, "pass@8": 0.5883824580273733, "pass@16": 0.6688317187866314, "pass@32": 0.7301793076629046, "pass@64": 0.7783423771084799, "pass@128": 0.8188620420888543 }, "per_op_pass_at_k": { "10": { "pass@1": 0.1796875, "pass@2": 0.28581424206560085, "pass@4": 0.41622926026117907, "pass@8": 0.5547185574304757, "pass@16": 0.6839554801528199, "pass@32": 0.791310676081314, "pass@64": 0.8702003501063394, "pass@128": 0.9171779141104295 }, "2": { "pass@1": 0.9739504179331308, "pass@2": 0.9832606113730465, "pass@4": 0.9877440161355209, "pass@8": 0.9903351488073417, "pass@16": 0.9921767708249385, "pass@32": 0.9934576683623969, "pass@64": 0.9939005764597241, "pass@128": 0.993920972644377 }, "3": { "pass@1": 0.9731060606060606, "pass@2": 0.9891795663326179, "pass@4": 0.9937926068900476, "pass@8": 0.9955231703667864, "pass@16": 0.9969587575218833, "pass@32": 0.9984506286504, "pass@64": 0.9996207784594809, "pass@128": 1.0 }, "4": { "pass@1": 0.7112610946745562, "pass@2": 0.8105136601360479, "pass@4": 0.8784895620480868, "pass@8": 0.9216631817833484, "pass@16": 0.9464260283198788, "pass@32": 0.9590859028287348, "pass@64": 0.9663197612118946, "pass@128": 0.9704142011834319 }, "5": { "pass@1": 0.3811141304347826, "pass@2": 0.4672704125299555, "pass@4": 0.547470422990604, "pass@8": 0.6177559162397293, "pass@16": 0.6765391597013457, "pass@32": 0.7272516871961043, "pass@64": 0.7712716097104394, "pass@128": 0.8115942028985508 }, "6": { "pass@1": 0.41883058739255014, "pass@2": 0.5677878866502717, "pass@4": 0.7127793116118358, "pass@8": 0.8278492564390045, "pass@16": 0.8988014599895945, "pass@32": 0.9373003380525965, "pass@64": 0.9604309989332116, "pass@128": 0.9742120343839542 }, "7": { "pass@1": 0.33487626445086704, "pass@2": 0.46564541042738106, "pass@4": 0.5980823447466463, "pass@8": 0.714922667139698, "pass@16": 0.8033677613399027, "pass@32": 0.8641866922325745, "pass@64": 0.9019377879132182, "pass@128": 0.9219653179190751 }, "8": { "pass@1": 0.28347271126760565, "pass@2": 0.4233832621714537, "pass@4": 0.5744736294406855, "pass@8": 0.7088275207620468, "pass@16": 0.8087186748834445, "pass@32": 0.8784898814053248, "pass@64": 0.9269338685585622, "pass@128": 0.9605633802816902 }, "9": { "pass@1": 0.21727638473053892, "pass@2": 0.33913595053986484, "pass@4": 0.48270442579408074, "pass@8": 0.6238536005858858, "pass@16": 0.7419666380525494, "pass@32": 0.8299157749956388, "pass@64": 0.891739006586996, "pass@128": 0.9341317365269461 }, "11": { "pass@1": 0.1560617469879518, "pass@2": 0.2487022400626127, "pass@4": 0.363962768961109, "pass@8": 0.4871624174058488, "pass@16": 0.603968486823371, "pass@32": 0.7099225827109938, "pass@64": 0.7991877595200533, "pass@128": 0.8644578313253012 }, "12": { "pass@1": 0.18140184169278997, "pass@2": 0.28670233937254724, "pass@4": 0.41065066504226133, "pass@8": 0.5328363958358436, "pass@16": 0.6391938939640938, "pass@32": 0.7286125927091673, "pass@64": 0.8047119385141658, "pass@128": 0.8714733542319749 }, "13": { "pass@1": 0.14758403361344538, "pass@2": 0.2368842221928142, "pass@4": 0.34356088472134266, "pass@8": 0.44992350437981904, "pass@16": 0.5427658276100494, "pass@32": 0.6231971069674507, "pass@64": 0.6981882477135789, "pass@128": 0.773109243697479 }, "14": { "pass@1": 0.1457419590643275, "pass@2": 0.2322633190588018, "pass@4": 0.33790162002775964, "pass@8": 0.44425079336957135, "pass@16": 0.5325720105945908, "pass@32": 0.6005776223976965, "pass@64": 0.662597142298673, "pass@128": 0.7309941520467836 }, "15": { "pass@1": 0.11176609848484849, "pass@2": 0.1877408434741111, "pass@4": 0.28652309569258383, "pass@8": 0.39240725982235936, "pass@16": 0.4878854499058535, "pass@32": 0.5707635144848856, "pass@64": 0.6501004572511563, "pass@128": 0.7272727272727273 }, "16": { "pass@1": 0.1274928774928775, "pass@2": 0.20607284866635253, "pass@4": 0.302366788019019, "pass@8": 0.39909323245321887, "pass@16": 0.479867770840156, "pass@32": 0.5436816319849745, "pass@64": 0.5996621177657357, "pass@128": 0.6552706552706553 }, "17": { "pass@1": 0.1155861801242236, "pass@2": 0.19869586614173215, "pass@4": 0.31046421002498914, "pass@8": 0.43139028333994395, "pass@16": 0.5326025992065495, "pass@32": 0.6042294785582262, "pass@64": 0.6602624716039003, "pass@128": 0.717391304347826 }, "18": { "pass@1": 0.11557904411764706, "pass@2": 0.19096840261695236, "pass@4": 0.28819167300778564, "pass@8": 0.39090138579917816, "pass@16": 0.4763199553646875, "pass@32": 0.5355086068492452, "pass@64": 0.5739343318595364, "pass@128": 0.6029411764705882 }, "19": { "pass@1": 0.10209728422619048, "pass@2": 0.1712254229940008, "pass@4": 0.26240663471753534, "pass@8": 0.3616555082558499, "pass@16": 0.44495512112719054, "pass@32": 0.5003037881586762, "pass@64": 0.5392674835648679, "pass@128": 0.5773809523809523 }, "20": { "pass@1": 0.09110684880239521, "pass@2": 0.15343316811259372, "pass@4": 0.23748076793544526, "pass@8": 0.33288966241110335, "pass@16": 0.4179387610759886, "pass@32": 0.4779039161554723, "pass@64": 0.5203573952212737, "pass@128": 0.5568862275449101 } } }, "movie_festival_awards": { "count": 801152, "correct": 243966, "answer_accuracy": 0.3045189926505832, "avg_response_len": 267.7847087693721, "resp_tokens_sum": 214536255, "pass_at_k": { "pass@1": 0.3045189926505832, "pass@2": 0.3914387101471566, "pass@4": 0.4888976512891585, "pass@8": 0.5832940609223775, "pass@16": 0.6619421974877205, "pass@32": 0.7214642063143812, "pass@64": 0.7664130005582344, "pass@128": 0.8017255152580285 }, "per_op_pass_at_k": { "10": { "pass@1": 0.20205223880597015, "pass@2": 0.3135319955341403, "pass@4": 0.4415771505240951, "pass@8": 0.569811918955316, "pass@16": 0.6850570876181581, "pass@32": 0.7808957342914687, "pass@64": 0.8544946545097453, "pass@128": 0.9044776119402985 }, "2": { "pass@1": 0.9875892857142857, "pass@2": 0.9917403683914513, "pass@4": 0.9935481662113664, "pass@8": 0.9943054704732391, "pass@16": 0.9946362839316639, "pass@32": 0.9949999953558449, "pass@64": 0.9957142857142857, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9700482871720116, "pass@2": 0.9877388174054772, "pass@4": 0.994211654756712, "pass@8": 0.9968066288226595, "pass@16": 0.9979756293527906, "pass@32": 0.9987791216280217, "pass@64": 0.9996441771291553, "pass@128": 1.0 }, "4": { "pass@1": 0.7102065058479532, "pass@2": 0.8100167063820963, "pass@4": 0.8802413790929641, "pass@8": 0.9272992602475084, "pass@16": 0.9585762077324128, "pass@32": 0.9767911949187461, "pass@64": 0.9843987458071987, "pass@128": 0.9853801169590644 }, "5": { "pass@1": 0.41737132352941175, "pass@2": 0.513387274201019, "pass@4": 0.6009593699684594, "pass@8": 0.6746747650464919, "pass@16": 0.7341430684121006, "pass@32": 0.7796575711960942, "pass@64": 0.814747638086809, "pass@128": 0.8411764705882353 }, "6": { "pass@1": 0.3975749211356467, "pass@2": 0.5503043574107656, "pass@4": 0.6984590626881418, "pass@8": 0.8139611823607342, "pass@16": 0.887723354151534, "pass@32": 0.9308254188372326, "pass@64": 0.9556506522537759, "pass@128": 0.9684542586750788 }, "7": { "pass@1": 0.3171726661392405, "pass@2": 0.46342187967208204, "pass@4": 0.6122294229637435, "pass@8": 0.7384669154900192, "pass@16": 0.8279188406130379, "pass@32": 0.8852386895678486, "pass@64": 0.9234833724808077, "pass@128": 0.9493670886075949 }, "8": { "pass@1": 0.2432667042042042, "pass@2": 0.3739248598992691, "pass@4": 0.5204787287287282, "pass@8": 0.6566691861840236, "pass@16": 0.7674609415621451, "pass@32": 0.8523845738554838, "pass@64": 0.9119541812947833, "pass@128": 0.9519519519519519 }, "9": { "pass@1": 0.18521402733118972, "pass@2": 0.28941596830138966, "pass@4": 0.41672779638477636, "pass@8": 0.5506735350456974, "pass@16": 0.6746045152270984, "pass@32": 0.7816134842691489, "pass@64": 0.8652622165514577, "pass@128": 0.9163987138263665 }, "11": { "pass@1": 0.17552083333333332, "pass@2": 0.2797076146731657, "pass@4": 0.40224519435070627, "pass@8": 0.5271396689310396, "pass@16": 0.644746749599154, "pass@32": 0.7499209088304015, "pass@64": 0.8377807881434293, "pass@128": 0.9047619047619048 }, "12": { "pass@1": 0.15553325688073394, "pass@2": 0.24843031375665203, "pass@4": 0.3597259003748386, "pass@8": 0.47338739712545613, "pass@16": 0.576015274972535, "pass@32": 0.664281855201088, "pass@64": 0.7428282365695322, "pass@128": 0.8134556574923547 }, "13": { "pass@1": 0.14749710648148148, "pass@2": 0.2358992235345583, "pass@4": 0.3428141392626847, "pass@8": 0.4519424575970455, "pass@16": 0.5471952037565342, "pass@32": 0.6258993050569552, "pass@64": 0.6955199471953205, "pass@128": 0.7623456790123457 }, "14": { "pass@1": 0.13357752225519287, "pass@2": 0.21566179992523188, "pass@4": 0.3157897633901105, "pass@8": 0.41891290583411983, "pass@16": 0.510946115208545, "pass@32": 0.5898814297305477, "pass@64": 0.6612483786173892, "pass@128": 0.7240356083086054 }, "15": { "pass@1": 0.10616801697530864, "pass@2": 0.18171486159716133, "pass@4": 0.2844029273539879, "pass@8": 0.3966455323521176, "pass@16": 0.4939536315132476, "pass@32": 0.5693545900367545, "pass@64": 0.6337045083992685, "pass@128": 0.691358024691358 }, "16": { "pass@1": 0.12898691002949852, "pass@2": 0.21023106694074747, "pass@4": 0.3090409769022235, "pass@8": 0.40610456188689503, "pass@16": 0.4821112975879393, "pass@32": 0.5350102394509227, "pass@64": 0.57746091806195, "pass@128": 0.6224188790560472 }, "17": { "pass@1": 0.1130929969879518, "pass@2": 0.18889633336495587, "pass@4": 0.28544219764321627, "pass@8": 0.38457376013687594, "pass@16": 0.46493667993499693, "pass@32": 0.5194219776626258, "pass@64": 0.5543655991302966, "pass@128": 0.5813253012048193 }, "18": { "pass@1": 0.10257056451612903, "pass@2": 0.16746570993141993, "pass@4": 0.250047667428668, "pass@8": 0.33849332443951263, "pass@16": 0.41364696069505363, "pass@32": 0.4656988345709459, "pass@64": 0.5029973247060224, "pass@128": 0.5387096774193548 }, "19": { "pass@1": 0.0993202416918429, "pass@2": 0.16870480172229227, "pass@4": 0.2604445992589294, "pass@8": 0.3578538662514504, "pass@16": 0.4353394819087087, "pass@32": 0.47947822088860853, "pass@64": 0.5000096375040654, "pass@128": 0.513595166163142 }, "20": { "pass@1": 0.09295232732732733, "pass@2": 0.15922049017521456, "pass@4": 0.24903409427425166, "pass@8": 0.3498747830497944, "pass@16": 0.43915606624257963, "pass@32": 0.4983363412761255, "pass@64": 0.5302439838985393, "pass@128": 0.5525525525525525 } } } } } ]