[ { "checkpoint": "checkpoint-39470", "id_acc": 0.9900894097222223, "ood_acc": 0.87708359375, "total_acc": 0.9306126644736842, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.35493229166667, "ood_avg_resp_len": 383.22090859375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 751583, "answer_accuracy": 0.9281919360575404, "avg_response_len": 283.65910527979764, "resp_tokens_sum": 229686720, "pass_at_k": { "pass@1": 0.9281919360575404, "pass@2": 0.9499665173848083, "pass@4": 0.9640744475635182, "pass@8": 0.9737280900028716, "pass@16": 0.980602757648027, "pass@32": 0.9857206319262101, "pass@64": 0.9895329614718181, "pass@128": 0.9920961112867531 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9392514749262537, "pass@2": 0.9528702268134623, "pass@4": 0.9647196697204883, "pass@8": 0.9752039016665607, "pass@16": 0.9841804774412467, "pass@32": 0.9916424699073122, "pass@64": 0.9975593656716509, "pass@128": 1.0 }, "2": { "pass@1": 0.9999756619937694, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9964162844036697, "pass@2": 0.9992663259409089, "pass@4": 0.9999513056854131, "pass@8": 0.9999997214289341, "pass@16": 0.9999999999945326, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99814453125, "pass@2": 0.9990688053641733, "pass@4": 0.9997314116985375, "pass@8": 0.9999793655718723, "pass@16": 0.9999999169698948, "pass@32": 0.9999999999998159, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9982886904761905, "pass@2": 0.9999117297837768, "pass@4": 0.9999994979794193, "pass@8": 0.9999999999856092, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9993450598802395, "pass@2": 0.9999620591494178, "pass@4": 0.9999997176625378, "pass@8": 0.9999999999937114, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9974112426035503, "pass@2": 0.9990299439733494, "pass@4": 0.999743682649876, "pass@8": 0.9999804637850213, "pass@16": 0.9999999213916163, "pass@32": 0.9999999999998258, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9911608573717948, "pass@2": 0.9963473052947709, "pass@4": 0.9984855673088942, "pass@8": 0.9994635469329705, "pass@16": 0.9999228935849399, "pass@32": 0.9999990402568896, "pass@64": 0.9999999999948799, "pass@128": 1.0 }, "9": { "pass@1": 0.9897447183098591, "pass@2": 0.9935465648220028, "pass@4": 0.9950367868629096, "pass@8": 0.9961759013482411, "pass@16": 0.9975146715824619, "pass@32": 0.9990370240116533, "pass@64": 0.9999221200063837, "pass@128": 1.0 }, "11": { "pass@1": 0.9560685198300283, "pass@2": 0.972025147219558, "pass@4": 0.9833676431380923, "pass@8": 0.9902832202553371, "pass@16": 0.9946957855807873, "pass@32": 0.997549645536328, "pass@64": 0.9992147611382037, "pass@128": 1.0 }, "12": { "pass@1": 0.943215925141243, "pass@2": 0.9660565138573776, "pass@4": 0.9814934266691235, "pass@8": 0.9904757233909742, "pass@16": 0.9957168814771249, "pass@32": 0.9984134645336938, "pass@64": 0.9996455904087194, "pass@128": 1.0 }, "13": { "pass@1": 0.9108787225705329, "pass@2": 0.9434656776837063, "pass@4": 0.9667659061230197, "pass@8": 0.9808162550485344, "pass@16": 0.9881944688814267, "pass@32": 0.9930606223263609, "pass@64": 0.9966675055170503, "pass@128": 1.0 }, "14": { "pass@1": 0.9186380451713395, "pass@2": 0.9460888249074009, "pass@4": 0.9653372817299704, "pass@8": 0.9779994019698924, "pass@16": 0.9860626693919856, "pass@32": 0.9919049064530143, "pass@64": 0.9963014598578678, "pass@128": 1.0 }, "15": { "pass@1": 0.9521540823699421, "pass@2": 0.9741484513677119, "pass@4": 0.985112245633313, "pass@8": 0.9908469302823427, "pass@16": 0.9934400862091718, "pass@32": 0.994992084796452, "pass@64": 0.9963530196364373, "pass@128": 0.9971098265895953 }, "16": { "pass@1": 0.8628276209677419, "pass@2": 0.9018546005842007, "pass@4": 0.9287450191709906, "pass@8": 0.9468683936291992, "pass@16": 0.9613419346632679, "pass@32": 0.9736111775442221, "pass@64": 0.9823099408096163, "pass@128": 0.9870967741935484 }, "17": { "pass@1": 0.884709176300578, "pass@2": 0.9211279214646582, "pass@4": 0.9415992674867953, "pass@8": 0.9565759689885837, "pass@16": 0.9692118557186207, "pass@32": 0.9795491000031725, "pass@64": 0.98639472632689, "pass@128": 0.9884393063583815 }, "18": { "pass@1": 0.82296875, "pass@2": 0.8701905230596182, "pass@4": 0.8998161420001068, "pass@8": 0.9205166862174053, "pass@16": 0.9355630508173823, "pass@32": 0.9477589149914283, "pass@64": 0.9590373492455058, "pass@128": 0.9685714285714285 }, "19": { "pass@1": 0.7173188813813813, "pass@2": 0.7844284983802695, "pass@4": 0.8315280263390493, "pass@8": 0.8687886876667537, "pass@16": 0.8990525439832475, "pass@32": 0.9216565068026786, "pass@64": 0.9364336919565242, "pass@128": 0.9459459459459459 }, "20": { "pass@1": 0.7638419669669669, "pass@2": 0.8345680818613895, "pass@4": 0.8795241003095331, "pass@8": 0.9096289064963395, "pass@16": 0.9287254882088332, "pass@32": 0.9412760341681837, "pass@64": 0.9527062784829503, "pass@128": 0.963963963963964 } } }, "teachers_in_school": { "count": 821120, "correct": 766904, "answer_accuracy": 0.933973109898675, "avg_response_len": 292.6585639127046, "resp_tokens_sum": 240307800, "pass_at_k": { "pass@1": 0.933973109898675, "pass@2": 0.9541137397892366, "pass@4": 0.9676575266507484, "pass@8": 0.9769106752250937, "pass@16": 0.9831209304949401, "pass@32": 0.9874264895458542, "pass@64": 0.9906284085148328, "pass@128": 0.9929851909586905 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9263564033742331, "pass@2": 0.943031511158881, "pass@4": 0.9574623036959338, "pass@8": 0.9692292659689711, "pass@16": 0.9785950061939993, "pass@32": 0.985816650956142, "pass@64": 0.9927118784293236, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9957859848484848, "pass@2": 0.9979222590073968, "pass@4": 0.9992018259081249, "pass@8": 0.9998472966011496, "pass@16": 0.999994268588126, "pass@32": 0.9999999968257123, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9996764053254438, "pass@2": 0.9999919920328006, "pass@4": 0.999999990293373, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.996286231884058, "pass@2": 0.9990335786830994, "pass@4": 0.9998567722512948, "pass@8": 0.9999954258079576, "pass@16": 0.9999999965927238, "pass@32": 0.9999999999999999, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9997089899713467, "pass@2": 0.9999936545360197, "pass@4": 0.9999999959711339, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9963872832369942, "pass@2": 0.9973061080515223, "pass@4": 0.9977670850211643, "pass@8": 0.9983013813782712, "pass@16": 0.999038785005553, "pass@32": 0.9997319443095561, "pass@64": 0.9999910524153849, "pass@128": 1.0 }, "8": { "pass@1": 0.9921654929577465, "pass@2": 0.9949418459576357, "pass@4": 0.9969772593566649, "pass@8": 0.998742979098145, "pass@16": 0.9996984788969664, "pass@32": 0.9999800029461254, "pass@64": 0.9999999852572564, "pass@128": 1.0 }, "9": { "pass@1": 0.9932634730538922, "pass@2": 0.9957594653213259, "pass@4": 0.9968023138200539, "pass@8": 0.9971748134078343, "pass@16": 0.9973801780816433, "pass@32": 0.9977544910178684, "pass@64": 0.9985029940119761, "pass@128": 1.0 }, "11": { "pass@1": 0.9698795180722891, "pass@2": 0.9801018789725835, "pass@4": 0.9859459858556837, "pass@8": 0.9903607486554723, "pass@16": 0.9944354686155655, "pass@32": 0.997734424893915, "pass@64": 0.9996912428943647, "pass@128": 1.0 }, "12": { "pass@1": 0.9312793887147336, "pass@2": 0.9578514921136424, "pass@4": 0.9785946927088653, "pass@8": 0.9911220322102386, "pass@16": 0.9969256254585147, "pass@32": 0.999356859530079, "pass@64": 0.9999770727927124, "pass@128": 1.0 }, "13": { "pass@1": 0.9410014005602241, "pass@2": 0.9629413281060455, "pass@4": 0.9783460898795218, "pass@8": 0.9885164313887403, "pass@16": 0.9938448297567265, "pass@32": 0.99557976291731, "pass@64": 0.9965040977444787, "pass@128": 0.9971988795518207 }, "14": { "pass@1": 0.938093932748538, "pass@2": 0.9617224553345302, "pass@4": 0.9780836096145873, "pass@8": 0.9891690673671412, "pass@16": 0.9953444109429628, "pass@32": 0.9985866789814201, "pass@64": 0.9998999204723729, "pass@128": 1.0 }, "15": { "pass@1": 0.946875, "pass@2": 0.9708445180147938, "pass@4": 0.9852852896228875, "pass@8": 0.9932527012437333, "pass@16": 0.9965859651004563, "pass@32": 0.9976875706827749, "pass@64": 0.9984848257468957, "pass@128": 1.0 }, "16": { "pass@1": 0.8979255698005698, "pass@2": 0.9284205560042177, "pass@4": 0.9476878840999577, "pass@8": 0.9601126751922175, "pass@16": 0.9681501715762721, "pass@32": 0.9745545081734679, "pass@64": 0.9806737994273104, "pass@128": 0.9857549857549858 }, "17": { "pass@1": 0.883103649068323, "pass@2": 0.9193444484520957, "pass@4": 0.9431839631265031, "pass@8": 0.9588158770909269, "pass@16": 0.969734921800441, "pass@32": 0.9777583996276256, "pass@64": 0.9834804448754739, "pass@128": 0.9875776397515528 }, "18": { "pass@1": 0.8681985294117647, "pass@2": 0.9134885942566008, "pass@4": 0.9388149804803808, "pass@8": 0.956182315324883, "pass@16": 0.969199828469146, "pass@32": 0.9786850953286746, "pass@64": 0.98495366662752, "pass@128": 0.9882352941176471 }, "19": { "pass@1": 0.7562779017857143, "pass@2": 0.8173970929415075, "pass@4": 0.8586776217704924, "pass@8": 0.8873359588818277, "pass@16": 0.9093487259281797, "pass@32": 0.9286139494295144, "pass@64": 0.9465647885804772, "pass@128": 0.9613095238095238 }, "20": { "pass@1": 0.7049962574850299, "pass@2": 0.7821469511763871, "pass@4": 0.8387314392587147, "pass@8": 0.8803005442248173, "pass@16": 0.9090276962957424, "pass@32": 0.927911101416634, "pass@64": 0.9395962270441276, "pass@128": 0.9461077844311377 } } }, "movie_festival_awards": { "count": 801152, "correct": 744763, "answer_accuracy": 0.9296151042498801, "avg_response_len": 313.71964995406614, "resp_tokens_sum": 251337125, "pass_at_k": { "pass@1": 0.9296151042498801, "pass@2": 0.951017935747319, "pass@4": 0.9651440403059918, "pass@8": 0.974651918770272, "pass@16": 0.9810899167866008, "pass@32": 0.9857287259116158, "pass@64": 0.9891995583526448, "pass@128": 0.9915321936411567 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9518889925373134, "pass@2": 0.9682519978845928, "pass@4": 0.9806292692891003, "pass@8": 0.9888008490677124, "pass@16": 0.9928884788352844, "pass@32": 0.9939672147924427, "pass@64": 0.9940297978227394, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998177842565598, "pass@2": 0.9999960543835081, "pass@4": 0.9999999986335526, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9992233187134503, "pass@2": 0.9998215683565871, "pass@4": 0.9999901437758876, "pass@8": 0.9999999784883499, "pass@16": 0.9999999999999812, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9940946691176471, "pass@2": 0.9972234686197314, "pass@4": 0.9979508072520346, "pass@8": 0.9985971409541529, "pass@16": 0.9993657240569218, "pass@32": 0.999893411003342, "pass@64": 0.9999991012534062, "pass@128": 1.0 }, "6": { "pass@1": 0.9998767744479495, "pass@2": 0.9999976713281503, "pass@4": 0.9999999997042957, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9922122231012658, "pass@2": 0.9943055105651352, "pass@4": 0.9964102716986326, "pass@8": 0.9984351822242498, "pass@16": 0.9996795269701011, "pass@32": 0.9999902592414398, "pass@64": 0.9999999992697213, "pass@128": 1.0 }, "8": { "pass@1": 0.9930555555555556, "pass@2": 0.9965503150788586, "pass@4": 0.9984686342698152, "pass@8": 0.9996429053090278, "pass@16": 0.9999814246833897, "pass@32": 0.999999972034264, "pass@64": 0.9999999999999997, "pass@128": 1.0 }, "9": { "pass@1": 0.9924135852090032, "pass@2": 0.9953062099653139, "pass@4": 0.9966570869758643, "pass@8": 0.9975856061587218, "pass@16": 0.9983673492925828, "pass@32": 0.9992571222626693, "pass@64": 0.9999073347342836, "pass@128": 1.0 }, "11": { "pass@1": 0.9599702380952381, "pass@2": 0.9765876921634795, "pass@4": 0.989391186815934, "pass@8": 0.9963180825233957, "pass@16": 0.9992307852616401, "pass@32": 0.9999490539138667, "pass@64": 0.9999999121061375, "pass@128": 1.0 }, "12": { "pass@1": 0.939865252293578, "pass@2": 0.9603624124106047, "pass@4": 0.9760637159460566, "pass@8": 0.9866032202849236, "pass@16": 0.9926096934140741, "pass@32": 0.9959497775979793, "pass@64": 0.9984405444345842, "pass@128": 1.0 }, "13": { "pass@1": 0.9073832947530864, "pass@2": 0.9443962191358023, "pass@4": 0.9685251580242282, "pass@8": 0.9828957795842923, "pass@16": 0.9915318725469653, "pass@32": 0.996952650279957, "pass@64": 0.9995923226260632, "pass@128": 1.0 }, "14": { "pass@1": 0.9464252596439169, "pass@2": 0.9695290777821917, "pass@4": 0.9824109808974172, "pass@8": 0.9893288765335709, "pass@16": 0.9930767880716179, "pass@32": 0.9956401539621311, "pass@64": 0.996931099741985, "pass@128": 0.9970326409495549 }, "15": { "pass@1": 0.9288194444444444, "pass@2": 0.9530141197628071, "pass@4": 0.9679944280691763, "pass@8": 0.9773730797152483, "pass@16": 0.9826136664666306, "pass@32": 0.9854947655239579, "pass@64": 0.9876295766722463, "pass@128": 0.9907407407407407 }, "16": { "pass@1": 0.8941049041297935, "pass@2": 0.932356267275219, "pass@4": 0.9575642494356341, "pass@8": 0.9720565681989967, "pass@16": 0.9788168570235897, "pass@32": 0.9821212614436396, "pass@64": 0.9848713086920479, "pass@128": 0.9882005899705014 }, "17": { "pass@1": 0.8530685240963856, "pass@2": 0.8834865791907793, "pass@4": 0.9036101018246212, "pass@8": 0.9202718219997545, "pass@16": 0.9349628711134209, "pass@32": 0.9486017624674481, "pass@64": 0.9615355997566628, "pass@128": 0.9728915662650602 }, "18": { "pass@1": 0.8661290322580645, "pass@2": 0.9098425196850394, "pass@4": 0.9353601161749935, "pass@8": 0.9524239545873169, "pass@16": 0.9650574815675628, "pass@32": 0.9742872492835252, "pass@64": 0.9800047314029565, "pass@128": 0.9838709677419355 }, "19": { "pass@1": 0.7272233761329305, "pass@2": 0.7992720698432335, "pass@4": 0.8498617207403449, "pass@8": 0.886109822147316, "pass@16": 0.9119255337966973, "pass@32": 0.9312105063485466, "pass@64": 0.946458090289178, "pass@128": 0.9546827794561934 }, "20": { "pass@1": 0.714573948948949, "pass@2": 0.7877462846704968, "pass@4": 0.8363218603868204, "pass@8": 0.8718850949032955, "pass@16": 0.9008473092143473, "pass@32": 0.9259147136391661, "pass@64": 0.945745298634043, "pass@128": 0.9579579579579579 } } } } } ]