[ { "checkpoint": "checkpoint-21808", "id_acc": 0.9135225694444444, "ood_acc": 0.2918375, "total_acc": 0.5863199013157895, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.60449913194444, "ood_avg_resp_len": 312.06185859375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 467579, "answer_accuracy": 0.5774519344767626, "avg_response_len": 245.32765200165983, "resp_tokens_sum": 198648669, "pass_at_k": { "pass@1": 0.5774519344767626, "pass@2": 0.6452416567297637, "pass@4": 0.6996034111666334, "pass@8": 0.7413455058301636, "pass@16": 0.7729299247279426, "pass@32": 0.7971920686194446, "pass@64": 0.8170532304403487, "pass@128": 0.8352829592159342 }, "per_op_pass_at_k": { "10": { "pass@1": 0.811992994100295, "pass@2": 0.8843032134810588, "pass@4": 0.923425883711439, "pass@8": 0.9457277067863578, "pass@16": 0.9606133820258961, "pass@32": 0.971941456242833, "pass@64": 0.979374562311218, "pass@128": 0.9823008849557522 }, "2": { "pass@1": 0.9364047897196262, "pass@2": 0.9491822429906543, "pass@4": 0.9604580686175911, "pass@8": 0.9694622504718368, "pass@16": 0.9762818313978732, "pass@32": 0.9814895602246809, "pass@64": 0.985727686903887, "pass@128": 0.9906542056074766 }, "3": { "pass@1": 0.9801462155963303, "pass@2": 0.9896420272580605, "pass@4": 0.9937778259368953, "pass@8": 0.9962077916624781, "pass@16": 0.9981311364256852, "pass@32": 0.9995033618733702, "pass@64": 0.999978967901273, "pass@128": 1.0 }, "4": { "pass@1": 0.969677734375, "pass@2": 0.9814676119586613, "pass@4": 0.986887528707349, "pass@8": 0.9899307200629559, "pass@16": 0.9917621691650451, "pass@32": 0.993528368509755, "pass@64": 0.9960998638576377, "pass@128": 1.0 }, "5": { "pass@1": 0.8992063492063492, "pass@2": 0.9267841519810028, "pass@4": 0.9439647365507879, "pass@8": 0.9548172712010686, "pass@16": 0.9623409287636865, "pass@32": 0.9688380283149466, "pass@64": 0.9755950258820365, "pass@128": 0.9809523809523809 }, "6": { "pass@1": 0.9132204341317365, "pass@2": 0.9474187697439762, "pass@4": 0.965994401803816, "pass@8": 0.9784493955374186, "pass@16": 0.9872882423827155, "pass@32": 0.9922681964206024, "pass@64": 0.9939103062084376, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.8838063979289941, "pass@2": 0.9277586719004803, "pass@4": 0.9530765176172504, "pass@8": 0.9664595878668178, "pass@16": 0.9740977457411283, "pass@32": 0.9795563479381585, "pass@64": 0.9843567124179412, "pass@128": 0.9881656804733728 }, "8": { "pass@1": 0.8821864983974359, "pass@2": 0.9410403101655567, "pass@4": 0.9686909091291468, "pass@8": 0.9800378182202505, "pass@16": 0.9851975708878595, "pass@32": 0.988658383641544, "pass@64": 0.9911660518873375, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.8510783450704226, "pass@2": 0.924036195519574, "pass@4": 0.9606805254096751, "pass@8": 0.9797217797223858, "pass@16": 0.9887705852950439, "pass@32": 0.9929424597619723, "pass@64": 0.9953811357885473, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.7112694759206799, "pass@2": 0.8131120206999624, "pass@4": 0.8741029357874173, "pass@8": 0.9131565853749971, "pass@16": 0.9409259508048508, "pass@32": 0.9607761804421557, "pass@64": 0.9752001653135165, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.5198181497175142, "pass@2": 0.652291444236843, "pass@4": 0.7531961534469203, "pass@8": 0.8229850744031951, "pass@16": 0.8703747593722103, "pass@32": 0.9035354750046218, "pass@64": 0.92679855938621, "pass@128": 0.943502824858757 }, "13": { "pass@1": 0.34842770376175547, "pass@2": 0.4644909504356623, "pass@4": 0.5666685317352571, "pass@8": 0.6515867742597433, "pass@16": 0.7209558398105108, "pass@32": 0.774390651037868, "pass@64": 0.8159555350904601, "pass@128": 0.8495297805642633 }, "14": { "pass@1": 0.2620959890965732, "pass@2": 0.36483859494198756, "pass@4": 0.4648828545964464, "pass@8": 0.5501300452769146, "pass@16": 0.6169298321030059, "pass@32": 0.6706355826772032, "pass@64": 0.7209786790660362, "pass@128": 0.7757009345794392 }, "15": { "pass@1": 0.2202176661849711, "pass@2": 0.3138636583450913, "pass@4": 0.4027050682067921, "pass@8": 0.4772871808592236, "pass@16": 0.5401813834656242, "pass@32": 0.5942484724698351, "pass@64": 0.6404521097017122, "pass@128": 0.684971098265896 }, "16": { "pass@1": 0.1792086693548387, "pass@2": 0.25795577216154436, "pass@4": 0.33871481397486597, "pass@8": 0.4115471071605108, "pass@16": 0.47119022128456445, "pass@32": 0.5159831622872939, "pass@64": 0.551697044512079, "pass@128": 0.5870967741935483 }, "17": { "pass@1": 0.17575867052023122, "pass@2": 0.2527504295434891, "pass@4": 0.3287763653820727, "pass@8": 0.3946413381092982, "pass@16": 0.4472001973243355, "pass@32": 0.48985344057501407, "pass@64": 0.5292538770652234, "pass@128": 0.5664739884393064 }, "18": { "pass@1": 0.16078125, "pass@2": 0.2357448678290212, "pass@4": 0.31116799641116283, "pass@8": 0.3780366912754487, "pass@16": 0.4329714410571362, "pass@32": 0.47825937552060704, "pass@64": 0.5182588254375425, "pass@128": 0.5571428571428572 }, "19": { "pass@1": 0.15373967717717718, "pass@2": 0.23123566479865715, "pass@4": 0.30846989959588395, "pass@8": 0.37090323389681995, "pass@16": 0.4176420067808336, "pass@32": 0.4519367816266829, "pass@64": 0.4781589939232686, "pass@128": 0.5045045045045045 }, "20": { "pass@1": 0.1547484984984985, "pass@2": 0.23214011255349842, "pass@4": 0.3100945782678066, "pass@8": 0.3735420424835841, "pass@16": 0.4193360888579724, "pass@32": 0.45293241378176113, "pass@64": 0.47896014395846753, "pass@128": 0.5015015015015015 } } }, "teachers_in_school": { "count": 821120, "correct": 482531, "answer_accuracy": 0.5876497954014029, "avg_response_len": 255.93222793257988, "resp_tokens_sum": 210151071, "pass_at_k": { "pass@1": 0.5876497954014029, "pass@2": 0.6525561207737851, "pass@4": 0.7061835820210686, "pass@8": 0.7490327747499966, "pass@16": 0.7829268531033039, "pass@32": 0.8100206823311829, "pass@64": 0.8319087994567788, "pass@128": 0.8497272018706158 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8102233512269938, "pass@2": 0.8818788192599386, "pass@4": 0.9230943886998785, "pass@8": 0.9486758986818827, "pass@16": 0.9665855700413544, "pass@32": 0.9798571294366684, "pass@64": 0.9889181243773738, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 0.9407057370820668, "pass@2": 0.9543108142067347, "pass@4": 0.9648635555244045, "pass@8": 0.9735475712944428, "pass@16": 0.980713903139753, "pass@32": 0.9855875985518405, "pass@64": 0.9885083794652192, "pass@128": 0.9908814589665653 }, "3": { "pass@1": 0.9831912878787878, "pass@2": 0.9890449773323791, "pass@4": 0.9925742770790013, "pass@8": 0.9949117732864091, "pass@16": 0.9968032945575152, "pass@32": 0.998445165474543, "pass@64": 0.999620777962612, "pass@128": 1.0 }, "4": { "pass@1": 0.9603365384615384, "pass@2": 0.9773203448958673, "pass@4": 0.9862346477215493, "pass@8": 0.9916871794012287, "pass@16": 0.995383380617976, "pass@32": 0.9977698233271339, "pass@64": 0.9994625180003253, "pass@128": 1.0 }, "5": { "pass@1": 0.9483016304347827, "pass@2": 0.9635705523222642, "pass@4": 0.9734722803671282, "pass@8": 0.9809383138264673, "pass@16": 0.9851226251370658, "pass@32": 0.9872274839565104, "pass@64": 0.9891271668301083, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.9358882521489972, "pass@2": 0.9721159161157868, "pass@4": 0.9889131951271127, "pass@8": 0.9951489089035775, "pass@16": 0.9979499235991857, "pass@32": 0.9995757636157672, "pass@64": 0.999990075961284, "pass@128": 1.0 }, "7": { "pass@1": 0.9294391257225434, "pass@2": 0.9595930431022713, "pass@4": 0.9755940045969683, "pass@8": 0.9847787879592633, "pass@16": 0.9892688554315738, "pass@32": 0.9908053653135379, "pass@64": 0.9912895111919918, "pass@128": 0.9913294797687862 }, "8": { "pass@1": 0.8965448943661972, "pass@2": 0.9502491821004772, "pass@4": 0.977070321931589, "pass@8": 0.9902363110097167, "pass@16": 0.9961681334129603, "pass@32": 0.9986011198470229, "pass@64": 0.999793253291699, "pass@128": 1.0 }, "9": { "pass@1": 0.8508373877245509, "pass@2": 0.9181932640624261, "pass@4": 0.9504120931365609, "pass@8": 0.9681510539987431, "pass@16": 0.9796319664627892, "pass@32": 0.9865058729633788, "pass@64": 0.9904633864127251, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.719714796686747, "pass@2": 0.8202420904088796, "pass@4": 0.8819906120545167, "pass@8": 0.9167660961161666, "pass@16": 0.9370201729227511, "pass@32": 0.9507746557206528, "pass@64": 0.9612102815531016, "pass@128": 0.9698795180722891 }, "12": { "pass@1": 0.5531690830721003, "pass@2": 0.691132707402562, "pass@4": 0.7918788569336356, "pass@8": 0.8558394828962391, "pass@16": 0.8961570326737536, "pass@32": 0.9242513072207585, "pass@64": 0.9447949934870175, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.31256565126050423, "pass@2": 0.4224401867046028, "pass@4": 0.5283050091427646, "pass@8": 0.6220202116228649, "pass@16": 0.6990070859405011, "pass@32": 0.7615370265201747, "pass@64": 0.8117002962134, "pass@128": 0.8487394957983193 }, "14": { "pass@1": 0.28723501461988304, "pass@2": 0.3853191767969794, "pass@4": 0.4756250548177094, "pass@8": 0.5538791634637721, "pass@16": 0.6247279946652008, "pass@32": 0.6916342968490513, "pass@64": 0.7535727398766265, "pass@128": 0.8070175438596491 }, "15": { "pass@1": 0.19509943181818182, "pass@2": 0.2874798675733715, "pass@4": 0.38628292741816334, "pass@8": 0.47756921040509814, "pass@16": 0.5566065517223067, "pass@32": 0.6230961707223709, "pass@64": 0.6746829994145453, "pass@128": 0.7151515151515152 }, "16": { "pass@1": 0.19831730769230768, "pass@2": 0.27961203367207305, "pass@4": 0.35857375119776697, "pass@8": 0.4255978477392451, "pass@16": 0.47931263574118715, "pass@32": 0.5250995747508337, "pass@64": 0.5695799608843882, "pass@128": 0.6125356125356125 }, "17": { "pass@1": 0.19613742236024845, "pass@2": 0.2840029252457574, "pass@4": 0.3722219300421298, "pass@8": 0.45339473578248296, "pass@16": 0.5245596190614837, "pass@32": 0.5828725152246444, "pass@64": 0.6277436674236548, "pass@128": 0.6614906832298136 }, "18": { "pass@1": 0.16300551470588234, "pass@2": 0.24151480720240845, "pass@4": 0.32298752040186174, "pass@8": 0.39488191333318684, "pass@16": 0.4550903465829013, "pass@32": 0.5079746597085013, "pass@64": 0.5556431661505973, "pass@128": 0.5970588235294118 }, "19": { "pass@1": 0.13816034226190477, "pass@2": 0.20664604424446936, "pass@4": 0.27967271836556157, "pass@8": 0.34818998174106214, "pass@16": 0.4053455576540456, "pass@32": 0.4497394448722171, "pass@64": 0.4830647705525789, "pass@128": 0.5029761904761905 }, "20": { "pass@1": 0.1348942739520958, "pass@2": 0.20611348660945836, "pass@4": 0.2826266614689631, "pass@8": 0.35183716167265444, "pass@16": 0.4075823091006288, "pass@32": 0.44687399315461174, "pass@64": 0.474497256958554, "pass@128": 0.5029940119760479 } } }, "movie_festival_awards": { "count": 801152, "correct": 475820, "answer_accuracy": 0.5939197555520052, "avg_response_len": 275.3332975515258, "resp_tokens_sum": 220583822, "pass_at_k": { "pass@1": 0.5939197555520052, "pass@2": 0.6590625247674902, "pass@4": 0.7127488353628675, "pass@8": 0.7552895351531898, "pass@16": 0.7882845933606959, "pass@32": 0.8140099630225133, "pass@64": 0.8343875634908813, "pass@128": 0.8515737338232945 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8168376865671642, "pass@2": 0.8903752644259023, "pass@4": 0.9318934934439158, "pass@8": 0.9560157584451383, "pass@16": 0.9699892576230075, "pass@32": 0.9795367066929017, "pass@64": 0.9873081396584162, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 0.9385491071428571, "pass@2": 0.9541169853768277, "pass@4": 0.9660614711553913, "pass@8": 0.9742850734130257, "pass@16": 0.9801474858828125, "pass@32": 0.9856558188638964, "pass@64": 0.990288117840151, "pass@128": 0.9942857142857143 }, "3": { "pass@1": 0.9768358236151603, "pass@2": 0.9841314478547326, "pass@4": 0.9887277911325226, "pass@8": 0.9920268596185634, "pass@16": 0.9936995287368084, "pass@32": 0.9941390350274529, "pass@64": 0.9941690610183596, "pass@128": 0.9941690962099126 }, "4": { "pass@1": 0.9728846856725146, "pass@2": 0.9822816658608464, "pass@4": 0.9869530976939285, "pass@8": 0.9907192584205899, "pass@16": 0.9939338085875972, "pass@32": 0.9963771697187885, "pass@64": 0.9983234387880316, "pass@128": 1.0 }, "5": { "pass@1": 0.9124080882352941, "pass@2": 0.9346861973135713, "pass@4": 0.9515868884955558, "pass@8": 0.9647522806518978, "pass@16": 0.9746544901240447, "pass@32": 0.9818895051410447, "pass@64": 0.987542165474913, "pass@128": 0.9911764705882353 }, "6": { "pass@1": 0.9517202287066246, "pass@2": 0.9792325163317526, "pass@4": 0.9896437670251783, "pass@8": 0.992935360514013, "pass@16": 0.9936369730893512, "pass@32": 0.9936905429777773, "pass@64": 0.9936908517347554, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.9434582674050633, "pass@2": 0.9697599484202134, "pass@4": 0.9808773431604276, "pass@8": 0.9866220229775958, "pass@16": 0.990433865453002, "pass@32": 0.9935113432880496, "pass@64": 0.9968041500282304, "pass@128": 1.0 }, "8": { "pass@1": 0.894237987987988, "pass@2": 0.9414288796670683, "pass@4": 0.9675921847043887, "pass@8": 0.9825535104384212, "pass@16": 0.9912589117683784, "pass@32": 0.9960952463036303, "pass@64": 0.9983943194216062, "pass@128": 1.0 }, "9": { "pass@1": 0.8760048231511254, "pass@2": 0.940809191204396, "pass@4": 0.967987871291007, "pass@8": 0.9795005195077631, "pass@16": 0.9861004723470128, "pass@32": 0.9907761116277802, "pass@64": 0.9943375327526845, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.7388888888888889, "pass@2": 0.8396380139982507, "pass@4": 0.8983732589973874, "pass@8": 0.9293637318750155, "pass@16": 0.9469655881937245, "pass@32": 0.958052279914744, "pass@64": 0.9645245347844873, "pass@128": 0.9682539682539683 }, "12": { "pass@1": 0.5382979740061162, "pass@2": 0.6815369229935708, "pass@4": 0.7901882852372809, "pass@8": 0.8623947124188998, "pass@16": 0.9080716635283972, "pass@32": 0.9356809750706235, "pass@64": 0.9492487788739141, "pass@128": 0.9571865443425076 }, "13": { "pass@1": 0.3448350694444444, "pass@2": 0.4652192998444639, "pass@4": 0.571143411182398, "pass@8": 0.6531886175627993, "pass@16": 0.7148291106889169, "pass@32": 0.7647239391313766, "pass@64": 0.8073138228282857, "pass@128": 0.8395061728395061 }, "14": { "pass@1": 0.30837351632047477, "pass@2": 0.41777261150961487, "pass@4": 0.5206757623397962, "pass@8": 0.6089125239005821, "pass@16": 0.6839245743828597, "pass@32": 0.7492554568430402, "pass@64": 0.8034429167524338, "pass@128": 0.8486646884272997 }, "15": { "pass@1": 0.22497106481481483, "pass@2": 0.32391079153300256, "pass@4": 0.4264279091849627, "pass@8": 0.5198736275821021, "pass@16": 0.5952772518420743, "pass@32": 0.6529361327041301, "pass@64": 0.6997906537176186, "pass@128": 0.7407407407407407 }, "16": { "pass@1": 0.19065726401179942, "pass@2": 0.27215329071144856, "pass@4": 0.35222581856471474, "pass@8": 0.4258438236788404, "pass@16": 0.49209327147276405, "pass@32": 0.5494043824143943, "pass@64": 0.6004721355972974, "pass@128": 0.6489675516224189 }, "17": { "pass@1": 0.16488610692771086, "pass@2": 0.24255214756664453, "pass@4": 0.3247423436716496, "pass@8": 0.40012973755769266, "pass@16": 0.4646677130767527, "pass@32": 0.5190198430686861, "pass@64": 0.5650992824649722, "pass@128": 0.6054216867469879 }, "18": { "pass@1": 0.1730342741935484, "pass@2": 0.24966979933959876, "pass@4": 0.32699005588011176, "pass@8": 0.39331440500639975, "pass@16": 0.4431195946101498, "pass@32": 0.48034270092756165, "pass@64": 0.5123963377060167, "pass@128": 0.5483870967741935 }, "19": { "pass@1": 0.14971204682779457, "pass@2": 0.2218807984632585, "pass@4": 0.2958488075131092, "pass@8": 0.36166937316077835, "pass@16": 0.41616715536535226, "pass@32": 0.4580014870377535, "pass@64": 0.4888460828597657, "pass@128": 0.5166163141993958 }, "20": { "pass@1": 0.14198573573573572, "pass@2": 0.2137330489938758, "pass@4": 0.2915296775966066, "pass@8": 0.36565955064618744, "pass@16": 0.4282390230240574, "pass@32": 0.4767202275751524, "pass@64": 0.5107559543079784, "pass@128": 0.5315315315315315 } } } } } ]