[ { "checkpoint": "checkpoint-38796", "id_acc": 0.9899713541666667, "ood_acc": 0.8540453125, "total_acc": 0.9184313322368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.3366076388889, "ood_avg_resp_len": 381.87561171875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 742004, "answer_accuracy": 0.9163620376225102, "avg_response_len": 282.5668360239488, "resp_tokens_sum": 228802279, "pass_at_k": { "pass@1": 0.9163620376225102, "pass@2": 0.9432990699238839, "pass@4": 0.9594926690468993, "pass@8": 0.9702768212202704, "pass@16": 0.978067131021929, "pass@32": 0.9837285212462428, "pass@64": 0.9878001481817852, "pass@128": 0.9908314890926335 }, "per_op_pass_at_k": { "10": { "pass@1": 0.940058075221239, "pass@2": 0.952202445241911, "pass@4": 0.9639851915634436, "pass@8": 0.9747452943211968, "pass@16": 0.9837117746740343, "pass@32": 0.9903082598333327, "pass@64": 0.9945723951890973, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999641628440367, "pass@2": 0.9999793066291025, "pass@4": 0.999999905401733, "pass@8": 0.9999999999996472, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9995361328125, "pass@2": 0.9999784694881889, "pass@4": 0.9999999370195912, "pass@8": 0.9999999999999016, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.991170634920635, "pass@2": 0.9935695538057743, "pass@4": 0.9953833833270841, "pass@8": 0.996783521967192, "pass@16": 0.9975425302123753, "pass@32": 0.9982188768739435, "pass@64": 0.9992125984251853, "pass@128": 1.0 }, "6": { "pass@1": 0.9999298278443114, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9981046597633136, "pass@2": 0.9996745853328983, "pass@4": 0.9999718887764769, "pass@8": 0.9999998022755866, "pass@16": 0.9999999999967328, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9924879807692307, "pass@2": 0.995214390268524, "pass@4": 0.9969837258924363, "pass@8": 0.9983251613066434, "pass@16": 0.9993038302603201, "pass@32": 0.9998838431738969, "pass@64": 0.9999990205966608, "pass@128": 1.0 }, "9": { "pass@1": 0.9905809859154929, "pass@2": 0.992608059775979, "pass@4": 0.9937131769620348, "pass@8": 0.9952330962332059, "pass@16": 0.9970819743714158, "pass@32": 0.9986530587245823, "pass@64": 0.9996542900027615, "pass@128": 1.0 }, "11": { "pass@1": 0.9567767351274787, "pass@2": 0.9729954579420491, "pass@4": 0.9837077224412106, "pass@8": 0.9915053244533025, "pass@16": 0.9967604348473688, "pass@32": 0.9993037377604537, "pass@64": 0.999959962274875, "pass@128": 1.0 }, "12": { "pass@1": 0.9463497528248588, "pass@2": 0.9650545370123225, "pass@4": 0.9766829093397222, "pass@8": 0.9848453467690281, "pass@16": 0.9914138441913528, "pass@32": 0.9958525342115812, "pass@64": 0.9982239261012225, "pass@128": 1.0 }, "13": { "pass@1": 0.9212627351097179, "pass@2": 0.953259601856194, "pass@4": 0.974031526035734, "pass@8": 0.9872172251386195, "pass@16": 0.9946975877158775, "pass@32": 0.9979212628228779, "pass@64": 0.9992200434190286, "pass@128": 1.0 }, "14": { "pass@1": 0.9226781542056075, "pass@2": 0.9488315457355219, "pass@4": 0.966660107907072, "pass@8": 0.9807327520370567, "pass@16": 0.9900033030501925, "pass@32": 0.994705328348, "pass@64": 0.997677205060592, "pass@128": 1.0 }, "15": { "pass@1": 0.9338872832369942, "pass@2": 0.9658665826544082, "pass@4": 0.9800264382595235, "pass@8": 0.9852710411507393, "pass@16": 0.9867086223947849, "pass@32": 0.9875391269726735, "pass@64": 0.9882671703608444, "pass@128": 0.9884393063583815 }, "16": { "pass@1": 0.8443296370967742, "pass@2": 0.8949314992379985, "pass@4": 0.9236545057875828, "pass@8": 0.9436194889710628, "pass@16": 0.9597137566119098, "pass@32": 0.9731177144426382, "pass@64": 0.9836290640887158, "pass@128": 0.9935483870967742 }, "17": { "pass@1": 0.8775063222543352, "pass@2": 0.9258678343043105, "pass@4": 0.9494584514544068, "pass@8": 0.9630582535369274, "pass@16": 0.9726016060789089, "pass@32": 0.9786961371782323, "pass@64": 0.9822631761264358, "pass@128": 0.9855491329479769 }, "18": { "pass@1": 0.8074330357142857, "pass@2": 0.8726863048368955, "pass@4": 0.9079920633135147, "pass@8": 0.9290310338171512, "pass@16": 0.9431022516433624, "pass@32": 0.9531360758782476, "pass@64": 0.9606210100169015, "pass@128": 0.9657142857142857 }, "19": { "pass@1": 0.6631240615615616, "pass@2": 0.7647959228913951, "pass@4": 0.8269798164306035, "pass@8": 0.8654212985928951, "pass@16": 0.8938579593227545, "pass@32": 0.9182400396910518, "pass@64": 0.9393446013378842, "pass@128": 0.9579579579579579 }, "20": { "pass@1": 0.6333521021021021, "pass@2": 0.7295261550920993, "pass@4": 0.7942091112484806, "pass@8": 0.8420309365275893, "pass@16": 0.8790149220263148, "pass@32": 0.9073538190959931, "pass@64": 0.9277009935323411, "pass@128": 0.93993993993994 } } }, "teachers_in_school": { "count": 821120, "correct": 757653, "answer_accuracy": 0.9227067907248636, "avg_response_len": 292.3765210931411, "resp_tokens_sum": 240076209, "pass_at_k": { "pass@1": 0.9227067907248636, "pass@2": 0.9491983678141017, "pass@4": 0.9648446673072715, "pass@8": 0.974732532480983, "pass@16": 0.9813596376382586, "pass@32": 0.9859942113928409, "pass@64": 0.9894281143827801, "pass@128": 0.9918939984411536 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9300469708588958, "pass@2": 0.9466099916670692, "pass@4": 0.9608256378803874, "pass@8": 0.9735479075977559, "pass@16": 0.9852696934862603, "pass@32": 0.9940972426092773, "pass@64": 0.9987733649475956, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9961647727272728, "pass@2": 0.9972918157957528, "pass@4": 0.9983365499767074, "pass@8": 0.9991928917832018, "pass@16": 0.9998063298472625, "pass@32": 0.9999922499316101, "pass@64": 0.9999999987964725, "pass@128": 1.0 }, "4": { "pass@1": 0.997965976331361, "pass@2": 0.9991187596095608, "pass@4": 0.9997857705841503, "pass@8": 0.9999865024173374, "pass@16": 0.9999999643806659, "pass@32": 0.9999999999999731, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.99375, "pass@2": 0.9967833504507589, "pass@4": 0.9986635988436225, "pass@8": 0.9996831882834758, "pass@16": 0.9999789012150662, "pass@32": 0.9999999417280491, "pass@64": 0.9999999999999963, "pass@128": 1.0 }, "6": { "pass@1": 0.9955676934097422, "pass@2": 0.9973236243034089, "pass@4": 0.9983310997300121, "pass@8": 0.9991142096348365, "pass@16": 0.9997496114237934, "pass@32": 0.9999854792844564, "pass@64": 0.999999993573057, "pass@128": 1.0 }, "7": { "pass@1": 0.9948067196531792, "pass@2": 0.9959908089071959, "pass@4": 0.9969885587820309, "pass@8": 0.9979235613025731, "pass@16": 0.9987206336864413, "pass@32": 0.9995059858135572, "pass@64": 0.9999600471131015, "pass@128": 1.0 }, "8": { "pass@1": 0.9922975352112676, "pass@2": 0.9943263419097261, "pass@4": 0.99571091852955, "pass@8": 0.9969972148105722, "pass@16": 0.99814919003857, "pass@32": 0.9991126715814442, "pass@64": 0.9998322253384277, "pass@128": 1.0 }, "9": { "pass@1": 0.995158121257485, "pass@2": 0.9969157403696542, "pass@4": 0.997358424601865, "pass@8": 0.9976945425380946, "pass@16": 0.9982569113741001, "pass@32": 0.9990676801627342, "pass@64": 0.9998216794756942, "pass@128": 1.0 }, "11": { "pass@1": 0.97265625, "pass@2": 0.9817142586092406, "pass@4": 0.9872070927128084, "pass@8": 0.9907918466392928, "pass@16": 0.9932876981463241, "pass@32": 0.9946033623345881, "pass@64": 0.9954810039531555, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9539086990595611, "pass@2": 0.9742443783476912, "pass@4": 0.9873180110910899, "pass@8": 0.9939946254019711, "pass@16": 0.9967163926023948, "pass@32": 0.9983956273931865, "pass@64": 0.9996077018477316, "pass@128": 1.0 }, "13": { "pass@1": 0.9435836834733894, "pass@2": 0.9670878686120119, "pass@4": 0.9795068925207874, "pass@8": 0.9867661154939795, "pass@16": 0.9912037146425846, "pass@32": 0.9943074150090554, "pass@64": 0.9971910964276763, "pass@128": 1.0 }, "14": { "pass@1": 0.9339820906432749, "pass@2": 0.961808433485288, "pass@4": 0.9791213154276766, "pass@8": 0.9903633423119838, "pass@16": 0.9965715597469695, "pass@32": 0.998978060543375, "pass@64": 0.9998257129593244, "pass@128": 1.0 }, "15": { "pass@1": 0.936032196969697, "pass@2": 0.9698241022429014, "pass@4": 0.986383716808126, "pass@8": 0.993887020431712, "pass@16": 0.997341288437486, "pass@32": 0.9992550901802849, "pass@64": 0.999953724719699, "pass@128": 1.0 }, "16": { "pass@1": 0.8875979344729344, "pass@2": 0.9302362344931245, "pass@4": 0.9518918415112638, "pass@8": 0.9653093003879017, "pass@16": 0.97478416069728, "pass@32": 0.9809651521660178, "pass@64": 0.986492391819359, "pass@128": 0.9914529914529915 }, "17": { "pass@1": 0.8626746894409938, "pass@2": 0.9124707322590117, "pass@4": 0.9368205890024611, "pass@8": 0.9508498521267209, "pass@16": 0.960456726487503, "pass@32": 0.9678057412486193, "pass@64": 0.9734389189717876, "pass@128": 0.9782608695652174 }, "18": { "pass@1": 0.8121553308823529, "pass@2": 0.8809753502779064, "pass@4": 0.9216372758552233, "pass@8": 0.9462221099752758, "pass@16": 0.9606949776103674, "pass@32": 0.9685754572068952, "pass@64": 0.9731564892213497, "pass@128": 0.9764705882352941 }, "19": { "pass@1": 0.66943359375, "pass@2": 0.7666238985751784, "pass@4": 0.8258763982627161, "pass@8": 0.8655574931646675, "pass@16": 0.8941625232220163, "pass@32": 0.9161714667742615, "pass@64": 0.9350029611318369, "pass@128": 0.9523809523809523 }, "20": { "pass@1": 0.6556652320359282, "pass@2": 0.7598498868404915, "pass@4": 0.8262946623625637, "pass@8": 0.8690020135890967, "pass@16": 0.8983220264807161, "pass@32": 0.9212925306509724, "pass@64": 0.9391524696479382, "pass@128": 0.9491017964071856 } } }, "movie_festival_awards": { "count": 801152, "correct": 733968, "answer_accuracy": 0.9161407573094743, "avg_response_len": 312.93695453546894, "resp_tokens_sum": 250710067, "pass_at_k": { "pass@1": 0.9161407573094743, "pass@2": 0.9444789070667229, "pass@4": 0.9620470564981104, "pass@8": 0.9734481996656786, "pass@16": 0.9808410217800196, "pass@32": 0.985989950434011, "pass@64": 0.9899999651668862, "pass@128": 0.993129892954146 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9534048507462687, "pass@2": 0.970000514161476, "pass@4": 0.9828655804218498, "pass@8": 0.9913297716090343, "pass@16": 0.9958306599914305, "pass@32": 0.9979223480778879, "pass@64": 0.9992399818731803, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9987928206997084, "pass@2": 0.99955952572255, "pass@4": 0.9999370614387487, "pass@8": 0.9999989051962135, "pass@16": 0.9999999998462147, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986979166666666, "pass@2": 0.9994841310954552, "pass@4": 0.9999133192232549, "pass@8": 0.9999978720766682, "pass@16": 0.9999999993392, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9966681985294118, "pass@2": 0.9992585543075498, "pass@4": 0.999930265610181, "pass@8": 0.9999993599537156, "pass@16": 0.9999999999763772, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.996352523659306, "pass@2": 0.9972878735189645, "pass@4": 0.9978065682444268, "pass@8": 0.9984953578892186, "pass@16": 0.9993197040358152, "pass@32": 0.9998856774168334, "pass@64": 0.999999036044663, "pass@128": 1.0 }, "7": { "pass@1": 0.9953273338607594, "pass@2": 0.9967836607943786, "pass@4": 0.9975761380302146, "pass@8": 0.998262935853195, "pass@16": 0.9990878595282221, "pass@32": 0.9997847637219769, "pass@64": 0.9999954280485364, "pass@128": 1.0 }, "8": { "pass@1": 0.9900525525525525, "pass@2": 0.9931176254995152, "pass@4": 0.9950761472946511, "pass@8": 0.996932633631575, "pass@16": 0.9985345359444987, "pass@32": 0.9996000242336316, "pass@64": 0.9999802487805222, "pass@128": 1.0 }, "9": { "pass@1": 0.9923382234726688, "pass@2": 0.9954557466389852, "pass@4": 0.9969244255440737, "pass@8": 0.9981047741525653, "pass@16": 0.9990691147291293, "pass@32": 0.9997813019589095, "pass@64": 0.9999953545444935, "pass@128": 1.0 }, "11": { "pass@1": 0.9581845238095238, "pass@2": 0.973211942257218, "pass@4": 0.9851733461888689, "pass@8": 0.9935984850997558, "pass@16": 0.9981004697081209, "pass@32": 0.9997117617660194, "pass@64": 0.9999952151978636, "pass@128": 1.0 }, "12": { "pass@1": 0.9543434633027523, "pass@2": 0.9737747266970067, "pass@4": 0.9860677859648275, "pass@8": 0.994249571393947, "pass@16": 0.9986259898728691, "pass@32": 0.9998862026515789, "pass@64": 0.9999995629648154, "pass@128": 1.0 }, "13": { "pass@1": 0.9164496527777778, "pass@2": 0.9467269824778841, "pass@4": 0.966515018955963, "pass@8": 0.9796284799576312, "pass@16": 0.9873678160183381, "pass@32": 0.9923304170452849, "pass@64": 0.9960751154460078, "pass@128": 1.0 }, "14": { "pass@1": 0.9455675074183977, "pass@2": 0.9688518569359099, "pass@4": 0.9830591241005852, "pass@8": 0.9918079912361211, "pass@16": 0.9964965731596999, "pass@32": 0.9989974984823478, "pass@64": 0.999910515524606, "pass@128": 1.0 }, "15": { "pass@1": 0.9213927469135802, "pass@2": 0.9536509216729855, "pass@4": 0.9729866107593029, "pass@8": 0.9846111586848604, "pass@16": 0.9907910506066029, "pass@32": 0.9946077911424446, "pass@64": 0.9975593713245301, "pass@128": 1.0 }, "16": { "pass@1": 0.8636845501474927, "pass@2": 0.9176269656005392, "pass@4": 0.9461433612280757, "pass@8": 0.9632940350686785, "pass@16": 0.9731921968948374, "pass@32": 0.9788198865927932, "pass@64": 0.9817049567519268, "pass@128": 0.9823008849557522 }, "17": { "pass@1": 0.816382718373494, "pass@2": 0.8697396623897163, "pass@4": 0.8989555943120052, "pass@8": 0.918514496266187, "pass@16": 0.9335748802757035, "pass@32": 0.9474461636352892, "pass@64": 0.9617706386333493, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8135332661290322, "pass@2": 0.8886441929133857, "pass@4": 0.9286830575613528, "pass@8": 0.9493191452530526, "pass@16": 0.9612486799216202, "pass@32": 0.9699809772790603, "pass@64": 0.9777500114141688, "pass@128": 0.9838709677419355 }, "19": { "pass@1": 0.6800651435045317, "pass@2": 0.7825676636058709, "pass@4": 0.8478114498338757, "pass@8": 0.890404706092961, "pass@16": 0.9181360078721577, "pass@32": 0.9381630090208142, "pass@64": 0.9558007487757464, "pass@128": 0.9697885196374623 }, "20": { "pass@1": 0.6123545420420421, "pass@2": 0.7181795476578936, "pass@4": 0.7932386998922434, "pass@8": 0.8470943217645397, "pass@16": 0.8868402120185244, "pass@32": 0.9171541099551569, "pass@64": 0.9405105380920781, "pass@128": 0.9579579579579579 } } } } } ]