[ { "checkpoint": "checkpoint-22350", "id_acc": 0.9594730902777778, "ood_acc": 0.34515703125, "total_acc": 0.6361488486842105, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.4464236111111, "ood_avg_resp_len": 307.698259375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 519514, "answer_accuracy": 0.6415907564021499, "avg_response_len": 243.61820636065445, "resp_tokens_sum": 197264483, "pass_at_k": { "pass@1": 0.6415907564021499, "pass@2": 0.6795674052342421, "pass@4": 0.7104374509007589, "pass@8": 0.7361038308882039, "pass@16": 0.7581496440955067, "pass@32": 0.7774416276004311, "pass@64": 0.7941477275875496, "pass@128": 0.8080935820423648 }, "per_op_pass_at_k": { "10": { "pass@1": 0.903530604719764, "pass@2": 0.9280135095115319, "pass@4": 0.9426811864224937, "pass@8": 0.9534636795009439, "pass@16": 0.9602740600181923, "pass@32": 0.9643415604348307, "pass@64": 0.9677587233741899, "pass@128": 0.9705014749262537 }, "2": { "pass@1": 0.9996592679127726, "pass@2": 0.9999701044962838, "pass@4": 0.9999997912060058, "pass@8": 0.9999999999971957, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9957234327217125, "pass@2": 0.9966740109321197, "pass@4": 0.9969229032724121, "pass@8": 0.9969418134426237, "pass@16": 0.9969418960240432, "pass@32": 0.9969418960244648, "pass@64": 0.9969418960244648, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.9931884765625, "pass@2": 0.9937711460383859, "pass@4": 0.9938475112485939, "pass@8": 0.993945312498918, "pass@16": 0.994140625, "pass@32": 0.99453125, "pass@64": 0.9953125, "pass@128": 0.996875 }, "5": { "pass@1": 0.9752232142857142, "pass@2": 0.9805809430071241, "pass@4": 0.9836755715059426, "pass@8": 0.9860623829306817, "pass@16": 0.9877131700577393, "pass@32": 0.988865914595378, "pass@64": 0.9904761738606371, "pass@128": 0.9936507936507937 }, "6": { "pass@1": 0.9744807260479041, "pass@2": 0.9818922686123818, "pass@4": 0.9849555498115132, "pass@8": 0.9867536020108717, "pass@16": 0.9885545833866755, "pass@32": 0.9907015576214719, "pass@64": 0.9928842794533806, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.9752218934911243, "pass@2": 0.9823286003820528, "pass@4": 0.9855232493127706, "pass@8": 0.9876027174492761, "pass@16": 0.9900139637057965, "pass@32": 0.9935202105812683, "pass@64": 0.9977249219587196, "pass@128": 1.0 }, "8": { "pass@1": 0.9639172676282052, "pass@2": 0.9756736763072885, "pass@4": 0.9796417202657361, "pass@8": 0.9811646330398488, "pass@16": 0.9820810896845035, "pass@32": 0.9829762025036928, "pass@64": 0.9837834645669173, "pass@128": 0.9839743589743589 }, "9": { "pass@1": 0.9407350352112676, "pass@2": 0.9646823361428418, "pass@4": 0.97650347359749, "pass@8": 0.9829905290283689, "pass@16": 0.9880542555562098, "pass@32": 0.9923821609397722, "pass@64": 0.9953117085182914, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.8119909702549575, "pass@2": 0.8574068445941428, "pass@4": 0.8905700936639295, "pass@8": 0.9141548025681611, "pass@16": 0.9310147321010921, "pass@32": 0.9431402746849071, "pass@64": 0.9527867386839005, "pass@128": 0.9603399433427762 }, "12": { "pass@1": 0.72314177259887, "pass@2": 0.7903432092174907, "pass@4": 0.8314442794756585, "pass@8": 0.8607643836517445, "pass@16": 0.8863590398904893, "pass@32": 0.9076966113232872, "pass@64": 0.924174427048371, "pass@128": 0.9350282485875706 }, "13": { "pass@1": 0.5077390282131662, "pass@2": 0.6026861748080863, "pass@4": 0.6746213919576663, "pass@8": 0.7280871199402102, "pass@16": 0.7710803144173798, "pass@32": 0.8074469976041617, "pass@64": 0.8368862455010191, "pass@128": 0.8589341692789969 }, "14": { "pass@1": 0.32676207165109034, "pass@2": 0.4022562673240613, "pass@4": 0.47134501353218683, "pass@8": 0.5316685145841253, "pass@16": 0.583733926077304, "pass@32": 0.6325814592753726, "pass@64": 0.6818888012957016, "pass@128": 0.7289719626168224 }, "15": { "pass@1": 0.2350298049132948, "pass@2": 0.2988132794365298, "pass@4": 0.3616546562460039, "pass@8": 0.4242653266749827, "pass@16": 0.4833672306164851, "pass@32": 0.5327217007147764, "pass@64": 0.5712522977311489, "pass@128": 0.6040462427745664 }, "16": { "pass@1": 0.20788810483870968, "pass@2": 0.27152257429514853, "pass@4": 0.327680491853841, "pass@8": 0.3755843283869693, "pass@16": 0.41732714469475174, "pass@32": 0.4543796479160046, "pass@64": 0.4873940801671596, "pass@128": 0.5161290322580645 }, "17": { "pass@1": 0.19716401734104047, "pass@2": 0.25503077920895717, "pass@4": 0.30815697911316003, "pass@8": 0.35562136170719266, "pass@16": 0.3982127691151806, "pass@32": 0.4370893005671065, "pass@64": 0.47201922867722684, "pass@128": 0.5028901734104047 }, "18": { "pass@1": 0.16732142857142857, "pass@2": 0.22041725253093356, "pass@4": 0.2704885762493973, "pass@8": 0.3153377965478373, "pass@16": 0.3545991358083943, "pass@32": 0.38914450742366186, "pass@64": 0.4186827347023023, "pass@128": 0.44285714285714284 }, "19": { "pass@1": 0.17396302552552553, "pass@2": 0.23597921248019668, "pass@4": 0.2925832649297215, "pass@8": 0.33894785591823995, "pass@16": 0.3781170566234184, "pass@32": 0.41324923316576234, "pass@64": 0.44271916868136607, "pass@128": 0.46546546546546547 }, "20": { "pass@1": 0.15355198948948948, "pass@2": 0.20524424522948134, "pass@4": 0.25447808072414363, "pass@8": 0.29844947745611566, "pass@16": 0.3366465437604806, "pass@32": 0.3712929813911508, "pass@64": 0.4015159625628319, "pass@128": 0.4264264264264264 } } }, "teachers_in_school": { "count": 821120, "correct": 521029, "answer_accuracy": 0.6345345381917381, "avg_response_len": 251.66157199922057, "resp_tokens_sum": 206644350, "pass_at_k": { "pass@1": 0.6345345381917381, "pass@2": 0.6713106661306874, "pass@4": 0.7017797563301446, "pass@8": 0.7275078394786831, "pass@16": 0.7501136396213048, "pass@32": 0.7705007575565151, "pass@64": 0.7884642294169927, "pass@128": 0.8038971161340608 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8560678680981595, "pass@2": 0.8841039533114343, "pass@4": 0.9030514761191657, "pass@8": 0.9173781283934727, "pass@16": 0.9282431728633065, "pass@32": 0.9367494154153667, "pass@64": 0.9431173930127746, "pass@128": 0.9478527607361963 }, "2": { "pass@1": 0.9994300911854104, "pass@2": 0.9999536294665294, "pass@4": 0.9999994814477978, "pass@8": 0.9999999999726388, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.994720643939394, "pass@2": 0.9957684621808638, "pass@4": 0.9967587758916499, "pass@8": 0.9976809114058548, "pass@16": 0.998458723990178, "pass@32": 0.9992998922984194, "pass@64": 0.9999126700071582, "pass@128": 1.0 }, "4": { "pass@1": 0.9996070636094675, "pass@2": 0.9999854400596374, "pass@4": 0.9999999800320818, "pass@8": 0.999999999999998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9763360507246377, "pass@2": 0.9799897295446762, "pass@4": 0.9829189930063094, "pass@8": 0.9854568297076476, "pass@16": 0.9878492664588717, "pass@32": 0.9902406260662067, "pass@64": 0.992681422004542, "pass@128": 0.9942028985507246 }, "6": { "pass@1": 0.9599973137535817, "pass@2": 0.9650911349637887, "pass@4": 0.9684557643940639, "pass@8": 0.9713198049675233, "pass@16": 0.9734459817059731, "pass@32": 0.9751500499585644, "pass@64": 0.9770377542411987, "pass@128": 0.9799426934097422 }, "7": { "pass@1": 0.9596278901734104, "pass@2": 0.9690860253288427, "pass@4": 0.9756533710020351, "pass@8": 0.9813592130007283, "pass@16": 0.9863418091645095, "pass@32": 0.9903767960067987, "pass@64": 0.9930983479380481, "pass@128": 0.9942196531791907 }, "8": { "pass@1": 0.9377860915492958, "pass@2": 0.958912678828879, "pass@4": 0.9724244942093503, "pass@8": 0.9806020235436738, "pass@16": 0.9851324251816836, "pass@32": 0.9876871801465659, "pass@64": 0.9886708658809289, "pass@128": 0.9887323943661972 }, "9": { "pass@1": 0.9292196856287425, "pass@2": 0.9527257001744545, "pass@4": 0.9627166653756599, "pass@8": 0.9688774460237405, "pass@16": 0.9742544799641086, "pass@32": 0.9797131952662533, "pass@64": 0.9843850850382375, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.8160062123493976, "pass@2": 0.8610585303813678, "pass@4": 0.8899617457456372, "pass@8": 0.9105718643838208, "pass@16": 0.9254026184703923, "pass@32": 0.9363826773485632, "pass@64": 0.9448460895593244, "pass@128": 0.9518072289156626 }, "12": { "pass@1": 0.7139743338557993, "pass@2": 0.7795156030163154, "pass@4": 0.8230373401600654, "pass@8": 0.8540952671660289, "pass@16": 0.8784286480198956, "pass@32": 0.8988179779778064, "pass@64": 0.914751375375745, "pass@128": 0.9247648902821317 }, "13": { "pass@1": 0.4431022408963585, "pass@2": 0.5229334844173891, "pass@4": 0.5902965916025199, "pass@8": 0.647391393076343, "pass@16": 0.699989527769502, "pass@32": 0.7489458173941425, "pass@64": 0.7897583683021384, "pass@128": 0.8207282913165266 }, "14": { "pass@1": 0.35249908625730997, "pass@2": 0.4206065524704149, "pass@4": 0.47809296233804116, "pass@8": 0.5299356203940037, "pass@16": 0.5811376255515341, "pass@32": 0.6335831039468219, "pass@64": 0.6852598413531686, "pass@128": 0.7309941520467836 }, "15": { "pass@1": 0.23425662878787878, "pass@2": 0.2991070895967547, "pass@4": 0.3562961430957494, "pass@8": 0.40513356161577346, "pass@16": 0.44987841221925085, "pass@32": 0.49389229180815036, "pass@64": 0.5351252376246326, "pass@128": 0.5727272727272728 }, "16": { "pass@1": 0.2116942663817664, "pass@2": 0.2658307535276038, "pass@4": 0.3131332528412581, "pass@8": 0.35405246824329895, "pass@16": 0.3925773455929453, "pass@32": 0.4296501256109476, "pass@64": 0.4638134199632514, "pass@128": 0.4985754985754986 }, "17": { "pass@1": 0.19443905279503104, "pass@2": 0.2521755942192007, "pass@4": 0.3052168306213276, "pass@8": 0.3531225638086416, "pass@16": 0.3969801368618287, "pass@32": 0.4357863106880458, "pass@64": 0.46768276618636456, "pass@128": 0.4906832298136646 }, "18": { "pass@1": 0.17817095588235293, "pass@2": 0.2388269308707734, "pass@4": 0.2978523059617549, "pass@8": 0.3508819219268661, "pass@16": 0.39719267118461893, "pass@32": 0.4392130178110388, "pass@64": 0.4782813253719725, "pass@128": 0.5147058823529411 }, "19": { "pass@1": 0.150390625, "pass@2": 0.20729415658980124, "pass@4": 0.2629280089988752, "pass@8": 0.3126190421848842, "pass@16": 0.35452784964061956, "pass@32": 0.38692673717859644, "pass@64": 0.41329885722988, "pass@128": 0.4375 }, "20": { "pass@1": 0.1395256362275449, "pass@2": 0.19365010137205899, "pass@4": 0.24706315462438455, "pass@8": 0.29426874511051465, "pass@16": 0.334118849246813, "pass@32": 0.3683037737540345, "pass@64": 0.39960945985179536, "pass@128": 0.4281437125748503 } } }, "movie_festival_awards": { "count": 801152, "correct": 506571, "answer_accuracy": 0.6323032333439846, "avg_response_len": 269.9253312729669, "resp_tokens_sum": 216251219, "pass_at_k": { "pass@1": 0.6323032333439846, "pass@2": 0.668776332160433, "pass@4": 0.6993617351943084, "pass@8": 0.725679187269482, "pass@16": 0.748700830889992, "pass@32": 0.7692323753744258, "pass@64": 0.7880763014515265, "pass@128": 0.8058795334718006 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8672341417910447, "pass@2": 0.8958841373839462, "pass@4": 0.9150610647549656, "pass@8": 0.9293536958508508, "pass@16": 0.9413545846623945, "pass@32": 0.9513829259023318, "pass@64": 0.9601114523123534, "pass@128": 0.9671641791044776 }, "2": { "pass@1": 0.9990848214285715, "pass@2": 0.999859392575928, "pass@4": 0.9999945069366328, "pass@8": 0.9999999937887084, "pass@16": 0.999999999999999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9923469387755102, "pass@2": 0.9945726251693029, "pass@4": 0.9963000212619196, "pass@8": 0.9976444803649415, "pass@16": 0.9985107643235256, "pass@32": 0.9993264176696698, "pass@64": 0.9999159798902688, "pass@128": 1.0 }, "4": { "pass@1": 0.9940606725146199, "pass@2": 0.9952050093244924, "pass@4": 0.9959562867141608, "pass@8": 0.9966627049626813, "pass@16": 0.9970259283691618, "pass@32": 0.9970756091990718, "pass@64": 0.997076023391154, "pass@128": 0.9970760233918129 }, "5": { "pass@1": 0.9534926470588235, "pass@2": 0.9598996931449746, "pass@4": 0.9647326030018306, "pass@8": 0.9687924250018215, "pass@16": 0.9727276001124571, "pass@32": 0.9771633497683939, "pass@64": 0.9818101565064178, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9694400630914827, "pass@2": 0.976782753545791, "pass@4": 0.9821603184854258, "pass@8": 0.9859407044747921, "pass@16": 0.988325805700107, "pass@32": 0.9902831845784584, "pass@64": 0.9929082222474788, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9503807357594937, "pass@2": 0.9640580615219775, "pass@4": 0.9723427231801719, "pass@8": 0.9783455795238412, "pass@16": 0.9829082029190926, "pass@32": 0.9853335592118175, "pass@64": 0.986555443080517, "pass@128": 0.9873417721518988 }, "8": { "pass@1": 0.9269894894894894, "pass@2": 0.9444093453689912, "pass@4": 0.9565674254119138, "pass@8": 0.9652994442275824, "pass@16": 0.972560615488667, "pass@32": 0.9789532952080486, "pass@64": 0.9852802738870954, "pass@128": 0.990990990990991 }, "9": { "pass@1": 0.9176798633440515, "pass@2": 0.9395939090310658, "pass@4": 0.9518787198424955, "pass@8": 0.961285947902821, "pass@16": 0.968386382790631, "pass@32": 0.9734806178809153, "pass@64": 0.9791055415271452, "pass@128": 0.9871382636655949 }, "11": { "pass@1": 0.8142113095238095, "pass@2": 0.8580154043244594, "pass@4": 0.8854600091655204, "pass@8": 0.9040861946350753, "pass@16": 0.917327529399465, "pass@32": 0.9277359940884433, "pass@64": 0.9392719044117785, "pass@128": 0.9555555555555556 }, "12": { "pass@1": 0.7052752293577982, "pass@2": 0.7677594271472944, "pass@4": 0.8131363341623575, "pass@8": 0.8499243285325645, "pass@16": 0.8787344801874143, "pass@32": 0.9009459229157835, "pass@64": 0.9185204439860378, "pass@128": 0.9327217125382263 }, "13": { "pass@1": 0.4840374228395062, "pass@2": 0.5678211122047245, "pass@4": 0.6344011032185792, "pass@8": 0.6896730840043367, "pass@16": 0.7377430046359499, "pass@32": 0.7808939941901869, "pass@64": 0.817655131521307, "pass@128": 0.845679012345679 }, "14": { "pass@1": 0.34226632047477745, "pass@2": 0.41757437381714485, "pass@4": 0.4810371275919886, "pass@8": 0.5324609913290231, "pass@16": 0.5765252565433672, "pass@32": 0.6195243218028179, "pass@64": 0.6638484308398551, "pass@128": 0.7091988130563798 }, "15": { "pass@1": 0.2175684799382716, "pass@2": 0.2820352751045008, "pass@4": 0.3425003804501287, "pass@8": 0.3957935260999967, "pass@16": 0.44250060778231465, "pass@32": 0.4848249916319935, "pass@64": 0.5263831871583865, "pass@128": 0.5709876543209876 }, "16": { "pass@1": 0.20268713126843657, "pass@2": 0.26056365119736136, "pass@4": 0.3140499846258156, "pass@8": 0.361826706999834, "pass@16": 0.40430848371338823, "pass@32": 0.44235122437377733, "pass@64": 0.4786072043273569, "pass@128": 0.5162241887905604 }, "17": { "pass@1": 0.1823465737951807, "pass@2": 0.2351432056968028, "pass@4": 0.285518104194054, "pass@8": 0.33299355661175367, "pass@16": 0.3771098697464484, "pass@32": 0.41711976481202867, "pass@64": 0.4538293890103033, "pass@128": 0.4879518072289157 }, "18": { "pass@1": 0.16305443548387097, "pass@2": 0.21881270637541272, "pass@4": 0.27318726731739185, "pass@8": 0.3228368526161366, "pass@16": 0.3684910532120487, "pass@32": 0.4120752139334768, "pass@64": 0.451199065109953, "pass@128": 0.4838709677419355 }, "19": { "pass@1": 0.15138783987915408, "pass@2": 0.19958979589409337, "pass@4": 0.2458423523154772, "pass@8": 0.2905585714895618, "pass@16": 0.33197918407873356, "pass@32": 0.36661461630839115, "pass@64": 0.3943805189143051, "pass@128": 0.4199395770392749 }, "20": { "pass@1": 0.15453734984984985, "pass@2": 0.20794946028705857, "pass@4": 0.2596704634893611, "pass@8": 0.3088198234624205, "pass@16": 0.3551515634200191, "pass@32": 0.39827724021815375, "pass@64": 0.4363332912293886, "pass@128": 0.46846846846846846 } } } } } ]