[ { "checkpoint": "checkpoint-86183", "id_acc": 0.9919913194444444, "ood_acc": 0.4688859375, "total_acc": 0.7166726973684211, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.9115217013889, "ood_avg_resp_len": 336.13270703125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 580465, "answer_accuracy": 0.7168641815523238, "avg_response_len": 259.1584087496048, "resp_tokens_sum": 209847820, "pass_at_k": { "pass@1": 0.7168641815523238, "pass@2": 0.7563181321430583, "pass@4": 0.7891167671974877, "pass@8": 0.8160179728357079, "pass@16": 0.8380543061281882, "pass@32": 0.8560953306144278, "pass@64": 0.8713585957411207, "pass@128": 0.8858678469807145 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9424317846607669, "pass@2": 0.9554277576475513, "pass@4": 0.9645232338214357, "pass@8": 0.9705729284703094, "pass@16": 0.9750282528248507, "pass@32": 0.979012371776588, "pass@64": 0.9825793273267351, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9998511904761904, "pass@2": 0.9999992188476441, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9982456961077845, "pass@2": 0.9989778071809138, "pass@4": 0.9996588796535164, "pass@8": 0.9999646694808768, "pass@16": 0.9999997257540574, "pass@32": 0.9999999999968382, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9997457470414202, "pass@2": 0.9999949040208731, "pass@4": 0.9999999986133391, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9944411057692307, "pass@2": 0.9981253470119119, "pass@4": 0.9996946421360792, "pass@8": 0.9999886513123039, "pass@16": 0.9999999830961206, "pass@32": 0.999999999999996, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9926496478873239, "pass@2": 0.996029721636908, "pass@4": 0.9975660128751512, "pass@8": 0.9983336498027551, "pass@16": 0.999063138456991, "pass@32": 0.9997387400876202, "pass@64": 0.9999912792555583, "pass@128": 1.0 }, "11": { "pass@1": 0.9241988314447592, "pass@2": 0.9451147085721935, "pass@4": 0.9583057509100311, "pass@8": 0.9683471315113082, "pass@16": 0.9758373276009441, "pass@32": 0.9813627292403387, "pass@64": 0.9854111072441946, "pass@128": 0.9886685552407932 }, "12": { "pass@1": 0.832229872881356, "pass@2": 0.8820969015525602, "pass@4": 0.9064526634912154, "pass@8": 0.9210831491573624, "pass@16": 0.9322645619669031, "pass@32": 0.9417992939933498, "pass@64": 0.9503938639071714, "pass@128": 0.9576271186440678 }, "13": { "pass@1": 0.6866918103448276, "pass@2": 0.7621581344259865, "pass@4": 0.8154033129283599, "pass@8": 0.8542870664981383, "pass@16": 0.8825616429284143, "pass@32": 0.9031559569013347, "pass@64": 0.9156100331712211, "pass@128": 0.9184952978056427 }, "14": { "pass@1": 0.6419635903426791, "pass@2": 0.724484570853877, "pass@4": 0.77979786434406, "pass@8": 0.8168612902209831, "pass@16": 0.8439180312348161, "pass@32": 0.8659116440785112, "pass@64": 0.8836119795991179, "pass@128": 0.897196261682243 }, "15": { "pass@1": 0.5370077673410405, "pass@2": 0.6450683571526107, "pass@4": 0.7338411369099088, "pass@8": 0.7986025998546395, "pass@16": 0.8436119684412422, "pass@32": 0.8759949765481339, "pass@64": 0.8985630011957301, "pass@128": 0.9161849710982659 }, "16": { "pass@1": 0.2983618951612903, "pass@2": 0.3907174720599443, "pass@4": 0.4704105074567288, "pass@8": 0.535630716107639, "pass@16": 0.5907458646622782, "pass@32": 0.6372065651541644, "pass@64": 0.6795116518435073, "pass@128": 0.7225806451612903 }, "17": { "pass@1": 0.2517837789017341, "pass@2": 0.3336255035046197, "pass@4": 0.41323942626752563, "pass@8": 0.48810200863925596, "pass@16": 0.5547621412716494, "pass@32": 0.6106711542212494, "pass@64": 0.6590666679799665, "pass@128": 0.708092485549133 }, "18": { "pass@1": 0.19428571428571428, "pass@2": 0.2676690804274464, "pass@4": 0.34212716508650715, "pass@8": 0.4139243368536438, "pass@16": 0.4805783850514051, "pass@32": 0.5388344572164389, "pass@64": 0.5889168734491658, "pass@128": 0.6371428571428571 }, "19": { "pass@1": 0.18327702702702703, "pass@2": 0.2533107664751364, "pass@4": 0.3211475995117731, "pass@8": 0.3820281677250221, "pass@16": 0.4348139659769961, "pass@32": 0.4792898587653093, "pass@64": 0.5180948362733667, "pass@128": 0.5585585585585585 }, "20": { "pass@1": 0.17429147897897898, "pass@2": 0.2449047226360218, "pass@4": 0.3154769668431085, "pass@8": 0.3776520541403074, "pass@16": 0.4277429922970302, "pass@32": 0.46807247378301114, "pass@64": 0.5071460121712338, "pass@128": 0.5525525525525525 } } }, "teachers_in_school": { "count": 821120, "correct": 587027, "answer_accuracy": 0.7149101227591582, "avg_response_len": 267.97832959859704, "resp_tokens_sum": 220042366, "pass_at_k": { "pass@1": 0.7149101227591582, "pass@2": 0.7551878248875328, "pass@4": 0.788089845667179, "pass@8": 0.8152628237248414, "pass@16": 0.8379713914459063, "pass@32": 0.8568640308079691, "pass@64": 0.8730298669312079, "pass@128": 0.8874512860483242 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9370446702453987, "pass@2": 0.9524521007439255, "pass@4": 0.9622697201193403, "pass@8": 0.9685217512597472, "pass@16": 0.9732031072775292, "pass@32": 0.9783710405726299, "pass@64": 0.985072569920195, "pass@128": 0.9938650306748467 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999306582840237, "pass@2": 0.9999989080044729, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9999094202898551, "pass@2": 0.9999989301609038, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9994179799426934, "pass@2": 0.9999027028856351, "pass@4": 0.9999971459513121, "pass@8": 0.9999999985260104, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9998419436416185, "pass@2": 0.9999996444176414, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9954225352112676, "pass@2": 0.9971879505378729, "pass@4": 0.9978147124215107, "pass@8": 0.9983444096217255, "pass@16": 0.9990631538362801, "pass@32": 0.9997387400876238, "pass@64": 0.9999912792555583, "pass@128": 1.0 }, "9": { "pass@1": 0.9916729041916168, "pass@2": 0.9941191681597434, "pass@4": 0.9953397361632191, "pass@8": 0.9964540303892805, "pass@16": 0.9977764166281999, "pass@32": 0.9990105981622154, "pass@64": 0.99982149369308, "pass@128": 1.0 }, "11": { "pass@1": 0.932652484939759, "pass@2": 0.9551909656341906, "pass@4": 0.967504908272008, "pass@8": 0.9743332008558964, "pass@16": 0.9776944087377336, "pass@32": 0.9797107289052188, "pass@64": 0.981380697080861, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8275127351097179, "pass@2": 0.8741306802754675, "pass@4": 0.8940140824873376, "pass@8": 0.9059417569605126, "pass@16": 0.9149301720126192, "pass@32": 0.9210451859792664, "pass@64": 0.9267178933506086, "pass@128": 0.9341692789968652 }, "13": { "pass@1": 0.6466211484593838, "pass@2": 0.7352568980348043, "pass@4": 0.7941474946934152, "pass@8": 0.8322874017240125, "pass@16": 0.8593893686275413, "pass@32": 0.8812230108196569, "pass@64": 0.8990334067498752, "pass@128": 0.9131652661064426 }, "14": { "pass@1": 0.6178956505847953, "pass@2": 0.7086092548464337, "pass@4": 0.7747260120445467, "pass@8": 0.8222148893486398, "pass@16": 0.856872436091234, "pass@32": 0.8836770532613494, "pass@64": 0.9065420227864525, "pass@128": 0.9269005847953217 }, "15": { "pass@1": 0.5457149621212121, "pass@2": 0.661665995585779, "pass@4": 0.7447602410494136, "pass@8": 0.8003608834636073, "pass@16": 0.837934698476195, "pass@32": 0.8649150245288428, "pass@64": 0.8867527597855025, "pass@128": 0.906060606060606 }, "16": { "pass@1": 0.3033297720797721, "pass@2": 0.38540860477376226, "pass@4": 0.4561851985382169, "pass@8": 0.5197053256135613, "pass@16": 0.5790541044663001, "pass@32": 0.6333955577098603, "pass@64": 0.6845196910184368, "pass@128": 0.7321937321937322 }, "17": { "pass@1": 0.2654794254658385, "pass@2": 0.3596355822370031, "pass@4": 0.4481225438823254, "pass@8": 0.5224691510802295, "pass@16": 0.580719897610317, "pass@32": 0.6246834181217727, "pass@64": 0.6622791915081849, "pass@128": 0.7080745341614907 }, "18": { "pass@1": 0.20091911764705883, "pass@2": 0.28180508916164887, "pass@4": 0.3687975969915525, "pass@8": 0.4557692555372953, "pass@16": 0.5351237386501738, "pass@32": 0.6004961265979195, "pass@64": 0.6507879587120721, "pass@128": 0.6911764705882353 }, "19": { "pass@1": 0.16022600446428573, "pass@2": 0.22008833368485176, "pass@4": 0.2805208366811292, "pass@8": 0.3395545181912359, "pass@16": 0.3959618657365702, "pass@32": 0.4475826196030161, "pass@64": 0.49460998571116155, "pass@128": 0.5327380952380952 }, "20": { "pass@1": 0.14796968562874252, "pass@2": 0.2126687815314254, "pass@4": 0.2800213515600969, "pass@8": 0.3452822815254244, "pass@16": 0.40522265822512654, "pass@32": 0.4578358076989143, "pass@64": 0.5011268713315313, "pass@128": 0.5329341317365269 } } }, "movie_festival_awards": { "count": 801152, "correct": 575456, "answer_accuracy": 0.7182856686371625, "avg_response_len": 287.9076030516057, "resp_tokens_sum": 230657752, "pass_at_k": { "pass@1": 0.7182856686371625, "pass@2": 0.7592052287226031, "pass@4": 0.7927262515206405, "pass@8": 0.8195860006912514, "pass@16": 0.8409061998859412, "pass@32": 0.8587729906883347, "pass@64": 0.874955447824755, "pass@128": 0.8894392075411408 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9499766791044776, "pass@2": 0.963093856504877, "pass@4": 0.9712197354062085, "pass@8": 0.9767062892294843, "pass@16": 0.9814928519708268, "pass@32": 0.9866806213656, "pass@64": 0.9918038267917523, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9995863970588236, "pass@2": 0.9999811834182493, "pass@4": 0.9999999611261827, "pass@8": 0.9999999999999815, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9982009069400631, "pass@2": 0.9996223670483617, "pass@4": 0.999974433992281, "pass@8": 0.999999864083505, "pass@16": 0.9999999999987256, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9979232594936709, "pass@2": 0.9995487547343765, "pass@4": 0.9999514044130559, "pass@8": 0.9999994224405141, "pass@16": 0.9999999999663859, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9945101351351351, "pass@2": 0.9986632794211536, "pass@4": 0.9998438415918729, "pass@8": 0.9999972864530364, "pass@16": 0.9999999995382962, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9925894292604501, "pass@2": 0.9951194869230575, "pass@4": 0.9964745589478163, "pass@8": 0.998042664097683, "pass@16": 0.9993761183512748, "pass@32": 0.9999366961026899, "pass@64": 0.9999998004436335, "pass@128": 1.0 }, "11": { "pass@1": 0.9227926587301587, "pass@2": 0.9470242000999873, "pass@4": 0.960732205795704, "pass@8": 0.9693603886062503, "pass@16": 0.9756707700497491, "pass@32": 0.9806056964313475, "pass@64": 0.9844370895991283, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.8355074541284404, "pass@2": 0.8936402122613109, "pass@4": 0.9236655713333995, "pass@8": 0.9403579265835318, "pass@16": 0.9498986664429232, "pass@32": 0.9564481186800877, "pass@64": 0.9620879995236127, "pass@128": 0.9663608562691132 }, "13": { "pass@1": 0.6571662808641975, "pass@2": 0.7358156836298242, "pass@4": 0.7884919118906429, "pass@8": 0.8258481457444613, "pass@16": 0.854241760542711, "pass@32": 0.8774085425557125, "pass@64": 0.8968649961634577, "pass@128": 0.9135802469135802 }, "14": { "pass@1": 0.6426418768545994, "pass@2": 0.7360609330825484, "pass@4": 0.7982204229293296, "pass@8": 0.8372049902894257, "pass@16": 0.8620695977837106, "pass@32": 0.8819375600885098, "pass@64": 0.901686644657056, "pass@128": 0.9198813056379822 }, "15": { "pass@1": 0.5323350694444444, "pass@2": 0.6377064195100618, "pass@4": 0.7172337514523645, "pass@8": 0.7761564818619724, "pass@16": 0.8170385569299585, "pass@32": 0.8452706602617858, "pass@64": 0.8681272297013932, "pass@128": 0.8858024691358025 }, "16": { "pass@1": 0.2906296091445428, "pass@2": 0.3808380078043341, "pass@4": 0.4639649562609984, "pass@8": 0.5348016643194988, "pass@16": 0.5918998310984556, "pass@32": 0.6414011766481142, "pass@64": 0.6911248562212414, "pass@128": 0.7433628318584071 }, "17": { "pass@1": 0.24602315512048192, "pass@2": 0.33004977587515394, "pass@4": 0.41139936376552333, "pass@8": 0.48491044209939826, "pass@16": 0.5510107149708241, "pass@32": 0.6105205843571422, "pass@64": 0.6619147865574031, "pass@128": 0.7048192771084337 }, "18": { "pass@1": 0.24233870967741936, "pass@2": 0.31930681673863354, "pass@4": 0.3933600641653665, "pass@8": 0.4584629930505219, "pass@16": 0.5141375485482685, "pass@32": 0.5606915077929782, "pass@64": 0.6005003708826756, "pass@128": 0.6387096774193548 }, "19": { "pass@1": 0.18074962235649547, "pass@2": 0.2564385243951755, "pass@4": 0.33101056353982955, "pass@8": 0.39293476132895816, "pass@16": 0.4417977642220465, "pass@32": 0.486950509381447, "pass@64": 0.5332543587477647, "pass@128": 0.5740181268882175 }, "20": { "pass@1": 0.1501266891891892, "pass@2": 0.22014177070771548, "pass@4": 0.29605818079046403, "pass@8": 0.36845622163096464, "pass@16": 0.4307900693887324, "pass@32": 0.4817076892328625, "pass@64": 0.5253941303577103, "pass@128": 0.5615615615615616 } } } } } ]