[ { "checkpoint": "checkpoint-21808", "id_acc": 0.9517916666666667, "ood_acc": 0.3380640625, "total_acc": 0.6287771381578947, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.2878142361111, "ood_avg_resp_len": 313.0753828125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 505986, "answer_accuracy": 0.6248839116345242, "avg_response_len": 244.86610071530194, "resp_tokens_sum": 198274938, "pass_at_k": { "pass@1": 0.6248839116345242, "pass@2": 0.672510150584643, "pass@4": 0.711296164429809, "pass@8": 0.7421341642455704, "pass@16": 0.766649491777681, "pass@32": 0.7871263323219219, "pass@64": 0.8054991086246542, "pass@128": 0.8226367372747392 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8832964601769911, "pass@2": 0.9172103279678535, "pass@4": 0.937207035237852, "pass@8": 0.9511780335899438, "pass@16": 0.9620826801919633, "pass@32": 0.9708490352016442, "pass@64": 0.978042327328383, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 0.9566053348909658, "pass@2": 0.962186787107219, "pass@4": 0.9660673160597917, "pass@8": 0.9685307941081832, "pass@16": 0.9705478781887977, "pass@32": 0.9729358037598592, "pass@64": 0.9762551112290103, "pass@128": 0.9813084112149533 }, "3": { "pass@1": 0.9943377293577982, "pass@2": 0.9960855667124179, "pass@4": 0.9972139947529495, "pass@8": 0.9981468940282231, "pass@16": 0.9989822280288635, "pass@32": 0.9997163691810725, "pass@64": 0.9999905325251474, "pass@128": 1.0 }, "4": { "pass@1": 0.9753662109375, "pass@2": 0.9802957369586615, "pass@4": 0.983529715621485, "pass@8": 0.9857348060465551, "pass@16": 0.9876058181840719, "pass@32": 0.9903074532142714, "pass@64": 0.9941683070866141, "pass@128": 0.996875 }, "5": { "pass@1": 0.934672619047619, "pass@2": 0.9468152418447692, "pass@4": 0.9547582409341686, "pass@8": 0.9609408470768459, "pass@16": 0.9669133908826691, "pass@32": 0.9727478686315325, "pass@64": 0.9775761611445652, "pass@128": 0.9809523809523809 }, "6": { "pass@1": 0.9568909056886228, "pass@2": 0.9743243581734176, "pass@4": 0.982969407080103, "pass@8": 0.9869105050103905, "pass@16": 0.9889078989792498, "pass@32": 0.9902473376106875, "pass@64": 0.9909312655090486, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.9426081730769231, "pass@2": 0.9565109141312955, "pass@4": 0.9643415889581849, "pass@8": 0.9704004882673377, "pass@16": 0.9755356821468868, "pass@32": 0.9802742371656721, "pass@64": 0.9851040508965963, "pass@128": 0.9881656804733728 }, "8": { "pass@1": 0.9438351362179487, "pass@2": 0.9649525855542097, "pass@4": 0.9742177161749009, "pass@8": 0.9796460229237937, "pass@16": 0.9837141372979744, "pass@32": 0.9876441458962127, "pass@64": 0.990970237640019, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.9236355633802817, "pass@2": 0.9530761339691695, "pass@4": 0.9679002519051312, "pass@8": 0.9780128398804888, "pass@16": 0.9860372032987277, "pass@32": 0.9927718536917328, "pass@64": 0.9977068384538204, "pass@128": 1.0 }, "11": { "pass@1": 0.8158640226628895, "pass@2": 0.8690087495259979, "pass@4": 0.9003459481693683, "pass@8": 0.9208178113433934, "pass@16": 0.936255057279863, "pass@32": 0.9486361034628609, "pass@64": 0.9589588726836659, "pass@128": 0.9688385269121813 }, "12": { "pass@1": 0.6557203389830508, "pass@2": 0.7513526513634943, "pass@4": 0.8129071535549578, "pass@8": 0.8537187806022632, "pass@16": 0.8834961886746142, "pass@32": 0.9062902420548817, "pass@64": 0.925781100999793, "pass@128": 0.943502824858757 }, "13": { "pass@1": 0.4441859326018809, "pass@2": 0.537450170315701, "pass@4": 0.612993540286304, "pass@8": 0.6709888225301639, "pass@16": 0.7145216319389676, "pass@32": 0.7501344384006141, "pass@64": 0.7802929625106815, "pass@128": 0.8025078369905956 }, "14": { "pass@1": 0.34971281152647976, "pass@2": 0.4340566512129908, "pass@4": 0.503801084735903, "pass@8": 0.559338023432466, "pass@16": 0.6046153041994793, "pass@32": 0.6462515330765554, "pass@64": 0.6884805551577264, "pass@128": 0.7320872274143302 }, "15": { "pass@1": 0.27420520231213874, "pass@2": 0.3648961272814163, "pass@4": 0.4489088842218423, "pass@8": 0.5161746732770215, "pass@16": 0.5669481987005399, "pass@32": 0.6095912062248127, "pass@64": 0.6513023988475989, "pass@128": 0.6936416184971098 }, "16": { "pass@1": 0.19057459677419356, "pass@2": 0.25924601536703096, "pass@4": 0.3243514381266857, "pass@8": 0.3820198056073008, "pass@16": 0.43185101815612564, "pass@32": 0.4727802423483779, "pass@64": 0.507659460573341, "pass@128": 0.5451612903225806 }, "17": { "pass@1": 0.1830970014450867, "pass@2": 0.25075561251194756, "pass@4": 0.3177439135137008, "pass@8": 0.3787659856147468, "pass@16": 0.43273696124530503, "pass@32": 0.4781735761884427, "pass@64": 0.5152145582863207, "pass@128": 0.546242774566474 }, "18": { "pass@1": 0.17191964285714287, "pass@2": 0.2386410292463441, "pass@4": 0.302181122181156, "pass@8": 0.35631130956435, "pass@16": 0.4024186189231844, "pass@32": 0.44305521959331307, "pass@64": 0.4812509405361627, "pass@128": 0.52 }, "19": { "pass@1": 0.15561655405405406, "pass@2": 0.22392286479392776, "pass@4": 0.29112672092790204, "pass@8": 0.3476218784231473, "pass@16": 0.3900725810744725, "pass@32": 0.42293491455607535, "pass@64": 0.45500925055441316, "pass@128": 0.4894894894894895 }, "20": { "pass@1": 0.1527777777777778, "pass@2": 0.22137873601948407, "pass@4": 0.29151925350547414, "pass@8": 0.3513487308384057, "pass@16": 0.39666827573276253, "pass@32": 0.43178798807608976, "pass@64": 0.46000330337155665, "pass@128": 0.4804804804804805 } } }, "teachers_in_school": { "count": 821120, "correct": 514444, "answer_accuracy": 0.6265150038971161, "avg_response_len": 254.71906298713952, "resp_tokens_sum": 209154917, "pass_at_k": { "pass@1": 0.6265150038971161, "pass@2": 0.6711105745331151, "pass@4": 0.7087658103970431, "pass@8": 0.7405722739721018, "pass@16": 0.7674043940596819, "pass@32": 0.7905383572324718, "pass@64": 0.8108683024339327, "pass@128": 0.8286827747466875 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8846338190184049, "pass@2": 0.9194517324042314, "pass@4": 0.9400236116957766, "pass@8": 0.9542828313525812, "pass@16": 0.9646392990472547, "pass@32": 0.9717789665917326, "pass@64": 0.9761131191473751, "pass@128": 0.9785276073619632 }, "2": { "pass@1": 0.9572568389057751, "pass@2": 0.9598992114017663, "pass@4": 0.9626694922063312, "pass@8": 0.9658379624318189, "pass@16": 0.9690636373795826, "pass@32": 0.972155719959404, "pass@64": 0.9757077858616603, "pass@128": 0.9787234042553191 }, "3": { "pass@1": 0.9955965909090909, "pass@2": 0.9970606657122405, "pass@4": 0.9975789384281509, "pass@8": 0.9980929433414543, "pass@16": 0.9988386081686187, "pass@32": 0.9996178951468804, "pass@64": 0.9999800850835996, "pass@128": 1.0 }, "4": { "pass@1": 0.9837740384615384, "pass@2": 0.9871876164795229, "pass@4": 0.988965550626438, "pass@8": 0.9903578338731447, "pass@16": 0.9910320953813216, "pass@32": 0.9911233149752038, "pass@64": 0.9911242603502997, "pass@128": 0.9911242603550295 }, "5": { "pass@1": 0.9695199275362318, "pass@2": 0.9757246376811596, "pass@4": 0.9795172891432047, "pass@8": 0.9828762612538346, "pass@16": 0.9857687021768708, "pass@32": 0.9885129019107062, "pass@64": 0.9915973976452171, "pass@128": 0.9942028985507246 }, "6": { "pass@1": 0.9840168338108882, "pass@2": 0.995195426189563, "pass@4": 0.9990684802472753, "pass@8": 0.9999207029235206, "pass@16": 0.9999989480294558, "pass@32": 0.9999999999297677, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9557893786127167, "pass@2": 0.9670289813845525, "pass@4": 0.9733226897035272, "pass@8": 0.9776014792969531, "pass@16": 0.9806254260386871, "pass@32": 0.9831697369077875, "pass@64": 0.9850731496000922, "pass@128": 0.9855491329479769 }, "8": { "pass@1": 0.9510343309859155, "pass@2": 0.9731552206942445, "pass@4": 0.9841903806038328, "pass@8": 0.9894380107543043, "pass@16": 0.9924766599199953, "pass@32": 0.9949998418278819, "pass@64": 0.9975485092564174, "pass@128": 1.0 }, "9": { "pass@1": 0.9023203592814372, "pass@2": 0.9278947027205433, "pass@4": 0.9418790857542502, "pass@8": 0.95232354181696, "pass@16": 0.9613164420633297, "pass@32": 0.9708170510530576, "pass@64": 0.9803618880355766, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.8197242093373494, "pass@2": 0.879619980907884, "pass@4": 0.9152084375183525, "pass@8": 0.9365968410373897, "pass@16": 0.9502230602590158, "pass@32": 0.959630325530834, "pass@64": 0.9672099492738422, "pass@128": 0.9759036144578314 }, "12": { "pass@1": 0.6610011755485894, "pass@2": 0.7461709821538761, "pass@4": 0.8065136687459518, "pass@8": 0.8485894222226024, "pass@16": 0.8752288825205958, "pass@32": 0.8914381380619772, "pass@64": 0.9036142400296753, "pass@128": 0.9153605015673981 }, "13": { "pass@1": 0.38804271708683474, "pass@2": 0.4778598447252919, "pass@4": 0.5540819498403033, "pass@8": 0.619142374924008, "pass@16": 0.6772598039791219, "pass@32": 0.7320106065745692, "pass@64": 0.781803259607603, "pass@128": 0.8207282913165266 }, "14": { "pass@1": 0.3611796418128655, "pass@2": 0.4434256573191508, "pass@4": 0.5109200042428902, "pass@8": 0.5672925734838943, "pass@16": 0.6168776944628133, "pass@32": 0.6620983039384781, "pass@64": 0.70456770688268, "pass@128": 0.7456140350877193 }, "15": { "pass@1": 0.2217092803030303, "pass@2": 0.3015707020997377, "pass@4": 0.37985566946177185, "pass@8": 0.4510174568078622, "pass@16": 0.5158476685485044, "pass@32": 0.5753658130979193, "pass@64": 0.6249801268738032, "pass@128": 0.6666666666666666 }, "16": { "pass@1": 0.2050391737891738, "pass@2": 0.2738355121475203, "pass@4": 0.338386615668768, "pass@8": 0.3932875901414371, "pass@16": 0.43886406008834783, "pass@32": 0.4774662643312779, "pass@64": 0.5142127541987865, "pass@128": 0.5498575498575499 }, "17": { "pass@1": 0.1947059394409938, "pass@2": 0.2682923381180613, "pass@4": 0.3440897881747702, "pass@8": 0.41833600483808464, "pass@16": 0.48600653081169176, "pass@32": 0.5444368907837626, "pass@64": 0.5933865414431194, "pass@128": 0.6335403726708074 }, "18": { "pass@1": 0.1689797794117647, "pass@2": 0.23628271190365893, "pass@4": 0.3063349933280399, "pass@8": 0.37162887977886394, "pass@16": 0.4290596832304357, "pass@32": 0.4805349660760809, "pass@64": 0.5257714119613119, "pass@128": 0.5647058823529412 }, "19": { "pass@1": 0.14950706845238096, "pass@2": 0.21081337000843647, "pass@4": 0.27120366957478526, "pass@8": 0.32483243652106647, "pass@16": 0.3707793523131844, "pass@32": 0.4109668939345245, "pass@64": 0.4485305995646864, "pass@128": 0.48214285714285715 }, "20": { "pass@1": 0.13929172904191617, "pass@2": 0.20208254750341842, "pass@4": 0.2660832878674596, "pass@8": 0.32449652847649324, "pass@16": 0.37296661066376247, "pass@32": 0.4105719174561622, "pass@64": 0.4410038284084724, "pass@128": 0.47005988023952094 } } }, "movie_festival_awards": { "count": 801152, "correct": 508756, "answer_accuracy": 0.6350305559993609, "avg_response_len": 273.8933398406295, "resp_tokens_sum": 219430197, "pass_at_k": { "pass@1": 0.6350305559993609, "pass@2": 0.6817842149823968, "pass@4": 0.7211618787655941, "pass@8": 0.7535477416864765, "pass@16": 0.7797154196182626, "pass@32": 0.8009384390103981, "pass@64": 0.8184491248628813, "pass@128": 0.8332001917239176 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8779384328358208, "pass@2": 0.919579636267481, "pass@4": 0.9469269764786861, "pass@8": 0.9633563596612335, "pass@16": 0.9729431971434862, "pass@32": 0.9793303717895566, "pass@64": 0.9847047119439397, "pass@128": 0.9880597014925373 }, "2": { "pass@1": 0.9700892857142858, "pass@2": 0.9764303290213722, "pass@4": 0.9802661671755313, "pass@8": 0.9824211322852868, "pass@16": 0.984386055526147, "pass@32": 0.9867049961682504, "pass@64": 0.988322267277566, "pass@128": 0.9885714285714285 }, "3": { "pass@1": 0.9918913994169096, "pass@2": 0.9946228421064712, "pass@4": 0.996332737835613, "pass@8": 0.9970006677667912, "pass@16": 0.9970835775181903, "pass@32": 0.9970845480611684, "pass@64": 0.9970845481049563, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.9822277046783626, "pass@2": 0.9848502181470739, "pass@4": 0.9881618657316958, "pass@8": 0.9918373424468889, "pass@16": 0.9948832071878148, "pass@32": 0.996599044812975, "pass@64": 0.9970563844205709, "pass@128": 0.9970760233918129 }, "5": { "pass@1": 0.9504136029411765, "pass@2": 0.9648897058823532, "pass@4": 0.9740956067072498, "pass@8": 0.9794277261657189, "pass@16": 0.9827043544002162, "pass@32": 0.985393110798894, "pass@64": 0.9876163898807621, "pass@128": 0.9882352941176471 }, "6": { "pass@1": 0.9810725552050473, "pass@2": 0.9868364061452098, "pass@4": 0.9897118630084484, "pass@8": 0.992015798703835, "pass@16": 0.9933635236323332, "pass@32": 0.9936785012993264, "pass@64": 0.9936908487339506, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.9652145965189873, "pass@2": 0.9762272781072461, "pass@4": 0.981000794402282, "pass@8": 0.9844839762371538, "pass@16": 0.988252756425543, "pass@32": 0.9920258959912787, "pass@64": 0.9952680507296997, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9480105105105106, "pass@2": 0.9651595493130931, "pass@4": 0.9750546725015227, "pass@8": 0.9822405401725065, "pass@16": 0.9875073932275126, "pass@32": 0.9912748387820756, "pass@64": 0.9934950075146307, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.934887459807074, "pass@2": 0.9595855381421378, "pass@4": 0.9720115560313806, "pass@8": 0.9803090736757587, "pass@16": 0.9863824259753894, "pass@32": 0.9910160432545946, "pass@64": 0.9944976406378414, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.8351438492063492, "pass@2": 0.888676806024247, "pass@4": 0.9195518057266644, "pass@8": 0.9383458921358946, "pass@16": 0.9516784351429641, "pass@32": 0.9627085924506177, "pass@64": 0.9702310379963924, "pass@128": 0.9746031746031746 }, "12": { "pass@1": 0.6418195718654435, "pass@2": 0.7379478045462207, "pass@4": 0.808911574699951, "pass@8": 0.8574357117385621, "pass@16": 0.8891356835786846, "pass@32": 0.9119187657061124, "pass@64": 0.9319113224555994, "pass@128": 0.9480122324159022 }, "13": { "pass@1": 0.43190586419753085, "pass@2": 0.5315685908185085, "pass@4": 0.6109124663815172, "pass@8": 0.6731000469060026, "pass@16": 0.7230315632688672, "pass@32": 0.7608469845357523, "pass@64": 0.7895432659192605, "pass@128": 0.8148148148148148 }, "14": { "pass@1": 0.3707807863501484, "pass@2": 0.4622490449543215, "pass@4": 0.5422512678311351, "pass@8": 0.6090936582126327, "pass@16": 0.665375170025168, "pass@32": 0.7157854633923258, "pass@64": 0.7613425230053639, "pass@128": 0.8011869436201781 }, "15": { "pass@1": 0.29398148148148145, "pass@2": 0.385362745455429, "pass@4": 0.46875469646155304, "pass@8": 0.5418755427677018, "pass@16": 0.6032998652447866, "pass@32": 0.6504141207197476, "pass@64": 0.6830392953220654, "pass@128": 0.7037037037037037 }, "16": { "pass@1": 0.20464601769911506, "pass@2": 0.27441648955938025, "pass@4": 0.3406489202677099, "pass@8": 0.3980154635469927, "pass@16": 0.44783345192625446, "pass@32": 0.4942909375581759, "pass@64": 0.5381229534278857, "pass@128": 0.5752212389380531 }, "17": { "pass@1": 0.17634600903614459, "pass@2": 0.24603556944312685, "pass@4": 0.3129089391875711, "pass@8": 0.37151393806373156, "pass@16": 0.42139763379994655, "pass@32": 0.4651711352279267, "pass@64": 0.5050089880673075, "pass@128": 0.5451807228915663 }, "18": { "pass@1": 0.17515120967741934, "pass@2": 0.2418537274574549, "pass@4": 0.30864086051743567, "pass@8": 0.36897845390139183, "pass@16": 0.41813086287003537, "pass@32": 0.45496716623619304, "pass@64": 0.48441455201866307, "pass@128": 0.5161290322580645 }, "19": { "pass@1": 0.1568400679758308, "pass@2": 0.22048024656849932, "pass@4": 0.28429913613517355, "pass@8": 0.34339241837815315, "pass@16": 0.3939768884772223, "pass@32": 0.4328402195492693, "pass@64": 0.4613644571598135, "pass@128": 0.48338368580060426 }, "20": { "pass@1": 0.15636730480480482, "pass@2": 0.22079904707857453, "pass@4": 0.2881610339248135, "pass@8": 0.3512828123002748, "pass@16": 0.40352597323708034, "pass@32": 0.4467676743469794, "pass@64": 0.4848656649074429, "pass@128": 0.5195195195195195 } } } } } ]