[ { "checkpoint": "checkpoint-22350", "id_acc": 0.9301788194444445, "ood_acc": 0.32165859375, "total_acc": 0.6099050164473684, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.3179861111111, "ood_avg_resp_len": 305.1192109375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 501082, "answer_accuracy": 0.6188275569079987, "avg_response_len": 242.87806769680682, "resp_tokens_sum": 196665172, "pass_at_k": { "pass@1": 0.6188275569079987, "pass@2": 0.6676054997996059, "pass@4": 0.7072495557581842, "pass@8": 0.7391898612522771, "pass@16": 0.7648119807124286, "pass@32": 0.7856728995183987, "pass@64": 0.8038704657760815, "pass@128": 0.8208978817578249 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8670953171091446, "pass@2": 0.9082642324576685, "pass@4": 0.9344302874751275, "pass@8": 0.9526031337738796, "pass@16": 0.9629272671248992, "pass@32": 0.9682270358052656, "pass@64": 0.9723204617269156, "pass@128": 0.976401179941003 }, "2": { "pass@1": 0.9599639797507789, "pass@2": 0.963836022395565, "pass@4": 0.967983610109484, "pass@8": 0.97280747375255, "pass@16": 0.9773608156099567, "pass@32": 0.9804489575520082, "pass@64": 0.9824826081902464, "pass@128": 0.9844236760124611 }, "3": { "pass@1": 0.9889860474006116, "pass@2": 0.9924104240410316, "pass@4": 0.9947170838163579, "pass@8": 0.9962247167758309, "pass@16": 0.9968678062677515, "pass@32": 0.9969414013686603, "pass@64": 0.9969418960237719, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.9893798828125, "pass@2": 0.99249969242126, "pass@4": 0.99361593257874, "pass@8": 0.993938088917673, "pass@16": 0.9941406211230228, "pass@32": 0.99453125, "pass@64": 0.9953125, "pass@128": 0.996875 }, "5": { "pass@1": 0.9679563492063492, "pass@2": 0.9771223909511312, "pass@4": 0.9827705200540406, "pass@8": 0.986222767171394, "pass@16": 0.9884563271556579, "pass@32": 0.9898710602540834, "pass@64": 0.9904321086955653, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.9334534056886228, "pass@2": 0.9554382868122028, "pass@4": 0.9689164457923299, "pass@8": 0.977704755737177, "pass@16": 0.9830688043745892, "pass@32": 0.986982241257549, "pass@64": 0.9900670186881447, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.9388406065088757, "pass@2": 0.9596263773703585, "pass@4": 0.9736909371165883, "pass@8": 0.9825845522685667, "pass@16": 0.9870793750133965, "pass@32": 0.9887747654939287, "pass@64": 0.9896445965776214, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.9217998798076923, "pass@2": 0.9474363706339591, "pass@4": 0.9611889871939082, "pass@8": 0.9712367571433583, "pass@16": 0.9788797736051013, "pass@32": 0.9848084145955653, "pass@64": 0.989404376252009, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.8873239436619719, "pass@2": 0.9320952783630917, "pass@4": 0.958994276683724, "pass@8": 0.9748610209136754, "pass@16": 0.9827990984511219, "pass@32": 0.9866721612838238, "pass@64": 0.9892717297087438, "pass@128": 0.9915492957746479 }, "11": { "pass@1": 0.7816041076487252, "pass@2": 0.8401684938993113, "pass@4": 0.8784991090879926, "pass@8": 0.9062103229646153, "pass@16": 0.928634342538452, "pass@32": 0.9474247894334389, "pass@64": 0.9635688141184316, "pass@128": 0.9773371104815864 }, "12": { "pass@1": 0.6643714689265536, "pass@2": 0.7570847374215929, "pass@4": 0.8195590050714002, "pass@8": 0.8638611475587994, "pass@16": 0.8949019817729846, "pass@32": 0.9149040498901152, "pass@64": 0.9297771688529582, "pass@128": 0.940677966101695 }, "13": { "pass@1": 0.4798932210031348, "pass@2": 0.5873326925925011, "pass@4": 0.6713887188474476, "pass@8": 0.7363040893811634, "pass@16": 0.7878785286983827, "pass@32": 0.8272839467742679, "pass@64": 0.8553831050059669, "pass@128": 0.8746081504702194 }, "14": { "pass@1": 0.31096670560747663, "pass@2": 0.40393041553217085, "pass@4": 0.48734351284360494, "pass@8": 0.5564510286229742, "pass@16": 0.613397458431016, "pass@32": 0.6620214973063218, "pass@64": 0.7095034575721406, "pass@128": 0.7601246105919003 }, "15": { "pass@1": 0.22798500722543352, "pass@2": 0.3055256076191342, "pass@4": 0.3797083284567752, "pass@8": 0.4463900949283278, "pass@16": 0.506202646502336, "pass@32": 0.5582980599444395, "pass@64": 0.6044086870879166, "pass@128": 0.6473988439306358 }, "16": { "pass@1": 0.2020665322580645, "pass@2": 0.272865601981204, "pass@4": 0.3412203653978737, "pass@8": 0.40107650120527977, "pass@16": 0.45179670722165527, "pass@32": 0.494565665067616, "pass@64": 0.5308362340060577, "pass@128": 0.5645161290322581 }, "17": { "pass@1": 0.19440932080924855, "pass@2": 0.2594631133084519, "pass@4": 0.3184659128591581, "pass@8": 0.36832649582409616, "pass@16": 0.4102210447790027, "pass@32": 0.44760686822032736, "pass@64": 0.48478608490693575, "pass@128": 0.5260115606936416 }, "18": { "pass@1": 0.1690625, "pass@2": 0.22736255624296942, "pass@4": 0.28266300096416525, "pass@8": 0.3321534924751763, "pass@16": 0.37531477121326395, "pass@32": 0.4134994424323006, "pass@64": 0.4477984271129805, "pass@128": 0.48 }, "19": { "pass@1": 0.1649540165165165, "pass@2": 0.22900225520796397, "pass@4": 0.28934828247820366, "pass@8": 0.3409590956948496, "pass@16": 0.38376503808474804, "pass@32": 0.41959872780770363, "pass@64": 0.44923215559385915, "pass@128": 0.4744744744744745 }, "20": { "pass@1": 0.1474756006006006, "pass@2": 0.20596506940010878, "pass@4": 0.26194670120063823, "pass@8": 0.3106229089309864, "pass@16": 0.3518269313013121, "pass@32": 0.3880737873045299, "pass@64": 0.42370540748975977, "pass@128": 0.4594594594594595 } } }, "teachers_in_school": { "count": 821120, "correct": 494355, "answer_accuracy": 0.6020496395167576, "avg_response_len": 249.6308298421668, "resp_tokens_sum": 204976867, "pass_at_k": { "pass@1": 0.6020496395167576, "pass@2": 0.6489359261941491, "pass@4": 0.6875917930124271, "pass@8": 0.720294447563385, "pass@16": 0.7485389992315395, "pass@32": 0.7730623756521464, "pass@64": 0.7946585235726484, "pass@128": 0.8138737334372564 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8187787576687117, "pass@2": 0.8631999208975412, "pass@4": 0.8889652101577791, "pass@8": 0.9047884008029274, "pass@16": 0.9158893168854205, "pass@32": 0.92519370344476, "pass@64": 0.9330087088515969, "pass@128": 0.9386503067484663 }, "2": { "pass@1": 0.9752089665653495, "pass@2": 0.9798478896919802, "pass@4": 0.9832519405818955, "pass@8": 0.9870105278631578, "pass@16": 0.9908648345867141, "pass@32": 0.9942662077657641, "pass@64": 0.996862131434271, "pass@128": 1.0 }, "3": { "pass@1": 0.9942945075757575, "pass@2": 0.9963180028632787, "pass@4": 0.9970773806683255, "pass@8": 0.9974988500184285, "pass@16": 0.9979768035033618, "pass@32": 0.9987316926293286, "pass@64": 0.9996301598663804, "pass@128": 1.0 }, "4": { "pass@1": 0.994221523668639, "pass@2": 0.9966635896659369, "pass@4": 0.9981680000798717, "pass@8": 0.9992769656375657, "pass@16": 0.9998620255616644, "pass@32": 0.9999962385209578, "pass@64": 0.999999999802363, "pass@128": 1.0 }, "5": { "pass@1": 0.9785326086956522, "pass@2": 0.9834570780554607, "pass@4": 0.9865280187802611, "pass@8": 0.9891545568020785, "pass@16": 0.9914041524744066, "pass@32": 0.9936531060212447, "pass@64": 0.996363467201046, "pass@128": 1.0 }, "6": { "pass@1": 0.921181053008596, "pass@2": 0.9443954041919544, "pass@4": 0.9595953014827298, "pass@8": 0.970122744120205, "pass@16": 0.9776500126894409, "pass@32": 0.9831185714417165, "pass@64": 0.9870883967891798, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.9094337066473989, "pass@2": 0.9308299861180648, "pass@4": 0.9450992366019274, "pass@8": 0.9571766432271784, "pass@16": 0.9673169836926407, "pass@32": 0.9756833949170814, "pass@64": 0.9829578324080214, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.8751760563380282, "pass@2": 0.912961974603527, "pass@4": 0.9388333774299338, "pass@8": 0.9557506365071035, "pass@16": 0.967302846071004, "pass@32": 0.9765007185796407, "pass@64": 0.9839139050837533, "pass@128": 0.9887323943661972 }, "9": { "pass@1": 0.8521940494011976, "pass@2": 0.9009305454052524, "pass@4": 0.931433134855149, "pass@8": 0.9526636585593052, "pass@16": 0.9679941054740805, "pass@32": 0.9783950530139177, "pass@64": 0.9854222638839946, "pass@128": 0.9910179640718563 }, "11": { "pass@1": 0.760706890060241, "pass@2": 0.827785181078645, "pass@4": 0.8710749813088118, "pass@8": 0.9003279944961914, "pass@16": 0.9204542771350541, "pass@32": 0.9356246993576314, "pass@64": 0.9476432584619248, "pass@128": 0.9548192771084337 }, "12": { "pass@1": 0.6126077586206896, "pass@2": 0.7043927257917213, "pass@4": 0.7676139801686229, "pass@8": 0.8119565346138998, "pass@16": 0.84612392537352, "pass@32": 0.875273652747933, "pass@64": 0.9018302739049897, "pass@128": 0.9216300940438872 }, "13": { "pass@1": 0.3867953431372549, "pass@2": 0.48342107512296223, "pass@4": 0.5674021487860234, "pass@8": 0.6372697246312006, "pass@16": 0.6963878858222297, "pass@32": 0.7491304071442696, "pass@64": 0.7981109820911362, "pass@128": 0.8431372549019608 }, "14": { "pass@1": 0.29877101608187134, "pass@2": 0.3703949526868352, "pass@4": 0.43101541802888665, "pass@8": 0.4865946482695511, "pass@16": 0.5415986897105027, "pass@32": 0.5971263312909335, "pass@64": 0.6528434464388628, "pass@128": 0.7046783625730995 }, "15": { "pass@1": 0.20783617424242423, "pass@2": 0.2801408524218563, "pass@4": 0.34825574416834243, "pass@8": 0.41106899030231925, "pass@16": 0.4680849755795498, "pass@32": 0.5172851517081245, "pass@64": 0.5596152305263337, "pass@128": 0.6 }, "16": { "pass@1": 0.19382122507122507, "pass@2": 0.25543441685173957, "pass@4": 0.3137022444843969, "pass@8": 0.36664821250482243, "pass@16": 0.4134268702032012, "pass@32": 0.45505377915145834, "pass@64": 0.49311278811442033, "pass@128": 0.5327635327635327 }, "17": { "pass@1": 0.19494856366459629, "pass@2": 0.25942795703526167, "pass@4": 0.3195814134452136, "pass@8": 0.375845879351016, "pass@16": 0.4307573783826269, "pass@32": 0.48238930194144775, "pass@64": 0.5288904166601099, "pass@128": 0.577639751552795 }, "18": { "pass@1": 0.16665900735294117, "pass@2": 0.22696524722093553, "pass@4": 0.2879940161891529, "pass@8": 0.3464190547517433, "pass@16": 0.3993496945302608, "pass@32": 0.4459605878543203, "pass@64": 0.4871371319920805, "pass@128": 0.5235294117647059 }, "19": { "pass@1": 0.14764694940476192, "pass@2": 0.20653436386857893, "pass@4": 0.2659328434950095, "pass@8": 0.3216636232453182, "pass@16": 0.36939694692013475, "pass@32": 0.4057203341614509, "pass@64": 0.4357478660293507, "pass@128": 0.46130952380952384 }, "20": { "pass@1": 0.1414436751497006, "pass@2": 0.1992263013343391, "pass@4": 0.25540902390943654, "pass@8": 0.3068920327741638, "pass@16": 0.3528934687646693, "pass@32": 0.39143395959153, "pass@64": 0.42026686041833455, "pass@128": 0.44011976047904194 } } }, "movie_festival_awards": { "count": 801152, "correct": 487852, "answer_accuracy": 0.6089381290941045, "avg_response_len": 268.44952143912764, "resp_tokens_sum": 215068871, "pass_at_k": { "pass@1": 0.6089381290941045, "pass@2": 0.6560205862612997, "pass@4": 0.695623838680178, "pass@8": 0.7291054496642166, "pass@16": 0.7572113855746511, "pass@32": 0.7810222919645945, "pass@64": 0.8016692044405078, "pass@128": 0.8197795174948075 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8403218283582089, "pass@2": 0.8831759019861319, "pass@4": 0.9079722738948675, "pass@8": 0.926427138967982, "pass@16": 0.9417213330071029, "pass@32": 0.9541837011986865, "pass@64": 0.9642190900906727, "pass@128": 0.9731343283582089 }, "2": { "pass@1": 0.9692410714285714, "pass@2": 0.9777741844769405, "pass@4": 0.9846253460281751, "pass@8": 0.9899382480241745, "pass@16": 0.9939865050728705, "pass@32": 0.9969488212990623, "pass@64": 0.9990814933468017, "pass@128": 1.0 }, "3": { "pass@1": 0.9933035714285714, "pass@2": 0.996758496705769, "pass@4": 0.9984645953512371, "pass@8": 0.999597081444929, "pass@16": 0.9999722102781452, "pass@32": 0.9999999117694275, "pass@64": 0.9999999999999893, "pass@128": 1.0 }, "4": { "pass@1": 0.9885325292397661, "pass@2": 0.9938638940691626, "pass@4": 0.996856481207393, "pass@8": 0.9989319154937989, "pass@16": 0.9998640737041085, "pass@32": 0.9999980433734141, "pass@64": 0.9999999999688675, "pass@128": 1.0 }, "5": { "pass@1": 0.9467830882352941, "pass@2": 0.9582825526864288, "pass@4": 0.9682023059617548, "pass@8": 0.9775505506527833, "pass@16": 0.9845769531894009, "pass@32": 0.9891263931155932, "pass@64": 0.9924383739804744, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.9375492902208202, "pass@2": 0.9561767244591276, "pass@4": 0.968417324608872, "pass@8": 0.9766326478533517, "pass@16": 0.9825492126854667, "pass@32": 0.987670213303071, "pass@64": 0.9928894756326212, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9044452136075949, "pass@2": 0.9300164924499152, "pass@4": 0.9481756708813297, "pass@8": 0.9610352164174365, "pass@16": 0.9694307347429958, "pass@32": 0.9756502423183432, "pass@64": 0.9817890720701818, "pass@128": 0.9873417721518988 }, "8": { "pass@1": 0.8971706081081081, "pass@2": 0.928977575015961, "pass@4": 0.9490096952970964, "pass@8": 0.9622935399559339, "pass@16": 0.9713158091696881, "pass@32": 0.9768844083204202, "pass@64": 0.9804212826643012, "pass@128": 0.984984984984985 }, "9": { "pass@1": 0.864022307073955, "pass@2": 0.9118283508874097, "pass@4": 0.9415764666273624, "pass@8": 0.9615825575217636, "pass@16": 0.9732077067762223, "pass@32": 0.9794089356361269, "pass@64": 0.9838241901688739, "pass@128": 0.9903536977491961 }, "11": { "pass@1": 0.7715277777777778, "pass@2": 0.8335602580927388, "pass@4": 0.8746409222656686, "pass@8": 0.9043520584352333, "pass@16": 0.9280951884818978, "pass@32": 0.9469414841706554, "pass@64": 0.9617656531177552, "pass@128": 0.9714285714285714 }, "12": { "pass@1": 0.6685540902140673, "pass@2": 0.750171566856895, "pass@4": 0.8076844448343031, "pass@8": 0.8501109077517056, "pass@16": 0.8825882158353087, "pass@32": 0.9082979572669679, "pass@64": 0.9282816365658905, "pass@128": 0.9418960244648318 }, "13": { "pass@1": 0.4363184799382716, "pass@2": 0.5391692032419559, "pass@4": 0.6226500897457258, "pass@8": 0.6898887508264963, "pass@16": 0.7448552505843882, "pass@32": 0.7913198741989679, "pass@64": 0.831231401781366, "pass@128": 0.8641975308641975 }, "14": { "pass@1": 0.29532177299703266, "pass@2": 0.3794510385756675, "pass@4": 0.4581800039275504, "pass@8": 0.5274378283252875, "pass@16": 0.5883019565685669, "pass@32": 0.6434770492088578, "pass@64": 0.6938611977483916, "pass@128": 0.7388724035608308 }, "15": { "pass@1": 0.1781684027777778, "pass@2": 0.2488718315592497, "pass@4": 0.323979556085582, "pass@8": 0.3959601546379987, "pass@16": 0.4615711022760134, "pass@32": 0.5171673358865758, "pass@64": 0.5637801348069706, "pass@128": 0.6111111111111112 }, "16": { "pass@1": 0.2089325221238938, "pass@2": 0.27069505899705004, "pass@4": 0.32480351792309137, "pass@8": 0.3727821911266633, "pass@16": 0.4175536083009712, "pass@32": 0.46334050373812846, "pass@64": 0.5105570019459841, "pass@128": 0.5545722713864307 }, "17": { "pass@1": 0.16488610692771086, "pass@2": 0.22542482923821255, "pass@4": 0.2861592093909947, "pass@8": 0.3442176842200421, "pass@16": 0.39808977250495636, "pass@32": 0.44461433904524217, "pass@64": 0.4821225595080291, "pass@128": 0.5090361445783133 }, "18": { "pass@1": 0.18180443548387096, "pass@2": 0.251581946913894, "pass@4": 0.3183337878128136, "pass@8": 0.37251228294233507, "pass@16": 0.4128352414166689, "pass@32": 0.44457500793368165, "pass@64": 0.47167915157723894, "pass@128": 0.4935483870967742 }, "19": { "pass@1": 0.15301642749244712, "pass@2": 0.21020693079905794, "pass@4": 0.26659490533849434, "pass@8": 0.31827335611728563, "pass@16": 0.36212186502412136, "pass@32": 0.3976268046073573, "pass@64": 0.4256205035352982, "pass@128": 0.4501510574018127 }, "20": { "pass@1": 0.13992117117117117, "pass@2": 0.1956980356340592, "pass@4": 0.2531280616949908, "pass@8": 0.3096118515891317, "pass@16": 0.36257072436356236, "pass@32": 0.411466262375835, "pass@64": 0.45802399495121837, "pass@128": 0.5045045045045045 } } } } } ]