[ { "checkpoint": "checkpoint-21546", "id_acc": 0.9743967013888889, "ood_acc": 0.39801640625, "total_acc": 0.6710386513157894, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.49886805555556, "ood_avg_resp_len": 321.86122421875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 540343, "answer_accuracy": 0.6673142092159342, "avg_response_len": 252.07113746838445, "resp_tokens_sum": 204109058, "pass_at_k": { "pass@1": 0.6673142092159342, "pass@2": 0.7184972467083713, "pass@4": 0.7582710489269354, "pass@8": 0.7898036372954599, "pass@16": 0.8149666354226288, "pass@32": 0.8356683187114132, "pass@64": 0.8536326257414687, "pass@128": 0.8695858362314258 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9078401548672567, "pass@2": 0.9451660598564567, "pass@4": 0.9627415558121603, "pass@8": 0.9724228068912785, "pass@16": 0.9786434853777529, "pass@32": 0.9833303999293327, "pass@64": 0.9874800169721869, "pass@128": 0.9911504424778761 }, "2": { "pass@1": 0.9968847352024922, "pass@2": 0.9968847352024922, "pass@4": 0.9968847352024922, "pass@8": 0.9968847352024922, "pass@16": 0.9968847352024922, "pass@32": 0.9968847352024922, "pass@64": 0.9968847352024922, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.9974913990825688, "pass@2": 0.998993925931277, "pass@4": 0.999781667199857, "pass@8": 0.9999876849218283, "pass@16": 0.9999999719746717, "pass@32": 0.9999999999999853, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9863525390625, "pass@2": 0.9885619156003939, "pass@4": 0.9898306295697413, "pass@8": 0.9915581459135234, "pass@16": 0.9938875771099299, "pass@32": 0.996529537062657, "pass@64": 0.9989487041172099, "pass@128": 1.0 }, "5": { "pass@1": 0.9679315476190476, "pass@2": 0.9789893450818652, "pass@4": 0.9852945253867076, "pass@8": 0.9903654763094896, "pass@16": 0.9944361907939756, "pass@32": 0.9969451642580184, "pass@64": 0.9983683333808681, "pass@128": 1.0 }, "6": { "pass@1": 0.9894975673652695, "pass@2": 0.996165027229007, "pass@4": 0.998533353147473, "pass@8": 0.999620189417031, "pass@16": 0.9999636861019171, "pass@32": 0.9999998029761207, "pass@64": 0.999999999999911, "pass@128": 1.0 }, "7": { "pass@1": 0.9860392011834319, "pass@2": 0.9959920124167174, "pass@4": 0.9977027054310518, "pass@8": 0.9983763394978522, "pass@16": 0.9991472297394309, "pass@32": 0.9997987731838602, "pass@64": 0.999995725631176, "pass@128": 1.0 }, "8": { "pass@1": 0.9701522435897436, "pass@2": 0.9889413612961848, "pass@4": 0.9943698822502955, "pass@8": 0.9962621361835249, "pass@16": 0.9970686948727937, "pass@32": 0.9975932975941004, "pass@64": 0.998397435811393, "pass@128": 1.0 }, "9": { "pass@1": 0.9600132042253521, "pass@2": 0.9878528058112463, "pass@4": 0.9946557779749362, "pass@8": 0.9974029094078771, "pass@16": 0.9989479245309896, "pass@32": 0.9997375771890578, "pass@64": 0.9999912792521506, "pass@128": 1.0 }, "11": { "pass@1": 0.8549043909348442, "pass@2": 0.9105383272735389, "pass@4": 0.9394676640101288, "pass@8": 0.9567641672474922, "pass@16": 0.9689281622131964, "pass@32": 0.9792995108322007, "pass@64": 0.989377221593619, "pass@128": 1.0 }, "12": { "pass@1": 0.7604387358757062, "pass@2": 0.8512497080608568, "pass@4": 0.9022677093859018, "pass@8": 0.9306185243521176, "pass@16": 0.947629076857396, "pass@32": 0.9613728628093047, "pass@64": 0.9722807480889516, "pass@128": 0.9774011299435028 }, "13": { "pass@1": 0.6151792711598746, "pass@2": 0.7272553717325303, "pass@4": 0.8012909736204601, "pass@8": 0.8513849982189186, "pass@16": 0.8858777402360082, "pass@32": 0.9113844472956131, "pass@64": 0.9327076612903673, "pass@128": 0.9529780564263323 }, "14": { "pass@1": 0.4572381230529595, "pass@2": 0.5804936437559792, "pass@4": 0.6836558835519382, "pass@8": 0.7584661690159947, "pass@16": 0.8090695107690499, "pass@32": 0.8452244630136615, "pass@64": 0.8744698913107548, "pass@128": 0.9034267912772586 }, "15": { "pass@1": 0.3591943641618497, "pass@2": 0.46582035694779467, "pass@4": 0.557234378759013, "pass@8": 0.6333810374573632, "pass@16": 0.6991402754904769, "pass@32": 0.7537907497738942, "pass@64": 0.7989936415700499, "pass@128": 0.838150289017341 }, "16": { "pass@1": 0.2480594758064516, "pass@2": 0.3369277051054099, "pass@4": 0.41882028587152403, "pass@8": 0.48785088132261173, "pass@16": 0.5464977216622237, "pass@32": 0.5997006252636178, "pass@64": 0.6464878664953922, "pass@128": 0.6838709677419355 }, "17": { "pass@1": 0.19413836705202311, "pass@2": 0.2639743866915478, "pass@4": 0.3301886702384742, "pass@8": 0.3913913054415934, "pass@16": 0.44674849852516724, "pass@32": 0.49537678199969953, "pass@64": 0.5397606358748235, "pass@128": 0.5809248554913294 }, "18": { "pass@1": 0.16417410714285716, "pass@2": 0.23900625703037112, "pass@4": 0.31737553055868006, "pass@8": 0.3877756897146819, "pass@16": 0.44484807064643955, "pass@32": 0.4911672177360723, "pass@64": 0.5336450417048393, "pass@128": 0.5771428571428572 }, "19": { "pass@1": 0.1477805930930931, "pass@2": 0.21373120167411505, "pass@4": 0.2819873895267596, "pass@8": 0.3444790109299255, "pass@16": 0.398624559518731, "pass@32": 0.4448169543649647, "pass@64": 0.48421600118684854, "pass@128": 0.5165165165165165 }, "20": { "pass@1": 0.15219125375375375, "pass@2": 0.21601005533092138, "pass@4": 0.2833787189889551, "pass@8": 0.34737669283247447, "pass@16": 0.40206167741449067, "pass@32": 0.4470107741738278, "pass@64": 0.4875770578788106, "pass@128": 0.5225225225225225 } } }, "teachers_in_school": { "count": 821120, "correct": 550391, "answer_accuracy": 0.6702930144193296, "avg_response_len": 260.58617132696804, "resp_tokens_sum": 213972517, "pass_at_k": { "pass@1": 0.6702930144193296, "pass@2": 0.7203984686174745, "pass@4": 0.7605172725681272, "pass@8": 0.7936184035372785, "pass@16": 0.8205157430956861, "pass@32": 0.8418489124402874, "pass@64": 0.8581997269526279, "pass@128": 0.8693686671862821 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8947948619631901, "pass@2": 0.9342487228877832, "pass@4": 0.9553914430174755, "pass@8": 0.9685340029247087, "pass@16": 0.9778976377826292, "pass@32": 0.9854515349169342, "pass@64": 0.991566746760942, "pass@128": 0.9938650306748467 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9943181818181818, "pass@2": 0.9965987383679314, "pass@4": 0.9986907074115734, "pass@8": 0.9997900403945096, "pass@16": 0.999993778423042, "pass@32": 0.9999999968162022, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986825073964497, "pass@2": 0.9998864324651727, "pass@4": 0.9999964917477031, "pass@8": 0.999999997761824, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9817028985507247, "pass@2": 0.9890858581536004, "pass@4": 0.9919462368834332, "pass@8": 0.9941619355764031, "pass@16": 0.9961781821357537, "pass@32": 0.9975886476682482, "pass@64": 0.9985464572309225, "pass@128": 1.0 }, "6": { "pass@1": 0.9912249283667621, "pass@2": 0.9971071734765248, "pass@4": 0.998236967692792, "pass@8": 0.9989742809016088, "pass@16": 0.9996596953889924, "pass@32": 0.9999716397876668, "pass@64": 0.9999999654169256, "pass@128": 1.0 }, "7": { "pass@1": 0.9866781069364162, "pass@2": 0.9940098595876383, "pass@4": 0.9954577830950322, "pass@8": 0.9965385997356417, "pass@16": 0.997911202425725, "pass@32": 0.9992275285274425, "pass@64": 0.9999158252269373, "pass@128": 1.0 }, "8": { "pass@1": 0.9713248239436619, "pass@2": 0.9873073084174341, "pass@4": 0.9928848371066289, "pass@8": 0.9962645987397817, "pass@16": 0.9977159520362265, "pass@32": 0.9984178461141198, "pass@64": 0.9993013196994364, "pass@128": 1.0 }, "9": { "pass@1": 0.9549728667664671, "pass@2": 0.9786595607996605, "pass@4": 0.9846184301812573, "pass@8": 0.9881064318849376, "pass@16": 0.9915007486161633, "pass@32": 0.9943169158716935, "pass@64": 0.996171853688097, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.8858951430722891, "pass@2": 0.9403234246039275, "pass@4": 0.9654911386264966, "pass@8": 0.9802103258206285, "pass@16": 0.9892633104282124, "pass@32": 0.9943230187012654, "pass@64": 0.9966323831791278, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.770865987460815, "pass@2": 0.8529438081603437, "pass@4": 0.8969163801625106, "pass@8": 0.9239001333151642, "pass@16": 0.944273209688222, "pass@32": 0.9607568063656658, "pass@64": 0.9735690051847953, "pass@128": 0.9843260188087775 }, "13": { "pass@1": 0.55249912464986, "pass@2": 0.670897295926244, "pass@4": 0.7573695231373386, "pass@8": 0.8195883022487538, "pass@16": 0.8639400641786898, "pass@32": 0.8959443632193429, "pass@64": 0.9194230288568553, "pass@128": 0.938375350140056 }, "14": { "pass@1": 0.4458607456140351, "pass@2": 0.5551821441727679, "pass@4": 0.6490961617736377, "pass@8": 0.7264317974040324, "pass@16": 0.7878208627637026, "pass@32": 0.8337530255493398, "pass@64": 0.8650871772849, "pass@128": 0.8830409356725146 }, "15": { "pass@1": 0.3692708333333333, "pass@2": 0.4844406167979, "pass@4": 0.5807708542114053, "pass@8": 0.6566659618047733, "pass@16": 0.71822453037185, "pass@32": 0.7690809020128775, "pass@64": 0.8074737398378564, "pass@128": 0.8272727272727273 }, "16": { "pass@1": 0.2302795584045584, "pass@2": 0.3132508075913587, "pass@4": 0.3953974561940442, "pass@8": 0.4699005180824398, "pass@16": 0.5316027123774437, "pass@32": 0.5810105949043947, "pass@64": 0.6208832644017411, "pass@128": 0.6495726495726496 }, "17": { "pass@1": 0.21753687888198758, "pass@2": 0.29977732063383367, "pass@4": 0.3802775075281738, "pass@8": 0.4530973687186417, "pass@16": 0.5146178409330402, "pass@32": 0.5636207169721692, "pass@64": 0.6026909514942224, "pass@128": 0.6335403726708074 }, "18": { "pass@1": 0.19386488970588237, "pass@2": 0.27067797591477505, "pass@4": 0.34721876282008873, "pass@8": 0.4179357468470566, "pass@16": 0.4785538356301472, "pass@32": 0.5267746944091646, "pass@64": 0.5637993560978994, "pass@128": 0.5882352941176471 }, "19": { "pass@1": 0.14381045386904762, "pass@2": 0.20448420509936244, "pass@4": 0.26923672822147243, "pass@8": 0.3304539572282448, "pass@16": 0.3857649133616538, "pass@32": 0.4348844940956561, "pass@64": 0.4749150124076846, "pass@128": 0.5029761904761905 }, "20": { "pass@1": 0.140812125748503, "pass@2": 0.2086588388184262, "pass@4": 0.2810939201462094, "pass@8": 0.3485268808493757, "pass@16": 0.4055889275550898, "pass@32": 0.45131218202379575, "pass@64": 0.4876867028458811, "pass@128": 0.5149700598802395 } } }, "movie_festival_awards": { "count": 801152, "correct": 541232, "answer_accuracy": 0.6755671832561112, "avg_response_len": 280.69016616072855, "resp_tokens_sum": 224875488, "pass_at_k": { "pass@1": 0.6755671832561112, "pass@2": 0.7267714805640489, "pass@4": 0.7671935018298011, "pass@8": 0.7993583792277895, "pass@16": 0.8249747156351722, "pass@32": 0.8456421305459204, "pass@64": 0.8626208528041777, "pass@128": 0.8763380731746285 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9066464552238805, "pass@2": 0.9443434892466809, "pass@4": 0.9631634325560046, "pass@8": 0.9746091114318679, "pass@16": 0.981989067498895, "pass@32": 0.9872894382051791, "pass@64": 0.9904128899753277, "pass@128": 0.991044776119403 }, "2": { "pass@1": 0.9999553571428571, "pass@2": 0.9999996484814399, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9983600583090378, "pass@2": 0.9995706451872088, "pass@4": 0.9999545667214339, "pass@8": 0.9999994678161623, "pass@16": 0.9999999999690319, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9955226608187134, "pass@2": 0.996956229566699, "pass@4": 0.9970747003664014, "pass@8": 0.9970760232358671, "pass@16": 0.9970760233918129, "pass@32": 0.9970760233918129, "pass@64": 0.9970760233918129, "pass@128": 0.9970760233918129 }, "5": { "pass@1": 0.9707720588235295, "pass@2": 0.9819418712366834, "pass@4": 0.9875488412294049, "pass@8": 0.9906721170939422, "pass@16": 0.992369679677989, "pass@32": 0.9934273310473238, "pass@64": 0.9940328828017424, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.994011238170347, "pass@2": 0.9966948384212229, "pass@4": 0.9969382767414325, "pass@8": 0.9970425818238966, "pass@16": 0.9972397476340691, "pass@32": 0.9976340694006309, "pass@64": 0.998422712933754, "pass@128": 1.0 }, "7": { "pass@1": 0.9809632120253164, "pass@2": 0.9907531707864047, "pass@4": 0.9939196128964891, "pass@8": 0.9955120902657174, "pass@16": 0.9963967136338048, "pass@32": 0.996791897857649, "pass@64": 0.9968353559542914, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9713072447447447, "pass@2": 0.9862585272280152, "pass@4": 0.9918887061977614, "pass@8": 0.9948537929562266, "pass@16": 0.9965289626590625, "pass@32": 0.9975971169999002, "pass@64": 0.9984964932745174, "pass@128": 1.0 }, "9": { "pass@1": 0.9532003617363344, "pass@2": 0.9822585417879847, "pass@4": 0.9900546741343824, "pass@8": 0.9928854174919578, "pass@16": 0.9951733768710284, "pass@32": 0.9976992193537383, "pass@64": 0.9996070290208584, "pass@128": 1.0 }, "11": { "pass@1": 0.8584325396825396, "pass@2": 0.9158269278840144, "pass@4": 0.947487876217854, "pass@8": 0.9662182976284491, "pass@16": 0.976791807385288, "pass@32": 0.9819247624165014, "pass@64": 0.9848252233387427, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.7979023318042814, "pass@2": 0.8880074767030265, "pass@4": 0.934417512604502, "pass@8": 0.9589038010248868, "pass@16": 0.9715989629294857, "pass@32": 0.9774837125997929, "pass@64": 0.9811677016027062, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.5472608024691358, "pass@2": 0.6648135996889273, "pass@4": 0.7540624178343447, "pass@8": 0.817288586664587, "pass@16": 0.8629907822666152, "pass@32": 0.8987447149981824, "pass@64": 0.927510348163161, "pass@128": 0.9475308641975309 }, "14": { "pass@1": 0.4938102744807122, "pass@2": 0.6095301584149166, "pass@4": 0.7005231909067747, "pass@8": 0.7657016935167862, "pass@16": 0.8124875164470288, "pass@32": 0.8503301329249001, "pass@64": 0.8858854705277092, "pass@128": 0.9169139465875371 }, "15": { "pass@1": 0.37755594135802467, "pass@2": 0.48756584463400393, "pass@4": 0.5814024534548923, "pass@8": 0.6590459728887901, "pass@16": 0.7236160111797545, "pass@32": 0.7766907707481191, "pass@64": 0.8177650626424972, "pass@128": 0.845679012345679 }, "16": { "pass@1": 0.24944690265486727, "pass@2": 0.33750624230599463, "pass@4": 0.4218391044592877, "pass@8": 0.49854943027379045, "pass@16": 0.5657431421036233, "pass@32": 0.6217821165667055, "pass@64": 0.6690913993944967, "pass@128": 0.7109144542772862 }, "17": { "pass@1": 0.20524284638554216, "pass@2": 0.2843695154634285, "pass@4": 0.3594979064175711, "pass@8": 0.4248822789520208, "pass@16": 0.48281238948550975, "pass@32": 0.5343758170889205, "pass@64": 0.5769024260817495, "pass@128": 0.608433734939759 }, "18": { "pass@1": 0.20216733870967743, "pass@2": 0.28294585026670055, "pass@4": 0.36513060706121425, "pass@8": 0.4360889347472986, "pass@16": 0.49089282710175347, "pass@32": 0.5346986024780715, "pass@64": 0.5735686830652535, "pass@128": 0.6096774193548387 }, "19": { "pass@1": 0.16880664652567975, "pass@2": 0.23587070616361772, "pass@4": 0.30202498139394224, "pass@8": 0.3625748851412769, "pass@16": 0.416689758692117, "pass@32": 0.4624262685514275, "pass@64": 0.49878692433702804, "pass@128": 0.5287009063444109 }, "20": { "pass@1": 0.14442567567567569, "pass@2": 0.20913137546995805, "pass@4": 0.27880532810650926, "pass@8": 0.34708860136362146, "pass@16": 0.40649366622612243, "pass@32": 0.4543060270928111, "pass@64": 0.493016680586197, "pass@128": 0.5255255255255256 } } } } } ]