[ { "checkpoint": "checkpoint-21318", "id_acc": 0.9816987847222223, "ood_acc": 0.39479375, "total_acc": 0.6728013980263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.10679774305555, "ood_avg_resp_len": 319.46821171875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 546995, "answer_accuracy": 0.6755293135472653, "avg_response_len": 251.14811640452103, "resp_tokens_sum": 203361662, "pass_at_k": { "pass@1": 0.6755293135472653, "pass@2": 0.7196513817179924, "pass@4": 0.7555064448551534, "pass@8": 0.7857795238083595, "pass@16": 0.8112385584760204, "pass@32": 0.8324009119261025, "pass@64": 0.8502747082397658, "pass@128": 0.8657919696490674 }, "per_op_pass_at_k": { "10": { "pass@1": 0.921367994100295, "pass@2": 0.942556993705433, "pass@4": 0.9523452629815077, "pass@8": 0.9599810668617611, "pass@16": 0.9690468589831898, "pass@32": 0.9794474637463372, "pass@64": 0.9898632415658825, "pass@128": 1.0 }, "2": { "pass@1": 0.9999756619937694, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9992593654434251, "pass@2": 0.9998762160177227, "pass@4": 0.9999957144118452, "pass@8": 0.9999999966583266, "pass@16": 0.9999999999999998, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99736328125, "pass@2": 0.9993798443651574, "pass@4": 0.9999391749859393, "pass@8": 0.9999991739649186, "pass@16": 0.9999999999245107, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9861359126984127, "pass@2": 0.991611204849394, "pass@4": 0.9944669389540592, "pass@8": 0.996760897169578, "pass@16": 0.9986649256178634, "pass@32": 0.9996964945037693, "pass@64": 0.9999901705935292, "pass@128": 1.0 }, "6": { "pass@1": 0.9982924775449101, "pass@2": 0.9999012801169317, "pass@4": 0.9999986065551687, "pass@8": 0.9999999997361979, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9889515532544378, "pass@2": 0.9963847668079953, "pass@4": 0.9985759837527707, "pass@8": 0.9995008943340471, "pass@16": 0.9999288169588653, "pass@32": 0.9999991140832817, "pass@64": 0.9999999999952738, "pass@128": 1.0 }, "8": { "pass@1": 0.9771133814102564, "pass@2": 0.99010227387442, "pass@4": 0.994939358541721, "pass@8": 0.9976662900354473, "pass@16": 0.9992086233865933, "pass@32": 0.9998824509929162, "pass@64": 0.9999990205833484, "pass@128": 1.0 }, "9": { "pass@1": 0.9688160211267606, "pass@2": 0.9858635078185651, "pass@4": 0.9923008531503981, "pass@8": 0.9961042400644126, "pass@16": 0.9983343697290811, "pass@32": 0.99957040851631, "pass@64": 0.9999810976203355, "pass@128": 1.0 }, "11": { "pass@1": 0.9010933073654391, "pass@2": 0.9405381460373406, "pass@4": 0.9605236921517953, "pass@8": 0.9731492354166332, "pass@16": 0.9803193660921512, "pass@32": 0.9834684971128619, "pass@64": 0.985051276604143, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.795263947740113, "pass@2": 0.8569532952978335, "pass@4": 0.8883985052715867, "pass@8": 0.911165348583991, "pass@16": 0.9298656744452459, "pass@32": 0.9459628198743538, "pass@64": 0.9601943452837457, "pass@128": 0.9689265536723164 }, "13": { "pass@1": 0.6330329153605015, "pass@2": 0.7355821742403673, "pass@4": 0.8028743001952341, "pass@8": 0.8463876802059203, "pass@16": 0.8751278066483327, "pass@32": 0.8951335522881178, "pass@64": 0.9119883682469684, "pass@128": 0.9310344827586207 }, "14": { "pass@1": 0.4673627336448598, "pass@2": 0.5878985300610783, "pass@4": 0.6836119921341605, "pass@8": 0.7527518029771204, "pass@16": 0.8019372520116447, "pass@32": 0.8389366384201076, "pass@64": 0.868017856594774, "pass@128": 0.8940809968847352 }, "15": { "pass@1": 0.34298229768786126, "pass@2": 0.44826347799371885, "pass@4": 0.5397064255154521, "pass@8": 0.6178997236796543, "pass@16": 0.6839055796605068, "pass@32": 0.7382863549854671, "pass@64": 0.7840059247267029, "pass@128": 0.8208092485549133 }, "16": { "pass@1": 0.22472278225806452, "pass@2": 0.3049534067818133, "pass@4": 0.3885627892279594, "pass@8": 0.4694046326091154, "pass@16": 0.5407811841689135, "pass@32": 0.5993510131658059, "pass@64": 0.6487200656912525, "pass@128": 0.7 }, "17": { "pass@1": 0.192038475433526, "pass@2": 0.26094375824951066, "pass@4": 0.328613439577278, "pass@8": 0.3906448676049513, "pass@16": 0.44859100495476445, "pass@32": 0.5035163751382083, "pass@64": 0.5543401123759589, "pass@128": 0.5982658959537572 }, "18": { "pass@1": 0.16830357142857144, "pass@2": 0.23052692632170968, "pass@4": 0.29466466093524013, "pass@8": 0.358136426787759, "pass@16": 0.41768060583849326, "pass@32": 0.47060100460200294, "pass@64": 0.5174976759132182, "pass@128": 0.56 }, "19": { "pass@1": 0.15233201951951952, "pass@2": 0.21569785829136226, "pass@4": 0.2814930098827737, "pass@8": 0.34293583552945484, "pass@16": 0.397087254979992, "pass@32": 0.4430136923284928, "pass@64": 0.47970105764191046, "pass@128": 0.5075075075075075 }, "20": { "pass@1": 0.15329391891891891, "pass@2": 0.21639651462486104, "pass@4": 0.28279032660331865, "pass@8": 0.3453021600837771, "pass@16": 0.39926121854111285, "pass@32": 0.44277060325279904, "pass@64": 0.477653961999068, "pass@128": 0.5045045045045045 } } }, "teachers_in_school": { "count": 821120, "correct": 553236, "answer_accuracy": 0.6737577942322681, "avg_response_len": 260.4063754384256, "resp_tokens_sum": 213824883, "pass_at_k": { "pass@1": 0.6737577942322681, "pass@2": 0.716840988455939, "pass@4": 0.7525902811008663, "pass@8": 0.7832099213498322, "pass@16": 0.8095657439825331, "pass@32": 0.8322222789827285, "pass@64": 0.8521636648447887, "pass@128": 0.8695245518316446 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9128163343558282, "pass@2": 0.9338867989469108, "pass@4": 0.9469359922801062, "pass@8": 0.9577439859621151, "pass@16": 0.9663107063932108, "pass@32": 0.9724515356619124, "pass@64": 0.9772733818568755, "pass@128": 0.9815950920245399 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9995738636363637, "pass@2": 0.9999757665235026, "pass@4": 0.9999999048414402, "pass@8": 0.9999999999996503, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9996070636094675, "pass@2": 0.9999890800447281, "pass@4": 0.999999995840017, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9939990942028986, "pass@2": 0.9963015662444369, "pass@4": 0.9970881311031774, "pass@8": 0.997280686402298, "pass@16": 0.9974637676538072, "pass@32": 0.9978260869565218, "pass@64": 0.9985507246376811, "pass@128": 1.0 }, "6": { "pass@1": 0.9941350286532952, "pass@2": 0.9956054136678474, "pass@4": 0.9964049499185096, "pass@8": 0.9970906621993734, "pass@16": 0.9974787490346466, "pass@32": 0.9978509774049679, "pass@64": 0.9985673352435527, "pass@128": 1.0 }, "7": { "pass@1": 0.9909230491329479, "pass@2": 0.9951096758226753, "pass@4": 0.9966716914179079, "pass@8": 0.9977106915233841, "pass@16": 0.9985226880456228, "pass@32": 0.9993322522213881, "pass@64": 0.9999167083883299, "pass@128": 1.0 }, "8": { "pass@1": 0.976012323943662, "pass@2": 0.9867372047244095, "pass@4": 0.9918025977386628, "pass@8": 0.9950307197147326, "pass@16": 0.9967035542036129, "pass@32": 0.9971586333157431, "pass@64": 0.9971830895979062, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.9633233532934131, "pass@2": 0.9805389221556892, "pass@4": 0.9861284051694138, "pass@8": 0.9888776547152858, "pass@16": 0.9908840151926985, "pass@32": 0.9931395416818996, "pass@64": 0.9957196473070987, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.8984375, "pass@2": 0.9381933491841383, "pass@4": 0.9563708170703052, "pass@8": 0.9651384651070724, "pass@16": 0.970847848961363, "pass@32": 0.9752184800758407, "pass@64": 0.9785014696339162, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8103938087774295, "pass@2": 0.8784228982055147, "pass@4": 0.9111158149706199, "pass@8": 0.9281237080976364, "pass@16": 0.9385156188047306, "pass@32": 0.945844647900169, "pass@64": 0.9537586608978759, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.5918461134453782, "pass@2": 0.6963641263592046, "pass@4": 0.7645203430453539, "pass@8": 0.8103114208848939, "pass@16": 0.8444682381702995, "pass@32": 0.8729812384432029, "pass@64": 0.8990546791683675, "pass@128": 0.9243697478991597 }, "14": { "pass@1": 0.4608004385964912, "pass@2": 0.5698905235529772, "pass@4": 0.6570574678713406, "pass@8": 0.7255123283563643, "pass@16": 0.7781650680366183, "pass@32": 0.8175260759375188, "pass@64": 0.8479233692091781, "pass@128": 0.8742690058479532 }, "15": { "pass@1": 0.3157670454545455, "pass@2": 0.42129339656406567, "pass@4": 0.5173501852041221, "pass@8": 0.5983073489401209, "pass@16": 0.662404884476687, "pass@32": 0.7136393554090066, "pass@64": 0.7549481851820344, "pass@128": 0.7848484848484848 }, "16": { "pass@1": 0.22930021367521367, "pass@2": 0.30401994021580625, "pass@4": 0.37868730591368366, "pass@8": 0.44972633534408485, "pass@16": 0.514676593961253, "pass@32": 0.5717405020184494, "pass@64": 0.6253495271451086, "pass@128": 0.6752136752136753 }, "17": { "pass@1": 0.2045322204968944, "pass@2": 0.28461082310363345, "pass@4": 0.366213108163498, "pass@8": 0.44316985996026803, "pass@16": 0.5150883177107987, "pass@32": 0.5817667643375678, "pass@64": 0.6391944649500597, "pass@128": 0.6801242236024845 }, "18": { "pass@1": 0.18235294117647058, "pass@2": 0.24705086266790183, "pass@4": 0.3139364523368402, "pass@8": 0.38127949190055477, "pass@16": 0.44793120123381897, "pass@32": 0.5112143577609161, "pass@64": 0.5714818783691119, "pass@128": 0.6294117647058823 }, "19": { "pass@1": 0.13630022321428573, "pass@2": 0.19533190675384327, "pass@4": 0.26009498226784167, "pass@8": 0.32214647965183274, "pass@16": 0.3777820314177309, "pass@32": 0.42567070543150887, "pass@64": 0.4646344346644764, "pass@128": 0.49107142857142855 }, "20": { "pass@1": 0.12829809131736528, "pass@2": 0.18442089737847145, "pass@4": 0.24735149161744008, "pass@8": 0.31292081333429284, "pass@16": 0.37478297318558906, "pass@32": 0.4300108038401019, "pass@64": 0.4807222226393277, "pass@128": 0.5299401197604791 } } }, "movie_festival_awards": { "count": 801152, "correct": 536022, "answer_accuracy": 0.6690640477712094, "avg_response_len": 277.42026107405337, "resp_tokens_sum": 222255797, "pass_at_k": { "pass@1": 0.6690640477712094, "pass@2": 0.712880715549134, "pass@4": 0.7494017679258717, "pass@8": 0.7802768116083039, "pass@16": 0.8061971639659127, "pass@32": 0.8280859994231852, "pass@64": 0.8469885933279211, "pass@128": 0.8629173989455184 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9251166044776119, "pass@2": 0.9515916970266775, "pass@4": 0.9665068801101353, "pass@8": 0.976736864518274, "pass@16": 0.9843464610123595, "pass@32": 0.9894092982952967, "pass@64": 0.9927038912238968, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9994305758017493, "pass@2": 0.9999723806845571, "pass@4": 0.9999999316776363, "pass@8": 0.9999999999999082, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9998400950292398, "pass@2": 0.9999946038817517, "pass@4": 0.9999999958886717, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9790211397058823, "pass@2": 0.9855901169522926, "pass@4": 0.9908074207635807, "pass@8": 0.9946330614330179, "pass@16": 0.9973377627249943, "pass@32": 0.9990025449700659, "pass@64": 0.9998246344139574, "pass@128": 1.0 }, "6": { "pass@1": 0.9963278785488959, "pass@2": 0.9980629331329641, "pass@4": 0.9988404745936725, "pass@8": 0.9995894054915941, "pass@16": 0.9999545159662533, "pass@32": 0.9999996947209199, "pass@64": 0.9999999999997397, "pass@128": 1.0 }, "7": { "pass@1": 0.981309335443038, "pass@2": 0.9896961122047246, "pass@4": 0.9931058592952778, "pass@8": 0.9950956310634967, "pass@16": 0.9966169936024034, "pass@32": 0.9975642800251153, "pass@64": 0.9984175251201582, "pass@128": 1.0 }, "8": { "pass@1": 0.9857591966966966, "pass@2": 0.9946191270010165, "pass@4": 0.9966438167638952, "pass@8": 0.9982029777127238, "pass@16": 0.9995019194164857, "pass@32": 0.9999710139814316, "pass@64": 0.9999999830905767, "pass@128": 1.0 }, "9": { "pass@1": 0.9708098874598071, "pass@2": 0.9887697166873437, "pass@4": 0.9942800896873418, "pass@8": 0.9963206131507204, "pass@16": 0.9971160831245206, "pass@32": 0.9975877585822082, "pass@64": 0.9983922829562545, "pass@128": 1.0 }, "11": { "pass@1": 0.8975446428571429, "pass@2": 0.940096550431196, "pass@4": 0.9623223132822685, "pass@8": 0.9750701141750376, "pass@16": 0.9819630715622439, "pass@32": 0.9869295105755491, "pass@64": 0.9911323522261105, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.7937213302752294, "pass@2": 0.8705671037106599, "pass@4": 0.9124972727032965, "pass@8": 0.9371350468935171, "pass@16": 0.9540943098751107, "pass@32": 0.96709032747095, "pass@64": 0.9767926069560982, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.5636574074074074, "pass@2": 0.6774110375959945, "pass@4": 0.7594694933966581, "pass@8": 0.8178451621208527, "pass@16": 0.8581321518540509, "pass@32": 0.8855340870030073, "pass@64": 0.906238411770351, "pass@128": 0.9228395061728395 }, "14": { "pass@1": 0.42050723293768544, "pass@2": 0.5323967119558866, "pass@4": 0.6295082539756716, "pass@8": 0.7075853620554116, "pass@16": 0.7639367836921418, "pass@32": 0.8011987751329052, "pass@64": 0.8282495474340722, "pass@128": 0.8516320474777448 }, "15": { "pass@1": 0.2934992283950617, "pass@2": 0.3915162188927772, "pass@4": 0.4861379998217816, "pass@8": 0.5699345482333399, "pass@16": 0.6400750718881144, "pass@32": 0.6978942493178334, "pass@64": 0.7444583522493792, "pass@128": 0.7839506172839507 }, "16": { "pass@1": 0.2207549778761062, "pass@2": 0.29584066441362966, "pass@4": 0.3715742169396969, "pass@8": 0.4423795603006429, "pass@16": 0.506508515814046, "pass@32": 0.5677344699516077, "pass@64": 0.6278069498553722, "pass@128": 0.6843657817109144 }, "17": { "pass@1": 0.18175828313253012, "pass@2": 0.2504491390759891, "pass@4": 0.31680082845442487, "pass@8": 0.3772025869780121, "pass@16": 0.43577824767928647, "pass@32": 0.49463490155797246, "pass@64": 0.5526981943701811, "pass@128": 0.5993975903614458 }, "18": { "pass@1": 0.17719254032258064, "pass@2": 0.24210018732537472, "pass@4": 0.30995016389080393, "pass@8": 0.374431450995483, "pass@16": 0.4320218199065214, "pass@32": 0.48272784789235007, "pass@64": 0.5276831506385287, "pass@128": 0.567741935483871 }, "19": { "pass@1": 0.150467333836858, "pass@2": 0.20744336834217472, "pass@4": 0.2637100587124496, "pass@8": 0.3160565580601192, "pass@16": 0.36417387095338233, "pass@32": 0.4085595977535364, "pass@64": 0.4476669031184992, "pass@128": 0.48036253776435045 }, "20": { "pass@1": 0.1573995870870871, "pass@2": 0.21469217741363408, "pass@4": 0.27489675811919906, "pass@8": 0.33699570666238976, "pass@16": 0.39716885563008186, "pass@32": 0.44952312446152337, "pass@64": 0.4929726530229929, "pass@128": 0.5285285285285285 } } } } } ]