| [ |
| { |
| "checkpoint": "checkpoint-21318", |
| "id_acc": 0.9816987847222223, |
| "ood_acc": 0.39479375, |
| "total_acc": 0.6728013980263158, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.10679774305555, |
| "ood_avg_resp_len": 319.46821171875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 546995, |
| "answer_accuracy": 0.6755293135472653, |
| "avg_response_len": 251.14811640452103, |
| "resp_tokens_sum": 203361662, |
| "pass_at_k": { |
| "pass@1": 0.6755293135472653, |
| "pass@2": 0.7196513817179924, |
| "pass@4": 0.7555064448551534, |
| "pass@8": 0.7857795238083595, |
| "pass@16": 0.8112385584760204, |
| "pass@32": 0.8324009119261025, |
| "pass@64": 0.8502747082397658, |
| "pass@128": 0.8657919696490674 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.921367994100295, |
| "pass@2": 0.942556993705433, |
| "pass@4": 0.9523452629815077, |
| "pass@8": 0.9599810668617611, |
| "pass@16": 0.9690468589831898, |
| "pass@32": 0.9794474637463372, |
| "pass@64": 0.9898632415658825, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 0.9999756619937694, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9992593654434251, |
| "pass@2": 0.9998762160177227, |
| "pass@4": 0.9999957144118452, |
| "pass@8": 0.9999999966583266, |
| "pass@16": 0.9999999999999998, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99736328125, |
| "pass@2": 0.9993798443651574, |
| "pass@4": 0.9999391749859393, |
| "pass@8": 0.9999991739649186, |
| "pass@16": 0.9999999999245107, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9861359126984127, |
| "pass@2": 0.991611204849394, |
| "pass@4": 0.9944669389540592, |
| "pass@8": 0.996760897169578, |
| "pass@16": 0.9986649256178634, |
| "pass@32": 0.9996964945037693, |
| "pass@64": 0.9999901705935292, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9982924775449101, |
| "pass@2": 0.9999012801169317, |
| "pass@4": 0.9999986065551687, |
| "pass@8": 0.9999999997361979, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9889515532544378, |
| "pass@2": 0.9963847668079953, |
| "pass@4": 0.9985759837527707, |
| "pass@8": 0.9995008943340471, |
| "pass@16": 0.9999288169588653, |
| "pass@32": 0.9999991140832817, |
| "pass@64": 0.9999999999952738, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9771133814102564, |
| "pass@2": 0.99010227387442, |
| "pass@4": 0.994939358541721, |
| "pass@8": 0.9976662900354473, |
| "pass@16": 0.9992086233865933, |
| "pass@32": 0.9998824509929162, |
| "pass@64": 0.9999990205833484, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9688160211267606, |
| "pass@2": 0.9858635078185651, |
| "pass@4": 0.9923008531503981, |
| "pass@8": 0.9961042400644126, |
| "pass@16": 0.9983343697290811, |
| "pass@32": 0.99957040851631, |
| "pass@64": 0.9999810976203355, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9010933073654391, |
| "pass@2": 0.9405381460373406, |
| "pass@4": 0.9605236921517953, |
| "pass@8": 0.9731492354166332, |
| "pass@16": 0.9803193660921512, |
| "pass@32": 0.9834684971128619, |
| "pass@64": 0.985051276604143, |
| "pass@128": 0.9858356940509915 |
| }, |
| "12": { |
| "pass@1": 0.795263947740113, |
| "pass@2": 0.8569532952978335, |
| "pass@4": 0.8883985052715867, |
| "pass@8": 0.911165348583991, |
| "pass@16": 0.9298656744452459, |
| "pass@32": 0.9459628198743538, |
| "pass@64": 0.9601943452837457, |
| "pass@128": 0.9689265536723164 |
| }, |
| "13": { |
| "pass@1": 0.6330329153605015, |
| "pass@2": 0.7355821742403673, |
| "pass@4": 0.8028743001952341, |
| "pass@8": 0.8463876802059203, |
| "pass@16": 0.8751278066483327, |
| "pass@32": 0.8951335522881178, |
| "pass@64": 0.9119883682469684, |
| "pass@128": 0.9310344827586207 |
| }, |
| "14": { |
| "pass@1": 0.4673627336448598, |
| "pass@2": 0.5878985300610783, |
| "pass@4": 0.6836119921341605, |
| "pass@8": 0.7527518029771204, |
| "pass@16": 0.8019372520116447, |
| "pass@32": 0.8389366384201076, |
| "pass@64": 0.868017856594774, |
| "pass@128": 0.8940809968847352 |
| }, |
| "15": { |
| "pass@1": 0.34298229768786126, |
| "pass@2": 0.44826347799371885, |
| "pass@4": 0.5397064255154521, |
| "pass@8": 0.6178997236796543, |
| "pass@16": 0.6839055796605068, |
| "pass@32": 0.7382863549854671, |
| "pass@64": 0.7840059247267029, |
| "pass@128": 0.8208092485549133 |
| }, |
| "16": { |
| "pass@1": 0.22472278225806452, |
| "pass@2": 0.3049534067818133, |
| "pass@4": 0.3885627892279594, |
| "pass@8": 0.4694046326091154, |
| "pass@16": 0.5407811841689135, |
| "pass@32": 0.5993510131658059, |
| "pass@64": 0.6487200656912525, |
| "pass@128": 0.7 |
| }, |
| "17": { |
| "pass@1": 0.192038475433526, |
| "pass@2": 0.26094375824951066, |
| "pass@4": 0.328613439577278, |
| "pass@8": 0.3906448676049513, |
| "pass@16": 0.44859100495476445, |
| "pass@32": 0.5035163751382083, |
| "pass@64": 0.5543401123759589, |
| "pass@128": 0.5982658959537572 |
| }, |
| "18": { |
| "pass@1": 0.16830357142857144, |
| "pass@2": 0.23052692632170968, |
| "pass@4": 0.29466466093524013, |
| "pass@8": 0.358136426787759, |
| "pass@16": 0.41768060583849326, |
| "pass@32": 0.47060100460200294, |
| "pass@64": 0.5174976759132182, |
| "pass@128": 0.56 |
| }, |
| "19": { |
| "pass@1": 0.15233201951951952, |
| "pass@2": 0.21569785829136226, |
| "pass@4": 0.2814930098827737, |
| "pass@8": 0.34293583552945484, |
| "pass@16": 0.397087254979992, |
| "pass@32": 0.4430136923284928, |
| "pass@64": 0.47970105764191046, |
| "pass@128": 0.5075075075075075 |
| }, |
| "20": { |
| "pass@1": 0.15329391891891891, |
| "pass@2": 0.21639651462486104, |
| "pass@4": 0.28279032660331865, |
| "pass@8": 0.3453021600837771, |
| "pass@16": 0.39926121854111285, |
| "pass@32": 0.44277060325279904, |
| "pass@64": 0.477653961999068, |
| "pass@128": 0.5045045045045045 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 553236, |
| "answer_accuracy": 0.6737577942322681, |
| "avg_response_len": 260.4063754384256, |
| "resp_tokens_sum": 213824883, |
| "pass_at_k": { |
| "pass@1": 0.6737577942322681, |
| "pass@2": 0.716840988455939, |
| "pass@4": 0.7525902811008663, |
| "pass@8": 0.7832099213498322, |
| "pass@16": 0.8095657439825331, |
| "pass@32": 0.8322222789827285, |
| "pass@64": 0.8521636648447887, |
| "pass@128": 0.8695245518316446 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9128163343558282, |
| "pass@2": 0.9338867989469108, |
| "pass@4": 0.9469359922801062, |
| "pass@8": 0.9577439859621151, |
| "pass@16": 0.9663107063932108, |
| "pass@32": 0.9724515356619124, |
| "pass@64": 0.9772733818568755, |
| "pass@128": 0.9815950920245399 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995738636363637, |
| "pass@2": 0.9999757665235026, |
| "pass@4": 0.9999999048414402, |
| "pass@8": 0.9999999999996503, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9996070636094675, |
| "pass@2": 0.9999890800447281, |
| "pass@4": 0.999999995840017, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9939990942028986, |
| "pass@2": 0.9963015662444369, |
| "pass@4": 0.9970881311031774, |
| "pass@8": 0.997280686402298, |
| "pass@16": 0.9974637676538072, |
| "pass@32": 0.9978260869565218, |
| "pass@64": 0.9985507246376811, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9941350286532952, |
| "pass@2": 0.9956054136678474, |
| "pass@4": 0.9964049499185096, |
| "pass@8": 0.9970906621993734, |
| "pass@16": 0.9974787490346466, |
| "pass@32": 0.9978509774049679, |
| "pass@64": 0.9985673352435527, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9909230491329479, |
| "pass@2": 0.9951096758226753, |
| "pass@4": 0.9966716914179079, |
| "pass@8": 0.9977106915233841, |
| "pass@16": 0.9985226880456228, |
| "pass@32": 0.9993322522213881, |
| "pass@64": 0.9999167083883299, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.976012323943662, |
| "pass@2": 0.9867372047244095, |
| "pass@4": 0.9918025977386628, |
| "pass@8": 0.9950307197147326, |
| "pass@16": 0.9967035542036129, |
| "pass@32": 0.9971586333157431, |
| "pass@64": 0.9971830895979062, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.9633233532934131, |
| "pass@2": 0.9805389221556892, |
| "pass@4": 0.9861284051694138, |
| "pass@8": 0.9888776547152858, |
| "pass@16": 0.9908840151926985, |
| "pass@32": 0.9931395416818996, |
| "pass@64": 0.9957196473070987, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.8984375, |
| "pass@2": 0.9381933491841383, |
| "pass@4": 0.9563708170703052, |
| "pass@8": 0.9651384651070724, |
| "pass@16": 0.970847848961363, |
| "pass@32": 0.9752184800758407, |
| "pass@64": 0.9785014696339162, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8103938087774295, |
| "pass@2": 0.8784228982055147, |
| "pass@4": 0.9111158149706199, |
| "pass@8": 0.9281237080976364, |
| "pass@16": 0.9385156188047306, |
| "pass@32": 0.945844647900169, |
| "pass@64": 0.9537586608978759, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.5918461134453782, |
| "pass@2": 0.6963641263592046, |
| "pass@4": 0.7645203430453539, |
| "pass@8": 0.8103114208848939, |
| "pass@16": 0.8444682381702995, |
| "pass@32": 0.8729812384432029, |
| "pass@64": 0.8990546791683675, |
| "pass@128": 0.9243697478991597 |
| }, |
| "14": { |
| "pass@1": 0.4608004385964912, |
| "pass@2": 0.5698905235529772, |
| "pass@4": 0.6570574678713406, |
| "pass@8": 0.7255123283563643, |
| "pass@16": 0.7781650680366183, |
| "pass@32": 0.8175260759375188, |
| "pass@64": 0.8479233692091781, |
| "pass@128": 0.8742690058479532 |
| }, |
| "15": { |
| "pass@1": 0.3157670454545455, |
| "pass@2": 0.42129339656406567, |
| "pass@4": 0.5173501852041221, |
| "pass@8": 0.5983073489401209, |
| "pass@16": 0.662404884476687, |
| "pass@32": 0.7136393554090066, |
| "pass@64": 0.7549481851820344, |
| "pass@128": 0.7848484848484848 |
| }, |
| "16": { |
| "pass@1": 0.22930021367521367, |
| "pass@2": 0.30401994021580625, |
| "pass@4": 0.37868730591368366, |
| "pass@8": 0.44972633534408485, |
| "pass@16": 0.514676593961253, |
| "pass@32": 0.5717405020184494, |
| "pass@64": 0.6253495271451086, |
| "pass@128": 0.6752136752136753 |
| }, |
| "17": { |
| "pass@1": 0.2045322204968944, |
| "pass@2": 0.28461082310363345, |
| "pass@4": 0.366213108163498, |
| "pass@8": 0.44316985996026803, |
| "pass@16": 0.5150883177107987, |
| "pass@32": 0.5817667643375678, |
| "pass@64": 0.6391944649500597, |
| "pass@128": 0.6801242236024845 |
| }, |
| "18": { |
| "pass@1": 0.18235294117647058, |
| "pass@2": 0.24705086266790183, |
| "pass@4": 0.3139364523368402, |
| "pass@8": 0.38127949190055477, |
| "pass@16": 0.44793120123381897, |
| "pass@32": 0.5112143577609161, |
| "pass@64": 0.5714818783691119, |
| "pass@128": 0.6294117647058823 |
| }, |
| "19": { |
| "pass@1": 0.13630022321428573, |
| "pass@2": 0.19533190675384327, |
| "pass@4": 0.26009498226784167, |
| "pass@8": 0.32214647965183274, |
| "pass@16": 0.3777820314177309, |
| "pass@32": 0.42567070543150887, |
| "pass@64": 0.4646344346644764, |
| "pass@128": 0.49107142857142855 |
| }, |
| "20": { |
| "pass@1": 0.12829809131736528, |
| "pass@2": 0.18442089737847145, |
| "pass@4": 0.24735149161744008, |
| "pass@8": 0.31292081333429284, |
| "pass@16": 0.37478297318558906, |
| "pass@32": 0.4300108038401019, |
| "pass@64": 0.4807222226393277, |
| "pass@128": 0.5299401197604791 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 536022, |
| "answer_accuracy": 0.6690640477712094, |
| "avg_response_len": 277.42026107405337, |
| "resp_tokens_sum": 222255797, |
| "pass_at_k": { |
| "pass@1": 0.6690640477712094, |
| "pass@2": 0.712880715549134, |
| "pass@4": 0.7494017679258717, |
| "pass@8": 0.7802768116083039, |
| "pass@16": 0.8061971639659127, |
| "pass@32": 0.8280859994231852, |
| "pass@64": 0.8469885933279211, |
| "pass@128": 0.8629173989455184 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9251166044776119, |
| "pass@2": 0.9515916970266775, |
| "pass@4": 0.9665068801101353, |
| "pass@8": 0.976736864518274, |
| "pass@16": 0.9843464610123595, |
| "pass@32": 0.9894092982952967, |
| "pass@64": 0.9927038912238968, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9994305758017493, |
| "pass@2": 0.9999723806845571, |
| "pass@4": 0.9999999316776363, |
| "pass@8": 0.9999999999999082, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9998400950292398, |
| "pass@2": 0.9999946038817517, |
| "pass@4": 0.9999999958886717, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9790211397058823, |
| "pass@2": 0.9855901169522926, |
| "pass@4": 0.9908074207635807, |
| "pass@8": 0.9946330614330179, |
| "pass@16": 0.9973377627249943, |
| "pass@32": 0.9990025449700659, |
| "pass@64": 0.9998246344139574, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9963278785488959, |
| "pass@2": 0.9980629331329641, |
| "pass@4": 0.9988404745936725, |
| "pass@8": 0.9995894054915941, |
| "pass@16": 0.9999545159662533, |
| "pass@32": 0.9999996947209199, |
| "pass@64": 0.9999999999997397, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.981309335443038, |
| "pass@2": 0.9896961122047246, |
| "pass@4": 0.9931058592952778, |
| "pass@8": 0.9950956310634967, |
| "pass@16": 0.9966169936024034, |
| "pass@32": 0.9975642800251153, |
| "pass@64": 0.9984175251201582, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9857591966966966, |
| "pass@2": 0.9946191270010165, |
| "pass@4": 0.9966438167638952, |
| "pass@8": 0.9982029777127238, |
| "pass@16": 0.9995019194164857, |
| "pass@32": 0.9999710139814316, |
| "pass@64": 0.9999999830905767, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9708098874598071, |
| "pass@2": 0.9887697166873437, |
| "pass@4": 0.9942800896873418, |
| "pass@8": 0.9963206131507204, |
| "pass@16": 0.9971160831245206, |
| "pass@32": 0.9975877585822082, |
| "pass@64": 0.9983922829562545, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8975446428571429, |
| "pass@2": 0.940096550431196, |
| "pass@4": 0.9623223132822685, |
| "pass@8": 0.9750701141750376, |
| "pass@16": 0.9819630715622439, |
| "pass@32": 0.9869295105755491, |
| "pass@64": 0.9911323522261105, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.7937213302752294, |
| "pass@2": 0.8705671037106599, |
| "pass@4": 0.9124972727032965, |
| "pass@8": 0.9371350468935171, |
| "pass@16": 0.9540943098751107, |
| "pass@32": 0.96709032747095, |
| "pass@64": 0.9767926069560982, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.5636574074074074, |
| "pass@2": 0.6774110375959945, |
| "pass@4": 0.7594694933966581, |
| "pass@8": 0.8178451621208527, |
| "pass@16": 0.8581321518540509, |
| "pass@32": 0.8855340870030073, |
| "pass@64": 0.906238411770351, |
| "pass@128": 0.9228395061728395 |
| }, |
| "14": { |
| "pass@1": 0.42050723293768544, |
| "pass@2": 0.5323967119558866, |
| "pass@4": 0.6295082539756716, |
| "pass@8": 0.7075853620554116, |
| "pass@16": 0.7639367836921418, |
| "pass@32": 0.8011987751329052, |
| "pass@64": 0.8282495474340722, |
| "pass@128": 0.8516320474777448 |
| }, |
| "15": { |
| "pass@1": 0.2934992283950617, |
| "pass@2": 0.3915162188927772, |
| "pass@4": 0.4861379998217816, |
| "pass@8": 0.5699345482333399, |
| "pass@16": 0.6400750718881144, |
| "pass@32": 0.6978942493178334, |
| "pass@64": 0.7444583522493792, |
| "pass@128": 0.7839506172839507 |
| }, |
| "16": { |
| "pass@1": 0.2207549778761062, |
| "pass@2": 0.29584066441362966, |
| "pass@4": 0.3715742169396969, |
| "pass@8": 0.4423795603006429, |
| "pass@16": 0.506508515814046, |
| "pass@32": 0.5677344699516077, |
| "pass@64": 0.6278069498553722, |
| "pass@128": 0.6843657817109144 |
| }, |
| "17": { |
| "pass@1": 0.18175828313253012, |
| "pass@2": 0.2504491390759891, |
| "pass@4": 0.31680082845442487, |
| "pass@8": 0.3772025869780121, |
| "pass@16": 0.43577824767928647, |
| "pass@32": 0.49463490155797246, |
| "pass@64": 0.5526981943701811, |
| "pass@128": 0.5993975903614458 |
| }, |
| "18": { |
| "pass@1": 0.17719254032258064, |
| "pass@2": 0.24210018732537472, |
| "pass@4": 0.30995016389080393, |
| "pass@8": 0.374431450995483, |
| "pass@16": 0.4320218199065214, |
| "pass@32": 0.48272784789235007, |
| "pass@64": 0.5276831506385287, |
| "pass@128": 0.567741935483871 |
| }, |
| "19": { |
| "pass@1": 0.150467333836858, |
| "pass@2": 0.20744336834217472, |
| "pass@4": 0.2637100587124496, |
| "pass@8": 0.3160565580601192, |
| "pass@16": 0.36417387095338233, |
| "pass@32": 0.4085595977535364, |
| "pass@64": 0.4476669031184992, |
| "pass@128": 0.48036253776435045 |
| }, |
| "20": { |
| "pass@1": 0.1573995870870871, |
| "pass@2": 0.21469217741363408, |
| "pass@4": 0.27489675811919906, |
| "pass@8": 0.33699570666238976, |
| "pass@16": 0.39716885563008186, |
| "pass@32": 0.44952312446152337, |
| "pass@64": 0.4929726530229929, |
| "pass@128": 0.5285285285285285 |
| } |
| } |
| } |
| } |
| } |
| ] |
