| [ |
| { |
| "checkpoint": "checkpoint-86183", |
| "id_acc": 0.9919913194444444, |
| "ood_acc": 0.4688859375, |
| "total_acc": 0.7166726973684211, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.9115217013889, |
| "ood_avg_resp_len": 336.13270703125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 580465, |
| "answer_accuracy": 0.7168641815523238, |
| "avg_response_len": 259.1584087496048, |
| "resp_tokens_sum": 209847820, |
| "pass_at_k": { |
| "pass@1": 0.7168641815523238, |
| "pass@2": 0.7563181321430583, |
| "pass@4": 0.7891167671974877, |
| "pass@8": 0.8160179728357079, |
| "pass@16": 0.8380543061281882, |
| "pass@32": 0.8560953306144278, |
| "pass@64": 0.8713585957411207, |
| "pass@128": 0.8858678469807145 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9424317846607669, |
| "pass@2": 0.9554277576475513, |
| "pass@4": 0.9645232338214357, |
| "pass@8": 0.9705729284703094, |
| "pass@16": 0.9750282528248507, |
| "pass@32": 0.979012371776588, |
| "pass@64": 0.9825793273267351, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9998511904761904, |
| "pass@2": 0.9999992188476441, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9982456961077845, |
| "pass@2": 0.9989778071809138, |
| "pass@4": 0.9996588796535164, |
| "pass@8": 0.9999646694808768, |
| "pass@16": 0.9999997257540574, |
| "pass@32": 0.9999999999968382, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9997457470414202, |
| "pass@2": 0.9999949040208731, |
| "pass@4": 0.9999999986133391, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9944411057692307, |
| "pass@2": 0.9981253470119119, |
| "pass@4": 0.9996946421360792, |
| "pass@8": 0.9999886513123039, |
| "pass@16": 0.9999999830961206, |
| "pass@32": 0.999999999999996, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9926496478873239, |
| "pass@2": 0.996029721636908, |
| "pass@4": 0.9975660128751512, |
| "pass@8": 0.9983336498027551, |
| "pass@16": 0.999063138456991, |
| "pass@32": 0.9997387400876202, |
| "pass@64": 0.9999912792555583, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9241988314447592, |
| "pass@2": 0.9451147085721935, |
| "pass@4": 0.9583057509100311, |
| "pass@8": 0.9683471315113082, |
| "pass@16": 0.9758373276009441, |
| "pass@32": 0.9813627292403387, |
| "pass@64": 0.9854111072441946, |
| "pass@128": 0.9886685552407932 |
| }, |
| "12": { |
| "pass@1": 0.832229872881356, |
| "pass@2": 0.8820969015525602, |
| "pass@4": 0.9064526634912154, |
| "pass@8": 0.9210831491573624, |
| "pass@16": 0.9322645619669031, |
| "pass@32": 0.9417992939933498, |
| "pass@64": 0.9503938639071714, |
| "pass@128": 0.9576271186440678 |
| }, |
| "13": { |
| "pass@1": 0.6866918103448276, |
| "pass@2": 0.7621581344259865, |
| "pass@4": 0.8154033129283599, |
| "pass@8": 0.8542870664981383, |
| "pass@16": 0.8825616429284143, |
| "pass@32": 0.9031559569013347, |
| "pass@64": 0.9156100331712211, |
| "pass@128": 0.9184952978056427 |
| }, |
| "14": { |
| "pass@1": 0.6419635903426791, |
| "pass@2": 0.724484570853877, |
| "pass@4": 0.77979786434406, |
| "pass@8": 0.8168612902209831, |
| "pass@16": 0.8439180312348161, |
| "pass@32": 0.8659116440785112, |
| "pass@64": 0.8836119795991179, |
| "pass@128": 0.897196261682243 |
| }, |
| "15": { |
| "pass@1": 0.5370077673410405, |
| "pass@2": 0.6450683571526107, |
| "pass@4": 0.7338411369099088, |
| "pass@8": 0.7986025998546395, |
| "pass@16": 0.8436119684412422, |
| "pass@32": 0.8759949765481339, |
| "pass@64": 0.8985630011957301, |
| "pass@128": 0.9161849710982659 |
| }, |
| "16": { |
| "pass@1": 0.2983618951612903, |
| "pass@2": 0.3907174720599443, |
| "pass@4": 0.4704105074567288, |
| "pass@8": 0.535630716107639, |
| "pass@16": 0.5907458646622782, |
| "pass@32": 0.6372065651541644, |
| "pass@64": 0.6795116518435073, |
| "pass@128": 0.7225806451612903 |
| }, |
| "17": { |
| "pass@1": 0.2517837789017341, |
| "pass@2": 0.3336255035046197, |
| "pass@4": 0.41323942626752563, |
| "pass@8": 0.48810200863925596, |
| "pass@16": 0.5547621412716494, |
| "pass@32": 0.6106711542212494, |
| "pass@64": 0.6590666679799665, |
| "pass@128": 0.708092485549133 |
| }, |
| "18": { |
| "pass@1": 0.19428571428571428, |
| "pass@2": 0.2676690804274464, |
| "pass@4": 0.34212716508650715, |
| "pass@8": 0.4139243368536438, |
| "pass@16": 0.4805783850514051, |
| "pass@32": 0.5388344572164389, |
| "pass@64": 0.5889168734491658, |
| "pass@128": 0.6371428571428571 |
| }, |
| "19": { |
| "pass@1": 0.18327702702702703, |
| "pass@2": 0.2533107664751364, |
| "pass@4": 0.3211475995117731, |
| "pass@8": 0.3820281677250221, |
| "pass@16": 0.4348139659769961, |
| "pass@32": 0.4792898587653093, |
| "pass@64": 0.5180948362733667, |
| "pass@128": 0.5585585585585585 |
| }, |
| "20": { |
| "pass@1": 0.17429147897897898, |
| "pass@2": 0.2449047226360218, |
| "pass@4": 0.3154769668431085, |
| "pass@8": 0.3776520541403074, |
| "pass@16": 0.4277429922970302, |
| "pass@32": 0.46807247378301114, |
| "pass@64": 0.5071460121712338, |
| "pass@128": 0.5525525525525525 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 587027, |
| "answer_accuracy": 0.7149101227591582, |
| "avg_response_len": 267.97832959859704, |
| "resp_tokens_sum": 220042366, |
| "pass_at_k": { |
| "pass@1": 0.7149101227591582, |
| "pass@2": 0.7551878248875328, |
| "pass@4": 0.788089845667179, |
| "pass@8": 0.8152628237248414, |
| "pass@16": 0.8379713914459063, |
| "pass@32": 0.8568640308079691, |
| "pass@64": 0.8730298669312079, |
| "pass@128": 0.8874512860483242 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9370446702453987, |
| "pass@2": 0.9524521007439255, |
| "pass@4": 0.9622697201193403, |
| "pass@8": 0.9685217512597472, |
| "pass@16": 0.9732031072775292, |
| "pass@32": 0.9783710405726299, |
| "pass@64": 0.985072569920195, |
| "pass@128": 0.9938650306748467 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999306582840237, |
| "pass@2": 0.9999989080044729, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9999094202898551, |
| "pass@2": 0.9999989301609038, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9994179799426934, |
| "pass@2": 0.9999027028856351, |
| "pass@4": 0.9999971459513121, |
| "pass@8": 0.9999999985260104, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9998419436416185, |
| "pass@2": 0.9999996444176414, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9954225352112676, |
| "pass@2": 0.9971879505378729, |
| "pass@4": 0.9978147124215107, |
| "pass@8": 0.9983444096217255, |
| "pass@16": 0.9990631538362801, |
| "pass@32": 0.9997387400876238, |
| "pass@64": 0.9999912792555583, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9916729041916168, |
| "pass@2": 0.9941191681597434, |
| "pass@4": 0.9953397361632191, |
| "pass@8": 0.9964540303892805, |
| "pass@16": 0.9977764166281999, |
| "pass@32": 0.9990105981622154, |
| "pass@64": 0.99982149369308, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.932652484939759, |
| "pass@2": 0.9551909656341906, |
| "pass@4": 0.967504908272008, |
| "pass@8": 0.9743332008558964, |
| "pass@16": 0.9776944087377336, |
| "pass@32": 0.9797107289052188, |
| "pass@64": 0.981380697080861, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8275127351097179, |
| "pass@2": 0.8741306802754675, |
| "pass@4": 0.8940140824873376, |
| "pass@8": 0.9059417569605126, |
| "pass@16": 0.9149301720126192, |
| "pass@32": 0.9210451859792664, |
| "pass@64": 0.9267178933506086, |
| "pass@128": 0.9341692789968652 |
| }, |
| "13": { |
| "pass@1": 0.6466211484593838, |
| "pass@2": 0.7352568980348043, |
| "pass@4": 0.7941474946934152, |
| "pass@8": 0.8322874017240125, |
| "pass@16": 0.8593893686275413, |
| "pass@32": 0.8812230108196569, |
| "pass@64": 0.8990334067498752, |
| "pass@128": 0.9131652661064426 |
| }, |
| "14": { |
| "pass@1": 0.6178956505847953, |
| "pass@2": 0.7086092548464337, |
| "pass@4": 0.7747260120445467, |
| "pass@8": 0.8222148893486398, |
| "pass@16": 0.856872436091234, |
| "pass@32": 0.8836770532613494, |
| "pass@64": 0.9065420227864525, |
| "pass@128": 0.9269005847953217 |
| }, |
| "15": { |
| "pass@1": 0.5457149621212121, |
| "pass@2": 0.661665995585779, |
| "pass@4": 0.7447602410494136, |
| "pass@8": 0.8003608834636073, |
| "pass@16": 0.837934698476195, |
| "pass@32": 0.8649150245288428, |
| "pass@64": 0.8867527597855025, |
| "pass@128": 0.906060606060606 |
| }, |
| "16": { |
| "pass@1": 0.3033297720797721, |
| "pass@2": 0.38540860477376226, |
| "pass@4": 0.4561851985382169, |
| "pass@8": 0.5197053256135613, |
| "pass@16": 0.5790541044663001, |
| "pass@32": 0.6333955577098603, |
| "pass@64": 0.6845196910184368, |
| "pass@128": 0.7321937321937322 |
| }, |
| "17": { |
| "pass@1": 0.2654794254658385, |
| "pass@2": 0.3596355822370031, |
| "pass@4": 0.4481225438823254, |
| "pass@8": 0.5224691510802295, |
| "pass@16": 0.580719897610317, |
| "pass@32": 0.6246834181217727, |
| "pass@64": 0.6622791915081849, |
| "pass@128": 0.7080745341614907 |
| }, |
| "18": { |
| "pass@1": 0.20091911764705883, |
| "pass@2": 0.28180508916164887, |
| "pass@4": 0.3687975969915525, |
| "pass@8": 0.4557692555372953, |
| "pass@16": 0.5351237386501738, |
| "pass@32": 0.6004961265979195, |
| "pass@64": 0.6507879587120721, |
| "pass@128": 0.6911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.16022600446428573, |
| "pass@2": 0.22008833368485176, |
| "pass@4": 0.2805208366811292, |
| "pass@8": 0.3395545181912359, |
| "pass@16": 0.3959618657365702, |
| "pass@32": 0.4475826196030161, |
| "pass@64": 0.49460998571116155, |
| "pass@128": 0.5327380952380952 |
| }, |
| "20": { |
| "pass@1": 0.14796968562874252, |
| "pass@2": 0.2126687815314254, |
| "pass@4": 0.2800213515600969, |
| "pass@8": 0.3452822815254244, |
| "pass@16": 0.40522265822512654, |
| "pass@32": 0.4578358076989143, |
| "pass@64": 0.5011268713315313, |
| "pass@128": 0.5329341317365269 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 575456, |
| "answer_accuracy": 0.7182856686371625, |
| "avg_response_len": 287.9076030516057, |
| "resp_tokens_sum": 230657752, |
| "pass_at_k": { |
| "pass@1": 0.7182856686371625, |
| "pass@2": 0.7592052287226031, |
| "pass@4": 0.7927262515206405, |
| "pass@8": 0.8195860006912514, |
| "pass@16": 0.8409061998859412, |
| "pass@32": 0.8587729906883347, |
| "pass@64": 0.874955447824755, |
| "pass@128": 0.8894392075411408 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9499766791044776, |
| "pass@2": 0.963093856504877, |
| "pass@4": 0.9712197354062085, |
| "pass@8": 0.9767062892294843, |
| "pass@16": 0.9814928519708268, |
| "pass@32": 0.9866806213656, |
| "pass@64": 0.9918038267917523, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9995863970588236, |
| "pass@2": 0.9999811834182493, |
| "pass@4": 0.9999999611261827, |
| "pass@8": 0.9999999999999815, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9982009069400631, |
| "pass@2": 0.9996223670483617, |
| "pass@4": 0.999974433992281, |
| "pass@8": 0.999999864083505, |
| "pass@16": 0.9999999999987256, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9979232594936709, |
| "pass@2": 0.9995487547343765, |
| "pass@4": 0.9999514044130559, |
| "pass@8": 0.9999994224405141, |
| "pass@16": 0.9999999999663859, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9945101351351351, |
| "pass@2": 0.9986632794211536, |
| "pass@4": 0.9998438415918729, |
| "pass@8": 0.9999972864530364, |
| "pass@16": 0.9999999995382962, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9925894292604501, |
| "pass@2": 0.9951194869230575, |
| "pass@4": 0.9964745589478163, |
| "pass@8": 0.998042664097683, |
| "pass@16": 0.9993761183512748, |
| "pass@32": 0.9999366961026899, |
| "pass@64": 0.9999998004436335, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9227926587301587, |
| "pass@2": 0.9470242000999873, |
| "pass@4": 0.960732205795704, |
| "pass@8": 0.9693603886062503, |
| "pass@16": 0.9756707700497491, |
| "pass@32": 0.9806056964313475, |
| "pass@64": 0.9844370895991283, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.8355074541284404, |
| "pass@2": 0.8936402122613109, |
| "pass@4": 0.9236655713333995, |
| "pass@8": 0.9403579265835318, |
| "pass@16": 0.9498986664429232, |
| "pass@32": 0.9564481186800877, |
| "pass@64": 0.9620879995236127, |
| "pass@128": 0.9663608562691132 |
| }, |
| "13": { |
| "pass@1": 0.6571662808641975, |
| "pass@2": 0.7358156836298242, |
| "pass@4": 0.7884919118906429, |
| "pass@8": 0.8258481457444613, |
| "pass@16": 0.854241760542711, |
| "pass@32": 0.8774085425557125, |
| "pass@64": 0.8968649961634577, |
| "pass@128": 0.9135802469135802 |
| }, |
| "14": { |
| "pass@1": 0.6426418768545994, |
| "pass@2": 0.7360609330825484, |
| "pass@4": 0.7982204229293296, |
| "pass@8": 0.8372049902894257, |
| "pass@16": 0.8620695977837106, |
| "pass@32": 0.8819375600885098, |
| "pass@64": 0.901686644657056, |
| "pass@128": 0.9198813056379822 |
| }, |
| "15": { |
| "pass@1": 0.5323350694444444, |
| "pass@2": 0.6377064195100618, |
| "pass@4": 0.7172337514523645, |
| "pass@8": 0.7761564818619724, |
| "pass@16": 0.8170385569299585, |
| "pass@32": 0.8452706602617858, |
| "pass@64": 0.8681272297013932, |
| "pass@128": 0.8858024691358025 |
| }, |
| "16": { |
| "pass@1": 0.2906296091445428, |
| "pass@2": 0.3808380078043341, |
| "pass@4": 0.4639649562609984, |
| "pass@8": 0.5348016643194988, |
| "pass@16": 0.5918998310984556, |
| "pass@32": 0.6414011766481142, |
| "pass@64": 0.6911248562212414, |
| "pass@128": 0.7433628318584071 |
| }, |
| "17": { |
| "pass@1": 0.24602315512048192, |
| "pass@2": 0.33004977587515394, |
| "pass@4": 0.41139936376552333, |
| "pass@8": 0.48491044209939826, |
| "pass@16": 0.5510107149708241, |
| "pass@32": 0.6105205843571422, |
| "pass@64": 0.6619147865574031, |
| "pass@128": 0.7048192771084337 |
| }, |
| "18": { |
| "pass@1": 0.24233870967741936, |
| "pass@2": 0.31930681673863354, |
| "pass@4": 0.3933600641653665, |
| "pass@8": 0.4584629930505219, |
| "pass@16": 0.5141375485482685, |
| "pass@32": 0.5606915077929782, |
| "pass@64": 0.6005003708826756, |
| "pass@128": 0.6387096774193548 |
| }, |
| "19": { |
| "pass@1": 0.18074962235649547, |
| "pass@2": 0.2564385243951755, |
| "pass@4": 0.33101056353982955, |
| "pass@8": 0.39293476132895816, |
| "pass@16": 0.4417977642220465, |
| "pass@32": 0.486950509381447, |
| "pass@64": 0.5332543587477647, |
| "pass@128": 0.5740181268882175 |
| }, |
| "20": { |
| "pass@1": 0.1501266891891892, |
| "pass@2": 0.22014177070771548, |
| "pass@4": 0.29605818079046403, |
| "pass@8": 0.36845622163096464, |
| "pass@16": 0.4307900693887324, |
| "pass@32": 0.4817076892328625, |
| "pass@64": 0.5253941303577103, |
| "pass@128": 0.5615615615615616 |
| } |
| } |
| } |
| } |
| } |
| ] |
