| { |
| "checkpoint": "checkpoint-38796", |
| "id": { |
| "answer_accuracy": 0.9916449652777778, |
| "avg_response_len": 200.28180034722223, |
| "avg_loss": 0.0, |
| "count": 1152000, |
| "validation_metrics": {}, |
| "pass_at_k": { |
| "pass@1": 0.9916449652777778, |
| "pass@2": 0.9942652148950116, |
| "pass@4": 0.9960372402824641, |
| "pass@8": 0.9973541288508918, |
| "pass@16": 0.9983286171835405, |
| "pass@32": 0.9990140951617359, |
| "pass@64": 0.9995384212572507, |
| "pass@128": 0.9998888888888889 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9466015625, |
| "pass@2": 0.9610051673228348, |
| "pass@4": 0.9726838771091111, |
| "pass@8": 0.9821482544180546, |
| "pass@16": 0.9891967479504874, |
| "pass@32": 0.9936529337175701, |
| "pass@64": 0.9968173920169906, |
| "pass@128": 0.999 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997734375, |
| "pass@2": 0.9999870816929133, |
| "pass@4": 0.9999999652230971, |
| "pass@8": 0.9999999999999623, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999140625, |
| "pass@2": 0.9997561515748032, |
| "pass@4": 0.999979291807274, |
| "pass@8": 0.9999998345181814, |
| "pass@16": 0.9999999999930597, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99578125, |
| "pass@2": 0.9977439714566931, |
| "pass@4": 0.9987070024371953, |
| "pass@8": 0.9991198166176619, |
| "pass@16": 0.9993307064026963, |
| "pass@32": 0.9995814578957427, |
| "pass@64": 0.9998779527559055, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.99965625, |
| "pass@2": 0.9999856053149606, |
| "pass@4": 0.9999999461004874, |
| "pass@8": 0.9999999999996532, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.996609375, |
| "pass@2": 0.9978914862204724, |
| "pass@4": 0.9985337887139109, |
| "pass@8": 0.9990995140827746, |
| "pass@16": 0.9995617639865387, |
| "pass@32": 0.9998720986680215, |
| "pass@64": 0.9999934279136211, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9941640625, |
| "pass@2": 0.9970008612204726, |
| "pass@4": 0.9984830455568053, |
| "pass@8": 0.9991411297284428, |
| "pass@16": 0.9994807385991464, |
| "pass@32": 0.9997688993231002, |
| "pass@64": 0.9999711811023325, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.993078125, |
| "pass@2": 0.9950166092519686, |
| "pass@4": 0.9959482455943007, |
| "pass@8": 0.9966786102933006, |
| "pass@16": 0.9973875977199353, |
| "pass@32": 0.9982514668511898, |
| "pass@64": 0.9991858375264056, |
| "pass@128": 1.0 |
| } |
| }, |
| "per_op_accuracy": { |
| "10": 0.9466015625, |
| "2": 1.0, |
| "3": 0.9997734375, |
| "4": 0.999140625, |
| "5": 0.99578125, |
| "6": 0.99965625, |
| "7": 0.996609375, |
| "8": 0.9941640625, |
| "9": 0.993078125 |
| }, |
| "per_op_avg_response_len": { |
| "10": 281.451125, |
| "2": 115.6058828125, |
| "3": 144.85159375, |
| "4": 159.9671640625, |
| "5": 173.505046875, |
| "6": 199.2678984375, |
| "7": 224.5584296875, |
| "8": 243.6461640625, |
| "9": 259.6828984375 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "10": 281.451125, |
| "2": 115.6058828125, |
| "3": 144.85159375, |
| "4": 159.9671640625, |
| "5": 173.505046875, |
| "6": 199.2678984375, |
| "7": 224.5584296875, |
| "8": 243.6461640625, |
| "9": 259.6828984375 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 379008, |
| "correct": 375891, |
| "answer_accuracy": 0.9917758991894631, |
| "avg_response_len": 190.7352694402229, |
| "resp_tokens_sum": 72290193, |
| "pass_at_k": { |
| "pass@1": 0.9917758991894631, |
| "pass@2": 0.9941626312668362, |
| "pass@4": 0.995686200160897, |
| "pass@8": 0.9969115368390852, |
| "pass@16": 0.9979796890499223, |
| "pass@32": 0.998873696427674, |
| "pass@64": 0.9995997924414013, |
| "pass@128": 1.0 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9496450958702065, |
| "pass@2": 0.9618148706245786, |
| "pass@4": 0.9709195504323019, |
| "pass@8": 0.9791190874933853, |
| "pass@16": 0.9868342218489051, |
| "pass@32": 0.993298633445245, |
| "pass@64": 0.9981549989323758, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997849770642202, |
| "pass@2": 0.9999864552481399, |
| "pass@4": 0.9999999638806617, |
| "pass@8": 0.9999999999999809, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999755859375, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9954117063492064, |
| "pass@2": 0.9981478877640296, |
| "pass@4": 0.9995556263800359, |
| "pass@8": 0.9999615639259287, |
| "pass@16": 0.9999997091342684, |
| "pass@32": 0.9999999999966475, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997427020958084, |
| "pass@2": 0.999995948064501, |
| "pass@4": 0.9999999985967324, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9956083579881657, |
| "pass@2": 0.9976401976657503, |
| "pass@4": 0.9985032811172272, |
| "pass@8": 0.9992715079029393, |
| "pass@16": 0.9998386699365346, |
| "pass@32": 0.9999946547625592, |
| "pass@64": 0.9999999995148912, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9960186298076923, |
| "pass@2": 0.9985326916767614, |
| "pass@4": 0.9996494007960544, |
| "pass@8": 0.9999753031040152, |
| "pass@16": 0.9999998899899415, |
| "pass@32": 0.9999999999996564, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9917033450704226, |
| "pass@2": 0.9929716091826549, |
| "pass@4": 0.9939165605179635, |
| "pass@8": 0.9949288254759624, |
| "pass@16": 0.9958752495136213, |
| "pass@32": 0.9970100988051338, |
| "pass@64": 0.9984237773095228, |
| "pass@128": 1.0 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 390656, |
| "correct": 387076, |
| "answer_accuracy": 0.9908359272608126, |
| "avg_response_len": 198.307644065367, |
| "resp_tokens_sum": 77470071, |
| "pass_at_k": { |
| "pass@1": 0.9908359272608126, |
| "pass@2": 0.9934299891125998, |
| "pass@4": 0.9953776159799322, |
| "pass@8": 0.9968998465378482, |
| "pass@16": 0.9979383162737498, |
| "pass@32": 0.9986008798739462, |
| "pass@64": 0.9991527101183229, |
| "pass@128": 0.9996723460026212 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9329706671779141, |
| "pass@2": 0.9490174085792956, |
| "pass@4": 0.9641241520423445, |
| "pass@8": 0.976390759682789, |
| "pass@16": 0.9846418367918528, |
| "pass@32": 0.9893313200761069, |
| "pass@64": 0.9929169376529189, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995265151515151, |
| "pass@2": 0.9999742752326415, |
| "pass@4": 0.9999999304064264, |
| "pass@8": 0.9999999999999045, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9985207100591716, |
| "pass@2": 0.9996389134790105, |
| "pass@4": 0.9999800542683642, |
| "pass@8": 0.9999999564675484, |
| "pass@16": 0.999999999999962, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9973505434782609, |
| "pass@2": 0.9985510812507132, |
| "pass@4": 0.9993644259141521, |
| "pass@8": 0.9998696856114555, |
| "pass@16": 0.9999955044084027, |
| "pass@32": 0.9999999980554193, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9993508237822349, |
| "pass@2": 0.999962984793448, |
| "pass@4": 0.9999998469030911, |
| "pass@8": 0.9999999999990058, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9994806719653179, |
| "pass@2": 0.9999896881116016, |
| "pass@4": 0.9999999953943619, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.993155809859155, |
| "pass@2": 0.9950745813463459, |
| "pass@4": 0.9964881184746273, |
| "pass@8": 0.9976291893269761, |
| "pass@16": 0.9985375133295249, |
| "pass@32": 0.9993490121785924, |
| "pass@64": 0.9999188200065704, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9948774326347305, |
| "pass@2": 0.9968984275543402, |
| "pass@4": 0.9971880869868812, |
| "pass@8": 0.9973699249827601, |
| "pass@16": 0.9977102880852468, |
| "pass@32": 0.9983202885567448, |
| "pass@64": 0.9992573907303504, |
| "pass@128": 1.0 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 382336, |
| "correct": 379408, |
| "answer_accuracy": 0.9923418145296284, |
| "avg_response_len": 211.76235039337126, |
| "resp_tokens_sum": 80964370, |
| "pass_at_k": { |
| "pass@1": 0.9923418145296284, |
| "pass@2": 0.9952203066964725, |
| "pass@4": 0.9970592031787792, |
| "pass@8": 0.9982570363053186, |
| "pass@16": 0.99907330234602, |
| "pass@32": 0.9995754790619344, |
| "pass@64": 0.999871688856761, |
| "pass@128": 1.0 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9567863805970149, |
| "pass@2": 0.971851495475379, |
| "pass@4": 0.9827990326768856, |
| "pass@8": 0.9908164062722622, |
| "pass@16": 0.9960200237300442, |
| "pass@32": 0.9982169750293176, |
| "pass@64": 0.9992594797136242, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9989720394736842, |
| "pass@2": 0.9996438561956071, |
| "pass@4": 0.9999591621770964, |
| "pass@8": 0.9999995591583335, |
| "pass@16": 0.9999999999797444, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99453125, |
| "pass@2": 0.9965507758221399, |
| "pass@4": 0.9972536858444163, |
| "pass@8": 0.9975790660148303, |
| "pass@16": 0.9980363206014791, |
| "pass@32": 0.9987689957872913, |
| "pass@64": 0.9996410375173692, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999014195583596, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9945361946202531, |
| "pass@2": 0.9958628712747932, |
| "pass@4": 0.9969610167399962, |
| "pass@8": 0.997929570922725, |
| "pass@16": 0.9987857390759178, |
| "pass@32": 0.9996009663236596, |
| "pass@64": 0.9999792027771768, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9935013138138138, |
| "pass@2": 0.9976191742924027, |
| "pass@4": 0.9995169683181494, |
| "pass@8": 0.9999713902369773, |
| "pass@16": 0.9999998669378484, |
| "pass@32": 0.9999999999994204, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9927150321543409, |
| "pass@2": 0.9953299459452619, |
| "pass@4": 0.9969358378032327, |
| "pass@8": 0.997933512235023, |
| "pass@16": 0.9987673566627887, |
| "pass@32": 0.9995945511170884, |
| "pass@64": 0.9999788684166812, |
| "pass@128": 1.0 |
| } |
| } |
| } |
| } |
| }, |
| "ood": { |
| "answer_accuracy": 0.88128359375, |
| "avg_response_len": 381.877265625, |
| "avg_loss": 0.0, |
| "count": 1280000, |
| "validation_metrics": {}, |
| "pass_at_k": { |
| "pass@1": 0.88128359375, |
| "pass@2": 0.9169421874999801, |
| "pass@4": 0.9403016251687241, |
| "pass@8": 0.9566609083280344, |
| "pass@16": 0.968339831127487, |
| "pass@32": 0.9765863281317172, |
| "pass@64": 0.9824299016731352, |
| "pass@128": 0.9866 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.96628125, |
| "pass@2": 0.9785717273622044, |
| "pass@4": 0.9873329736595432, |
| "pass@8": 0.9931112776863448, |
| "pass@16": 0.9965805325156889, |
| "pass@32": 0.9983545476738703, |
| "pass@64": 0.9989335858094942, |
| "pass@128": 0.999 |
| }, |
| "12": { |
| "pass@1": 0.9478046875, |
| "pass@2": 0.9690962106299207, |
| "pass@4": 0.9831062861829775, |
| "pass@8": 0.9913355415158315, |
| "pass@16": 0.996070057574652, |
| "pass@32": 0.9987139354985618, |
| "pass@64": 0.9997237180814249, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.924203125, |
| "pass@2": 0.9514560777559056, |
| "pass@4": 0.9705920066554181, |
| "pass@8": 0.983524048574781, |
| "pass@16": 0.9914554111118253, |
| "pass@32": 0.995615897875724, |
| "pass@64": 0.9977893333892781, |
| "pass@128": 0.999 |
| }, |
| "14": { |
| "pass@1": 0.94225, |
| "pass@2": 0.9635307578740164, |
| "pass@4": 0.9773012244094489, |
| "pass@8": 0.9867741013499881, |
| "pass@16": 0.9931041553200217, |
| "pass@32": 0.9969675206156218, |
| "pass@64": 0.9990483415511645, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.952296875, |
| "pass@2": 0.9725359251968507, |
| "pass@4": 0.983482671259842, |
| "pass@8": 0.9906915517961334, |
| "pass@16": 0.9953419881536004, |
| "pass@32": 0.997607897381467, |
| "pass@64": 0.998434589414787, |
| "pass@128": 0.999 |
| }, |
| "16": { |
| "pass@1": 0.8923984375, |
| "pass@2": 0.9232349901574809, |
| "pass@4": 0.944543639295088, |
| "pass@8": 0.9600311462677881, |
| "pass@16": 0.9714450156787062, |
| "pass@32": 0.9800956769939823, |
| "pass@64": 0.986847916902909, |
| "pass@128": 0.992 |
| }, |
| "17": { |
| "pass@1": 0.87903125, |
| "pass@2": 0.9084288877952762, |
| "pass@4": 0.9289755383389573, |
| "pass@8": 0.9456187949405052, |
| "pass@16": 0.9588866468354461, |
| "pass@32": 0.9690113051788568, |
| "pass@64": 0.976362763314076, |
| "pass@128": 0.983 |
| }, |
| "18": { |
| "pass@1": 0.8645078125, |
| "pass@2": 0.9091412401574817, |
| "pass@4": 0.9346570592425941, |
| "pass@8": 0.9512213333892252, |
| "pass@16": 0.9633840273152701, |
| "pass@32": 0.9726363613074998, |
| "pass@64": 0.979410245686971, |
| "pass@128": 0.983 |
| }, |
| "19": { |
| "pass@1": 0.750859375, |
| "pass@2": 0.8167075541338586, |
| "pass@4": 0.860467028308961, |
| "pass@8": 0.8921891852379328, |
| "pass@16": 0.9155518651584171, |
| "pass@32": 0.9329156898194609, |
| "pass@64": 0.946665061674124, |
| "pass@128": 0.957 |
| }, |
| "20": { |
| "pass@1": 0.693203125, |
| "pass@2": 0.7767185039370087, |
| "pass@4": 0.8325578243344577, |
| "pass@8": 0.8721121025217957, |
| "pass@16": 0.9015786116111969, |
| "pass@32": 0.9239444489720972, |
| "pass@64": 0.9410834609070993, |
| "pass@128": 0.954 |
| } |
| }, |
| "per_op_accuracy": { |
| "11": 0.96628125, |
| "12": 0.9478046875, |
| "13": 0.924203125, |
| "14": 0.94225, |
| "15": 0.952296875, |
| "16": 0.8923984375, |
| "17": 0.87903125, |
| "18": 0.8645078125, |
| "19": 0.750859375, |
| "20": 0.693203125 |
| }, |
| "per_op_avg_response_len": { |
| "11": 300.90371875, |
| "12": 313.225015625, |
| "13": 334.0887734375, |
| "14": 361.206921875, |
| "15": 381.5723984375, |
| "16": 391.865984375, |
| "17": 413.0096484375, |
| "18": 431.9963671875, |
| "19": 435.01728125, |
| "20": 455.886546875 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "11": 300.90371875, |
| "12": 313.225015625, |
| "13": 334.0887734375, |
| "14": 361.206921875, |
| "15": 381.5723984375, |
| "16": 391.865984375, |
| "17": 413.0096484375, |
| "18": 431.9963671875, |
| "19": 435.01728125, |
| "20": 455.886546875 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 430720, |
| "correct": 378677, |
| "answer_accuracy": 0.8791720839524517, |
| "avg_response_len": 363.8872585438336, |
| "resp_tokens_sum": 156733520, |
| "pass_at_k": { |
| "pass@1": 0.8791720839524517, |
| "pass@2": 0.9144079570848619, |
| "pass@4": 0.9376800844730981, |
| "pass@8": 0.9540837108661213, |
| "pass@16": 0.9657676862206042, |
| "pass@32": 0.9739935535366472, |
| "pass@64": 0.9799957539858889, |
| "pass@128": 0.9842496285289747 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9635047804532578, |
| "pass@2": 0.977436093328277, |
| "pass@4": 0.9865923250386901, |
| "pass@8": 0.9927103321039095, |
| "pass@16": 0.9970124444229089, |
| "pass@32": 0.9993847243796702, |
| "pass@64": 0.9999852421618102, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9450918079096046, |
| "pass@2": 0.9655720338983056, |
| "pass@4": 0.9781990243804268, |
| "pass@8": 0.9861440250122284, |
| "pass@16": 0.9923112374071664, |
| "pass@32": 0.9969355017981401, |
| "pass@64": 0.9992276493263522, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9179075235109718, |
| "pass@2": 0.9502324099671706, |
| "pass@4": 0.9729623692218725, |
| "pass@8": 0.9865352063542692, |
| "pass@16": 0.9929416594162586, |
| "pass@32": 0.9950239922483407, |
| "pass@64": 0.9960875780112477, |
| "pass@128": 0.9968652037617555 |
| }, |
| "14": { |
| "pass@1": 0.9201956775700935, |
| "pass@2": 0.9446224121225502, |
| "pass@4": 0.9626010729383125, |
| "pass@8": 0.9762203698107813, |
| "pass@16": 0.986550455857625, |
| "pass@32": 0.9938274615772782, |
| "pass@64": 0.9980046339203555, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9622696893063584, |
| "pass@2": 0.9790736226161763, |
| "pass@4": 0.9878219928975642, |
| "pass@8": 0.9938863424829786, |
| "pass@16": 0.9974854612936562, |
| "pass@32": 0.9990553826707531, |
| "pass@64": 0.9998278336678724, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8738155241935484, |
| "pass@2": 0.9108982092964184, |
| "pass@4": 0.9368475640141757, |
| "pass@8": 0.9526611840974472, |
| "pass@16": 0.9624771254488251, |
| "pass@32": 0.9706463781509711, |
| "pass@64": 0.97815086262173, |
| "pass@128": 0.9838709677419355 |
| }, |
| "17": { |
| "pass@1": 0.8999954841040463, |
| "pass@2": 0.9269427597742481, |
| "pass@4": 0.9442069857777889, |
| "pass@8": 0.9584880211486807, |
| "pass@16": 0.9699099356325508, |
| "pass@32": 0.977956592718777, |
| "pass@64": 0.9832164176739884, |
| "pass@128": 0.9884393063583815 |
| }, |
| "18": { |
| "pass@1": 0.8340625, |
| "pass@2": 0.875173298650169, |
| "pass@4": 0.9003785883014621, |
| "pass@8": 0.9191113837595276, |
| "pass@16": 0.9347451165816725, |
| "pass@32": 0.9484697556158473, |
| "pass@64": 0.9606376972907202, |
| "pass@128": 0.9685714285714285 |
| }, |
| "19": { |
| "pass@1": 0.7546218093093093, |
| "pass@2": 0.8168980988862881, |
| "pass@4": 0.8594574675913256, |
| "pass@8": 0.8903657440259747, |
| "pass@16": 0.9127998493831533, |
| "pass@32": 0.9301485120186141, |
| "pass@64": 0.9444138248825593, |
| "pass@128": 0.954954954954955 |
| }, |
| "20": { |
| "pass@1": 0.7120167042042042, |
| "pass@2": 0.7915602610484499, |
| "pass@4": 0.8442564102347557, |
| "pass@8": 0.8824961305644767, |
| "pass@16": 0.9097409929460565, |
| "pass@32": 0.927053074881089, |
| "pass@64": 0.9392787772648254, |
| "pass@128": 0.948948948948949 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 430464, |
| "correct": 382426, |
| "answer_accuracy": 0.8884041406482307, |
| "avg_response_len": 377.3070268361582, |
| "resp_tokens_sum": 162417092, |
| "pass_at_k": { |
| "pass@1": 0.8884041406482307, |
| "pass@2": 0.9226806496004516, |
| "pass@4": 0.9454679196824326, |
| "pass@8": 0.961442993074218, |
| "pass@16": 0.972427449762373, |
| "pass@32": 0.9796245471643623, |
| "pass@64": 0.9842425816177335, |
| "pass@128": 0.9878085043116265 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9722797439759037, |
| "pass@2": 0.9806206864861019, |
| "pass@4": 0.987148461675724, |
| "pass@8": 0.992097089748973, |
| "pass@16": 0.9952356342084575, |
| "pass@32": 0.996694117094797, |
| "pass@64": 0.9969831820276938, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9480309561128527, |
| "pass@2": 0.9714501363759779, |
| "pass@4": 0.9865298393343465, |
| "pass@8": 0.9942286337372813, |
| "pass@16": 0.9978628049514008, |
| "pass@32": 0.9996281644114471, |
| "pass@64": 0.9999956190522251, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9407169117647058, |
| "pass@2": 0.9625553469419262, |
| "pass@4": 0.9774801687079033, |
| "pass@8": 0.9874219685475781, |
| "pass@16": 0.9934829131966962, |
| "pass@32": 0.9968338780653557, |
| "pass@64": 0.9985126025262858, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9497669956140351, |
| "pass@2": 0.9702634312980611, |
| "pass@4": 0.9843248648195289, |
| "pass@8": 0.9936793355130432, |
| "pass@16": 0.9984837564707297, |
| "pass@32": 0.999880078009628, |
| "pass@64": 0.999999559195243, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9554214015151515, |
| "pass@2": 0.9778524665950848, |
| "pass@4": 0.990172612230289, |
| "pass@8": 0.9965689152787764, |
| "pass@16": 0.9990389896488282, |
| "pass@32": 0.9998467083150427, |
| "pass@64": 0.9999979765261304, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8984820156695157, |
| "pass@2": 0.9263237628149055, |
| "pass@4": 0.9466511985147156, |
| "pass@8": 0.9628916229912796, |
| "pass@16": 0.9756373410907596, |
| "pass@32": 0.9843113854696494, |
| "pass@64": 0.9896611343512275, |
| "pass@128": 0.9943019943019943 |
| }, |
| "17": { |
| "pass@1": 0.8989955357142857, |
| "pass@2": 0.9258540372670809, |
| "pass@4": 0.9430381279009376, |
| "pass@8": 0.9567724437885808, |
| "pass@16": 0.9678264048731167, |
| "pass@32": 0.975789443612367, |
| "pass@64": 0.9804645618033802, |
| "pass@128": 0.984472049689441 |
| }, |
| "18": { |
| "pass@1": 0.8785386029411765, |
| "pass@2": 0.9263135421491432, |
| "pass@4": 0.9540938742032239, |
| "pass@8": 0.970755896754488, |
| "pass@16": 0.9808158406179858, |
| "pass@32": 0.9865832611536812, |
| "pass@64": 0.989914086501943, |
| "pass@128": 0.9911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.7576497395833334, |
| "pass@2": 0.8213040752718409, |
| "pass@4": 0.8629211197930617, |
| "pass@8": 0.8939620372022703, |
| "pass@16": 0.9169075579630253, |
| "pass@32": 0.9325509025737705, |
| "pass@64": 0.94412425958293, |
| "pass@128": 0.9553571428571429 |
| }, |
| "20": { |
| "pass@1": 0.6838978293413174, |
| "pass@2": 0.7640517439530387, |
| "pass@4": 0.8220046661457729, |
| "pass@8": 0.8655627671390692, |
| "pass@16": 0.8983598878087751, |
| "pass@32": 0.9234805889172719, |
| "pass@64": 0.9421120203018302, |
| "pass@128": 0.9550898203592815 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 418816, |
| "correct": 366940, |
| "answer_accuracy": 0.8761365372860636, |
| "avg_response_len": 405.07594743276286, |
| "resp_tokens_sum": 169652288, |
| "pass_at_k": { |
| "pass@1": 0.8761365372860636, |
| "pass@2": 0.9136503896097687, |
| "pass@4": 0.9376876997381768, |
| "pass@8": 0.9543962746048996, |
| "pass@16": 0.9667837816600432, |
| "pass@32": 0.9761300799366067, |
| "pass@64": 0.9830701413778614, |
| "pass@128": 0.9877750611246944 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9630704365079366, |
| "pass@2": 0.9776848206474192, |
| "pass@4": 0.9883574401414106, |
| "pass@8": 0.9946295131968437, |
| "pass@16": 0.997513997070458, |
| "pass@32": 0.9989501368011887, |
| "pass@64": 0.9998107266450182, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9505208333333334, |
| "pass@2": 0.9706150370825212, |
| "pass@4": 0.9850789382405178, |
| "pass@8": 0.994133402138561, |
| "pass@16": 0.9983903509266632, |
| "pass@32": 0.9997473498922584, |
| "pass@64": 0.9999954976826797, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9122058256172839, |
| "pass@2": 0.9404311114756491, |
| "pass@4": 0.9606684896448127, |
| "pass@8": 0.9762644289391496, |
| "pass@16": 0.9877580918111676, |
| "pass@32": 0.9948566354295425, |
| "pass@64": 0.9986678916722411, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.955628709198813, |
| "pass@2": 0.9747088132900301, |
| "pass@4": 0.9841755971156422, |
| "pass@8": 0.9898190798079135, |
| "pass@16": 0.9938872827202818, |
| "pass@32": 0.9970027263205429, |
| "pass@64": 0.99907716563192, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9384645061728395, |
| "pass@2": 0.9601393141829493, |
| "pass@4": 0.9720348755595365, |
| "pass@8": 0.981293627330329, |
| "pass@16": 0.9892875060551926, |
| "pass@32": 0.9937818556587116, |
| "pass@64": 0.995354403432346, |
| "pass@128": 0.9969135802469136 |
| }, |
| "16": { |
| "pass@1": 0.9030927359882006, |
| "pass@2": 0.9313183024411773, |
| "pass@4": 0.949399185168225, |
| "pass@8": 0.9638089071611805, |
| "pass@16": 0.9753050149460608, |
| "pass@32": 0.9843716916440536, |
| "pass@64": 0.9918881750232778, |
| "pass@128": 0.9970501474926253 |
| }, |
| "17": { |
| "pass@1": 0.8378200301204819, |
| "pass@2": 0.8722340147993546, |
| "pass@4": 0.8994627833606642, |
| "pass@8": 0.921389194949213, |
| "pass@16": 0.9387280323416873, |
| "pass@32": 0.953114829081259, |
| "pass@64": 0.9652418490909267, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8834929435483871, |
| "pass@2": 0.9286580041910086, |
| "pass@4": 0.9520407616386656, |
| "pass@8": 0.966049497344724, |
| "pass@16": 0.9765995183921604, |
| "pass@32": 0.9846246388700052, |
| "pass@64": 0.989084716853413, |
| "pass@128": 0.9903225806451613 |
| }, |
| "19": { |
| "pass@1": 0.7401812688821753, |
| "pass@2": 0.8118499030615883, |
| "pass@4": 0.8589915267389155, |
| "pass@8": 0.8922240120160714, |
| "pass@16": 0.9169443378799137, |
| "pass@32": 0.9360698853549126, |
| "pass@64": 0.9515090838923458, |
| "pass@128": 0.9607250755287009 |
| }, |
| "20": { |
| "pass@1": 0.6837227852852853, |
| "pass@2": 0.7745815451278046, |
| "pass@4": 0.8314440877885758, |
| "pass@8": 0.8682970775356633, |
| "pass@16": 0.8966446199160057, |
| "pass@32": 0.92130107609107, |
| "pass@64": 0.9418564963876911, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| }, |
| "total": { |
| "answer_accuracy": 0.9335600328947369, |
| "avg_loss": 0.0, |
| "count": 2432000, |
| "validation_metrics": { |
| "id": {}, |
| "ood": {} |
| }, |
| "pass_at_k": { |
| "pass@1": 0.9335600328947369, |
| "pass@2": 0.9535688846871059, |
| "pass@4": 0.9667027060120662, |
| "pass@8": 0.9759366443651816, |
| "pass@16": 0.9825450455750843, |
| "pass@32": 0.9872100072512014, |
| "pass@64": 0.9905339372656115, |
| "pass@128": 0.9928947368421053 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9466015625, |
| "pass@2": 0.9610051673228348, |
| "pass@4": 0.9726838771091111, |
| "pass@8": 0.9821482544180546, |
| "pass@16": 0.9891967479504874, |
| "pass@32": 0.9936529337175701, |
| "pass@64": 0.9968173920169906, |
| "pass@128": 0.999 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997734375, |
| "pass@2": 0.9999870816929133, |
| "pass@4": 0.9999999652230971, |
| "pass@8": 0.9999999999999623, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999140625, |
| "pass@2": 0.9997561515748032, |
| "pass@4": 0.999979291807274, |
| "pass@8": 0.9999998345181814, |
| "pass@16": 0.9999999999930597, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99578125, |
| "pass@2": 0.9977439714566931, |
| "pass@4": 0.9987070024371953, |
| "pass@8": 0.9991198166176619, |
| "pass@16": 0.9993307064026963, |
| "pass@32": 0.9995814578957427, |
| "pass@64": 0.9998779527559055, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.99965625, |
| "pass@2": 0.9999856053149606, |
| "pass@4": 0.9999999461004874, |
| "pass@8": 0.9999999999996532, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.996609375, |
| "pass@2": 0.9978914862204724, |
| "pass@4": 0.9985337887139109, |
| "pass@8": 0.9990995140827746, |
| "pass@16": 0.9995617639865387, |
| "pass@32": 0.9998720986680215, |
| "pass@64": 0.9999934279136211, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9941640625, |
| "pass@2": 0.9970008612204726, |
| "pass@4": 0.9984830455568053, |
| "pass@8": 0.9991411297284428, |
| "pass@16": 0.9994807385991464, |
| "pass@32": 0.9997688993231002, |
| "pass@64": 0.9999711811023325, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.993078125, |
| "pass@2": 0.9950166092519686, |
| "pass@4": 0.9959482455943007, |
| "pass@8": 0.9966786102933006, |
| "pass@16": 0.9973875977199353, |
| "pass@32": 0.9982514668511898, |
| "pass@64": 0.9991858375264056, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.96628125, |
| "pass@2": 0.9785717273622044, |
| "pass@4": 0.9873329736595432, |
| "pass@8": 0.9931112776863448, |
| "pass@16": 0.9965805325156889, |
| "pass@32": 0.9983545476738703, |
| "pass@64": 0.9989335858094942, |
| "pass@128": 0.999 |
| }, |
| "12": { |
| "pass@1": 0.9478046875, |
| "pass@2": 0.9690962106299207, |
| "pass@4": 0.9831062861829775, |
| "pass@8": 0.9913355415158315, |
| "pass@16": 0.996070057574652, |
| "pass@32": 0.9987139354985618, |
| "pass@64": 0.9997237180814249, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.924203125, |
| "pass@2": 0.9514560777559056, |
| "pass@4": 0.9705920066554181, |
| "pass@8": 0.983524048574781, |
| "pass@16": 0.9914554111118253, |
| "pass@32": 0.995615897875724, |
| "pass@64": 0.9977893333892781, |
| "pass@128": 0.999 |
| }, |
| "14": { |
| "pass@1": 0.94225, |
| "pass@2": 0.9635307578740164, |
| "pass@4": 0.9773012244094489, |
| "pass@8": 0.9867741013499881, |
| "pass@16": 0.9931041553200217, |
| "pass@32": 0.9969675206156218, |
| "pass@64": 0.9990483415511645, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.952296875, |
| "pass@2": 0.9725359251968507, |
| "pass@4": 0.983482671259842, |
| "pass@8": 0.9906915517961334, |
| "pass@16": 0.9953419881536004, |
| "pass@32": 0.997607897381467, |
| "pass@64": 0.998434589414787, |
| "pass@128": 0.999 |
| }, |
| "16": { |
| "pass@1": 0.8923984375, |
| "pass@2": 0.9232349901574809, |
| "pass@4": 0.944543639295088, |
| "pass@8": 0.9600311462677881, |
| "pass@16": 0.9714450156787062, |
| "pass@32": 0.9800956769939823, |
| "pass@64": 0.986847916902909, |
| "pass@128": 0.992 |
| }, |
| "17": { |
| "pass@1": 0.87903125, |
| "pass@2": 0.9084288877952762, |
| "pass@4": 0.9289755383389573, |
| "pass@8": 0.9456187949405052, |
| "pass@16": 0.9588866468354461, |
| "pass@32": 0.9690113051788568, |
| "pass@64": 0.976362763314076, |
| "pass@128": 0.983 |
| }, |
| "18": { |
| "pass@1": 0.8645078125, |
| "pass@2": 0.9091412401574817, |
| "pass@4": 0.9346570592425941, |
| "pass@8": 0.9512213333892252, |
| "pass@16": 0.9633840273152701, |
| "pass@32": 0.9726363613074998, |
| "pass@64": 0.979410245686971, |
| "pass@128": 0.983 |
| }, |
| "19": { |
| "pass@1": 0.750859375, |
| "pass@2": 0.8167075541338586, |
| "pass@4": 0.860467028308961, |
| "pass@8": 0.8921891852379328, |
| "pass@16": 0.9155518651584171, |
| "pass@32": 0.9329156898194609, |
| "pass@64": 0.946665061674124, |
| "pass@128": 0.957 |
| }, |
| "20": { |
| "pass@1": 0.693203125, |
| "pass@2": 0.7767185039370087, |
| "pass@4": 0.8325578243344577, |
| "pass@8": 0.8721121025217957, |
| "pass@16": 0.9015786116111969, |
| "pass@32": 0.9239444489720972, |
| "pass@64": 0.9410834609070993, |
| "pass@128": 0.954 |
| } |
| }, |
| "per_op_accuracy": { |
| "10": 0.9466015625, |
| "2": 1.0, |
| "3": 0.9997734375, |
| "4": 0.999140625, |
| "5": 0.99578125, |
| "6": 0.99965625, |
| "7": 0.996609375, |
| "8": 0.9941640625, |
| "9": 0.993078125, |
| "11": 0.96628125, |
| "12": 0.9478046875, |
| "13": 0.924203125, |
| "14": 0.94225, |
| "15": 0.952296875, |
| "16": 0.8923984375, |
| "17": 0.87903125, |
| "18": 0.8645078125, |
| "19": 0.750859375, |
| "20": 0.693203125 |
| }, |
| "per_op_avg_response_len": { |
| "10": 281.451125, |
| "2": 115.6058828125, |
| "3": 144.85159375, |
| "4": 159.9671640625, |
| "5": 173.505046875, |
| "6": 199.2678984375, |
| "7": 224.5584296875, |
| "8": 243.6461640625, |
| "9": 259.6828984375, |
| "11": 300.90371875, |
| "12": 313.225015625, |
| "13": 334.0887734375, |
| "14": 361.206921875, |
| "15": 381.5723984375, |
| "16": 391.865984375, |
| "17": 413.0096484375, |
| "18": 431.9963671875, |
| "19": 435.01728125, |
| "20": 455.886546875 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "10": 281.451125, |
| "2": 115.6058828125, |
| "3": 144.85159375, |
| "4": 159.9671640625, |
| "5": 173.505046875, |
| "6": 199.2678984375, |
| "7": 224.5584296875, |
| "8": 243.6461640625, |
| "9": 259.6828984375, |
| "11": 300.90371875, |
| "12": 313.225015625, |
| "13": 334.0887734375, |
| "14": 361.206921875, |
| "15": 381.5723984375, |
| "16": 391.865984375, |
| "17": 413.0096484375, |
| "18": 431.9963671875, |
| "19": 435.01728125, |
| "20": 455.886546875 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 754568, |
| "answer_accuracy": 0.9318783591527031, |
| "avg_response_len": 282.8403031635315, |
| "resp_tokens_sum": 229023713, |
| "pass_at_k": { |
| "pass@1": 0.9318783591527031, |
| "pass@2": 0.9517385910166891, |
| "pass@4": 0.964830907829337, |
| "pass@8": 0.9741300581165051, |
| "pass@16": 0.9808451032894651, |
| "pass@32": 0.9856391594646161, |
| "pass@64": 0.9891717827349858, |
| "pass@128": 0.9916218779639583 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9496450958702065, |
| "pass@2": 0.9618148706245786, |
| "pass@4": 0.9709195504323019, |
| "pass@8": 0.9791190874933853, |
| "pass@16": 0.9868342218489051, |
| "pass@32": 0.993298633445245, |
| "pass@64": 0.9981549989323758, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997849770642202, |
| "pass@2": 0.9999864552481399, |
| "pass@4": 0.9999999638806617, |
| "pass@8": 0.9999999999999809, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999755859375, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9954117063492064, |
| "pass@2": 0.9981478877640296, |
| "pass@4": 0.9995556263800359, |
| "pass@8": 0.9999615639259287, |
| "pass@16": 0.9999997091342684, |
| "pass@32": 0.9999999999966475, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997427020958084, |
| "pass@2": 0.999995948064501, |
| "pass@4": 0.9999999985967324, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9956083579881657, |
| "pass@2": 0.9976401976657503, |
| "pass@4": 0.9985032811172272, |
| "pass@8": 0.9992715079029393, |
| "pass@16": 0.9998386699365346, |
| "pass@32": 0.9999946547625592, |
| "pass@64": 0.9999999995148912, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9960186298076923, |
| "pass@2": 0.9985326916767614, |
| "pass@4": 0.9996494007960544, |
| "pass@8": 0.9999753031040152, |
| "pass@16": 0.9999998899899415, |
| "pass@32": 0.9999999999996564, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9917033450704226, |
| "pass@2": 0.9929716091826549, |
| "pass@4": 0.9939165605179635, |
| "pass@8": 0.9949288254759624, |
| "pass@16": 0.9958752495136213, |
| "pass@32": 0.9970100988051338, |
| "pass@64": 0.9984237773095228, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9635047804532578, |
| "pass@2": 0.977436093328277, |
| "pass@4": 0.9865923250386901, |
| "pass@8": 0.9927103321039095, |
| "pass@16": 0.9970124444229089, |
| "pass@32": 0.9993847243796702, |
| "pass@64": 0.9999852421618102, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9450918079096046, |
| "pass@2": 0.9655720338983056, |
| "pass@4": 0.9781990243804268, |
| "pass@8": 0.9861440250122284, |
| "pass@16": 0.9923112374071664, |
| "pass@32": 0.9969355017981401, |
| "pass@64": 0.9992276493263522, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9179075235109718, |
| "pass@2": 0.9502324099671706, |
| "pass@4": 0.9729623692218725, |
| "pass@8": 0.9865352063542692, |
| "pass@16": 0.9929416594162586, |
| "pass@32": 0.9950239922483407, |
| "pass@64": 0.9960875780112477, |
| "pass@128": 0.9968652037617555 |
| }, |
| "14": { |
| "pass@1": 0.9201956775700935, |
| "pass@2": 0.9446224121225502, |
| "pass@4": 0.9626010729383125, |
| "pass@8": 0.9762203698107813, |
| "pass@16": 0.986550455857625, |
| "pass@32": 0.9938274615772782, |
| "pass@64": 0.9980046339203555, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9622696893063584, |
| "pass@2": 0.9790736226161763, |
| "pass@4": 0.9878219928975642, |
| "pass@8": 0.9938863424829786, |
| "pass@16": 0.9974854612936562, |
| "pass@32": 0.9990553826707531, |
| "pass@64": 0.9998278336678724, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8738155241935484, |
| "pass@2": 0.9108982092964184, |
| "pass@4": 0.9368475640141757, |
| "pass@8": 0.9526611840974472, |
| "pass@16": 0.9624771254488251, |
| "pass@32": 0.9706463781509711, |
| "pass@64": 0.97815086262173, |
| "pass@128": 0.9838709677419355 |
| }, |
| "17": { |
| "pass@1": 0.8999954841040463, |
| "pass@2": 0.9269427597742481, |
| "pass@4": 0.9442069857777889, |
| "pass@8": 0.9584880211486807, |
| "pass@16": 0.9699099356325508, |
| "pass@32": 0.977956592718777, |
| "pass@64": 0.9832164176739884, |
| "pass@128": 0.9884393063583815 |
| }, |
| "18": { |
| "pass@1": 0.8340625, |
| "pass@2": 0.875173298650169, |
| "pass@4": 0.9003785883014621, |
| "pass@8": 0.9191113837595276, |
| "pass@16": 0.9347451165816725, |
| "pass@32": 0.9484697556158473, |
| "pass@64": 0.9606376972907202, |
| "pass@128": 0.9685714285714285 |
| }, |
| "19": { |
| "pass@1": 0.7546218093093093, |
| "pass@2": 0.8168980988862881, |
| "pass@4": 0.8594574675913256, |
| "pass@8": 0.8903657440259747, |
| "pass@16": 0.9127998493831533, |
| "pass@32": 0.9301485120186141, |
| "pass@64": 0.9444138248825593, |
| "pass@128": 0.954954954954955 |
| }, |
| "20": { |
| "pass@1": 0.7120167042042042, |
| "pass@2": 0.7915602610484499, |
| "pass@4": 0.8442564102347557, |
| "pass@8": 0.8824961305644767, |
| "pass@16": 0.9097409929460565, |
| "pass@32": 0.927053074881089, |
| "pass@64": 0.9392787772648254, |
| "pass@128": 0.948948948948949 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 769502, |
| "answer_accuracy": 0.9371370810600156, |
| "avg_response_len": 292.1462916504287, |
| "resp_tokens_sum": 239887163, |
| "pass_at_k": { |
| "pass@1": 0.9371370810600156, |
| "pass@2": 0.9563403509552427, |
| "pass@4": 0.9692129536808676, |
| "pass@8": 0.9783119434671995, |
| "pass@16": 0.984564498023125, |
| "pass@32": 0.9886527260310264, |
| "pass@64": 0.9913362234234722, |
| "pass@128": 0.9934528448947778 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9329706671779141, |
| "pass@2": 0.9490174085792956, |
| "pass@4": 0.9641241520423445, |
| "pass@8": 0.976390759682789, |
| "pass@16": 0.9846418367918528, |
| "pass@32": 0.9893313200761069, |
| "pass@64": 0.9929169376529189, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995265151515151, |
| "pass@2": 0.9999742752326415, |
| "pass@4": 0.9999999304064264, |
| "pass@8": 0.9999999999999045, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9985207100591716, |
| "pass@2": 0.9996389134790105, |
| "pass@4": 0.9999800542683642, |
| "pass@8": 0.9999999564675484, |
| "pass@16": 0.999999999999962, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9973505434782609, |
| "pass@2": 0.9985510812507132, |
| "pass@4": 0.9993644259141521, |
| "pass@8": 0.9998696856114555, |
| "pass@16": 0.9999955044084027, |
| "pass@32": 0.9999999980554193, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9993508237822349, |
| "pass@2": 0.999962984793448, |
| "pass@4": 0.9999998469030911, |
| "pass@8": 0.9999999999990058, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9994806719653179, |
| "pass@2": 0.9999896881116016, |
| "pass@4": 0.9999999953943619, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.993155809859155, |
| "pass@2": 0.9950745813463459, |
| "pass@4": 0.9964881184746273, |
| "pass@8": 0.9976291893269761, |
| "pass@16": 0.9985375133295249, |
| "pass@32": 0.9993490121785924, |
| "pass@64": 0.9999188200065704, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9948774326347305, |
| "pass@2": 0.9968984275543402, |
| "pass@4": 0.9971880869868812, |
| "pass@8": 0.9973699249827601, |
| "pass@16": 0.9977102880852468, |
| "pass@32": 0.9983202885567448, |
| "pass@64": 0.9992573907303504, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9722797439759037, |
| "pass@2": 0.9806206864861019, |
| "pass@4": 0.987148461675724, |
| "pass@8": 0.992097089748973, |
| "pass@16": 0.9952356342084575, |
| "pass@32": 0.996694117094797, |
| "pass@64": 0.9969831820276938, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9480309561128527, |
| "pass@2": 0.9714501363759779, |
| "pass@4": 0.9865298393343465, |
| "pass@8": 0.9942286337372813, |
| "pass@16": 0.9978628049514008, |
| "pass@32": 0.9996281644114471, |
| "pass@64": 0.9999956190522251, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9407169117647058, |
| "pass@2": 0.9625553469419262, |
| "pass@4": 0.9774801687079033, |
| "pass@8": 0.9874219685475781, |
| "pass@16": 0.9934829131966962, |
| "pass@32": 0.9968338780653557, |
| "pass@64": 0.9985126025262858, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9497669956140351, |
| "pass@2": 0.9702634312980611, |
| "pass@4": 0.9843248648195289, |
| "pass@8": 0.9936793355130432, |
| "pass@16": 0.9984837564707297, |
| "pass@32": 0.999880078009628, |
| "pass@64": 0.999999559195243, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9554214015151515, |
| "pass@2": 0.9778524665950848, |
| "pass@4": 0.990172612230289, |
| "pass@8": 0.9965689152787764, |
| "pass@16": 0.9990389896488282, |
| "pass@32": 0.9998467083150427, |
| "pass@64": 0.9999979765261304, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8984820156695157, |
| "pass@2": 0.9263237628149055, |
| "pass@4": 0.9466511985147156, |
| "pass@8": 0.9628916229912796, |
| "pass@16": 0.9756373410907596, |
| "pass@32": 0.9843113854696494, |
| "pass@64": 0.9896611343512275, |
| "pass@128": 0.9943019943019943 |
| }, |
| "17": { |
| "pass@1": 0.8989955357142857, |
| "pass@2": 0.9258540372670809, |
| "pass@4": 0.9430381279009376, |
| "pass@8": 0.9567724437885808, |
| "pass@16": 0.9678264048731167, |
| "pass@32": 0.975789443612367, |
| "pass@64": 0.9804645618033802, |
| "pass@128": 0.984472049689441 |
| }, |
| "18": { |
| "pass@1": 0.8785386029411765, |
| "pass@2": 0.9263135421491432, |
| "pass@4": 0.9540938742032239, |
| "pass@8": 0.970755896754488, |
| "pass@16": 0.9808158406179858, |
| "pass@32": 0.9865832611536812, |
| "pass@64": 0.989914086501943, |
| "pass@128": 0.9911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.7576497395833334, |
| "pass@2": 0.8213040752718409, |
| "pass@4": 0.8629211197930617, |
| "pass@8": 0.8939620372022703, |
| "pass@16": 0.9169075579630253, |
| "pass@32": 0.9325509025737705, |
| "pass@64": 0.94412425958293, |
| "pass@128": 0.9553571428571429 |
| }, |
| "20": { |
| "pass@1": 0.6838978293413174, |
| "pass@2": 0.7640517439530387, |
| "pass@4": 0.8220046661457729, |
| "pass@8": 0.8655627671390692, |
| "pass@16": 0.8983598878087751, |
| "pass@32": 0.9234805889172719, |
| "pass@64": 0.9421120203018302, |
| "pass@128": 0.9550898203592815 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 746348, |
| "answer_accuracy": 0.9315935053522927, |
| "avg_response_len": 312.82036117990094, |
| "resp_tokens_sum": 250616658, |
| "pass_at_k": { |
| "pass@1": 0.9315935053522927, |
| "pass@2": 0.9525782282961234, |
| "pass@4": 0.966021727662297, |
| "pass@8": 0.975328068054194, |
| "pass@16": 0.9821933995365425, |
| "pass@32": 0.9873189930516977, |
| "pass@64": 0.9910883906699982, |
| "pass@128": 0.9936092027480428 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9567863805970149, |
| "pass@2": 0.971851495475379, |
| "pass@4": 0.9827990326768856, |
| "pass@8": 0.9908164062722622, |
| "pass@16": 0.9960200237300442, |
| "pass@32": 0.9982169750293176, |
| "pass@64": 0.9992594797136242, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9989720394736842, |
| "pass@2": 0.9996438561956071, |
| "pass@4": 0.9999591621770964, |
| "pass@8": 0.9999995591583335, |
| "pass@16": 0.9999999999797444, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99453125, |
| "pass@2": 0.9965507758221399, |
| "pass@4": 0.9972536858444163, |
| "pass@8": 0.9975790660148303, |
| "pass@16": 0.9980363206014791, |
| "pass@32": 0.9987689957872913, |
| "pass@64": 0.9996410375173692, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999014195583596, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9945361946202531, |
| "pass@2": 0.9958628712747932, |
| "pass@4": 0.9969610167399962, |
| "pass@8": 0.997929570922725, |
| "pass@16": 0.9987857390759178, |
| "pass@32": 0.9996009663236596, |
| "pass@64": 0.9999792027771768, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9935013138138138, |
| "pass@2": 0.9976191742924027, |
| "pass@4": 0.9995169683181494, |
| "pass@8": 0.9999713902369773, |
| "pass@16": 0.9999998669378484, |
| "pass@32": 0.9999999999994204, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9927150321543409, |
| "pass@2": 0.9953299459452619, |
| "pass@4": 0.9969358378032327, |
| "pass@8": 0.997933512235023, |
| "pass@16": 0.9987673566627887, |
| "pass@32": 0.9995945511170884, |
| "pass@64": 0.9999788684166812, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9630704365079366, |
| "pass@2": 0.9776848206474192, |
| "pass@4": 0.9883574401414106, |
| "pass@8": 0.9946295131968437, |
| "pass@16": 0.997513997070458, |
| "pass@32": 0.9989501368011887, |
| "pass@64": 0.9998107266450182, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9505208333333334, |
| "pass@2": 0.9706150370825212, |
| "pass@4": 0.9850789382405178, |
| "pass@8": 0.994133402138561, |
| "pass@16": 0.9983903509266632, |
| "pass@32": 0.9997473498922584, |
| "pass@64": 0.9999954976826797, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9122058256172839, |
| "pass@2": 0.9404311114756491, |
| "pass@4": 0.9606684896448127, |
| "pass@8": 0.9762644289391496, |
| "pass@16": 0.9877580918111676, |
| "pass@32": 0.9948566354295425, |
| "pass@64": 0.9986678916722411, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.955628709198813, |
| "pass@2": 0.9747088132900301, |
| "pass@4": 0.9841755971156422, |
| "pass@8": 0.9898190798079135, |
| "pass@16": 0.9938872827202818, |
| "pass@32": 0.9970027263205429, |
| "pass@64": 0.99907716563192, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9384645061728395, |
| "pass@2": 0.9601393141829493, |
| "pass@4": 0.9720348755595365, |
| "pass@8": 0.981293627330329, |
| "pass@16": 0.9892875060551926, |
| "pass@32": 0.9937818556587116, |
| "pass@64": 0.995354403432346, |
| "pass@128": 0.9969135802469136 |
| }, |
| "16": { |
| "pass@1": 0.9030927359882006, |
| "pass@2": 0.9313183024411773, |
| "pass@4": 0.949399185168225, |
| "pass@8": 0.9638089071611805, |
| "pass@16": 0.9753050149460608, |
| "pass@32": 0.9843716916440536, |
| "pass@64": 0.9918881750232778, |
| "pass@128": 0.9970501474926253 |
| }, |
| "17": { |
| "pass@1": 0.8378200301204819, |
| "pass@2": 0.8722340147993546, |
| "pass@4": 0.8994627833606642, |
| "pass@8": 0.921389194949213, |
| "pass@16": 0.9387280323416873, |
| "pass@32": 0.953114829081259, |
| "pass@64": 0.9652418490909267, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8834929435483871, |
| "pass@2": 0.9286580041910086, |
| "pass@4": 0.9520407616386656, |
| "pass@8": 0.966049497344724, |
| "pass@16": 0.9765995183921604, |
| "pass@32": 0.9846246388700052, |
| "pass@64": 0.989084716853413, |
| "pass@128": 0.9903225806451613 |
| }, |
| "19": { |
| "pass@1": 0.7401812688821753, |
| "pass@2": 0.8118499030615883, |
| "pass@4": 0.8589915267389155, |
| "pass@8": 0.8922240120160714, |
| "pass@16": 0.9169443378799137, |
| "pass@32": 0.9360698853549126, |
| "pass@64": 0.9515090838923458, |
| "pass@128": 0.9607250755287009 |
| }, |
| "20": { |
| "pass@1": 0.6837227852852853, |
| "pass@2": 0.7745815451278046, |
| "pass@4": 0.8314440877885758, |
| "pass@8": 0.8682970775356633, |
| "pass@16": 0.8966446199160057, |
| "pass@32": 0.92130107609107, |
| "pass@64": 0.9418564963876911, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| } |
| } |
