| { |
| "checkpoint": "checkpoint-38796", |
| "id": { |
| "answer_accuracy": 0.9899713541666667, |
| "avg_response_len": 200.3366076388889, |
| "avg_loss": 0.0, |
| "count": 1152000, |
| "validation_metrics": {}, |
| "pass_at_k": { |
| "pass@1": 0.9899713541666667, |
| "pass@2": 0.9929163932633402, |
| "pass@4": 0.9950443724430285, |
| "pass@8": 0.9967556474644427, |
| "pass@16": 0.9981256002928118, |
| "pass@32": 0.9991009218100472, |
| "pass@64": 0.9996657523498159, |
| "pass@128": 0.9998888888888889 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.941265625, |
| "pass@2": 0.9563416584645673, |
| "pass@4": 0.9692801073303342, |
| "pass@8": 0.9799107461407804, |
| "pass@16": 0.9882794827881477, |
| "pass@32": 0.9940941877802166, |
| "pass@64": 0.9975055528695351, |
| "pass@128": 0.999 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.998203125, |
| "pass@2": 0.9989484498031497, |
| "pass@4": 0.999429442632171, |
| "pass@8": 0.9997332787706423, |
| "pass@16": 0.9999360887968483, |
| "pass@32": 0.9999974424774314, |
| "pass@64": 0.9999999996028359, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99871875, |
| "pass@2": 0.9995188238188978, |
| "pass@4": 0.9998979254780653, |
| "pass@8": 0.9999947100672492, |
| "pass@16": 0.9999999877346716, |
| "pass@32": 0.9999999999999909, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9939296875, |
| "pass@2": 0.9966125738188978, |
| "pass@4": 0.9980609976565431, |
| "pass@8": 0.9988772917617279, |
| "pass@16": 0.9992186179280644, |
| "pass@32": 0.9994389261114691, |
| "pass@64": 0.9997519685039321, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9972734375, |
| "pass@2": 0.9982062007874017, |
| "pass@4": 0.9987222359392577, |
| "pass@8": 0.9992138876134402, |
| "pass@16": 0.9996969605662573, |
| "pass@32": 0.9999586920114116, |
| "pass@64": 0.9999996921831551, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9960859375, |
| "pass@2": 0.9974864665354329, |
| "pass@4": 0.9981825993625796, |
| "pass@8": 0.9987325731094482, |
| "pass@16": 0.9992691028653226, |
| "pass@32": 0.9997610564276355, |
| "pass@64": 0.9999847315644707, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.991609375, |
| "pass@2": 0.994200910433071, |
| "pass@4": 0.9958966556055493, |
| "pass@8": 0.9973900285847406, |
| "pass@16": 0.9986377579744303, |
| "pass@32": 0.9995155655514678, |
| "pass@64": 0.9999335572652138, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.99265625, |
| "pass@2": 0.9949324557086615, |
| "pass@4": 0.9959293879827522, |
| "pass@8": 0.9969483111319597, |
| "pass@16": 0.9980924039815611, |
| "pass@32": 0.9991424259308008, |
| "pass@64": 0.9998162691591999, |
| "pass@128": 1.0 |
| } |
| }, |
| "per_op_accuracy": { |
| "10": 0.941265625, |
| "2": 1.0, |
| "3": 0.998203125, |
| "4": 0.99871875, |
| "5": 0.9939296875, |
| "6": 0.9972734375, |
| "7": 0.9960859375, |
| "8": 0.991609375, |
| "9": 0.99265625 |
| }, |
| "per_op_avg_response_len": { |
| "10": 281.453609375, |
| "2": 115.61171875, |
| "3": 145.2217734375, |
| "4": 160.020953125, |
| "5": 173.8887578125, |
| "6": 199.223328125, |
| "7": 224.244828125, |
| "8": 243.6493359375, |
| "9": 259.7151640625 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "10": 281.453609375, |
| "2": 115.61171875, |
| "3": 145.2217734375, |
| "4": 160.020953125, |
| "5": 173.8887578125, |
| "6": 199.223328125, |
| "7": 224.244828125, |
| "8": 243.6493359375, |
| "9": 259.7151640625 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 379008, |
| "correct": 375204, |
| "answer_accuracy": 0.9899632725430598, |
| "avg_response_len": 190.6923653326579, |
| "resp_tokens_sum": 72273932, |
| "pass_at_k": { |
| "pass@1": 0.9899632725430598, |
| "pass@2": 0.9924113957164932, |
| "pass@4": 0.994310804899388, |
| "pass@8": 0.9960184386538377, |
| "pass@16": 0.9974505520342333, |
| "pass@32": 0.9985272006809436, |
| "pass@64": 0.9992532853259626, |
| "pass@128": 0.999662276258021 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.940058075221239, |
| "pass@2": 0.952202445241911, |
| "pass@4": 0.9639851915634436, |
| "pass@8": 0.9747452943211968, |
| "pass@16": 0.9837117746740343, |
| "pass@32": 0.9903082598333327, |
| "pass@64": 0.9945723951890973, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999641628440367, |
| "pass@2": 0.9999793066291025, |
| "pass@4": 0.999999905401733, |
| "pass@8": 0.9999999999996472, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9995361328125, |
| "pass@2": 0.9999784694881889, |
| "pass@4": 0.9999999370195912, |
| "pass@8": 0.9999999999999016, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.991170634920635, |
| "pass@2": 0.9935695538057743, |
| "pass@4": 0.9953833833270841, |
| "pass@8": 0.996783521967192, |
| "pass@16": 0.9975425302123753, |
| "pass@32": 0.9982188768739435, |
| "pass@64": 0.9992125984251853, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999298278443114, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9981046597633136, |
| "pass@2": 0.9996745853328983, |
| "pass@4": 0.9999718887764769, |
| "pass@8": 0.9999998022755866, |
| "pass@16": 0.9999999999967328, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9924879807692307, |
| "pass@2": 0.995214390268524, |
| "pass@4": 0.9969837258924363, |
| "pass@8": 0.9983251613066434, |
| "pass@16": 0.9993038302603201, |
| "pass@32": 0.9998838431738969, |
| "pass@64": 0.9999990205966608, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9905809859154929, |
| "pass@2": 0.992608059775979, |
| "pass@4": 0.9937131769620348, |
| "pass@8": 0.9952330962332059, |
| "pass@16": 0.9970819743714158, |
| "pass@32": 0.9986530587245823, |
| "pass@64": 0.9996542900027615, |
| "pass@128": 1.0 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 390656, |
| "correct": 386226, |
| "answer_accuracy": 0.9886600999344692, |
| "avg_response_len": 198.37706319626474, |
| "resp_tokens_sum": 77497190, |
| "pass_at_k": { |
| "pass@1": 0.9886600999344692, |
| "pass@2": 0.9917850676850604, |
| "pass@4": 0.9941407065986837, |
| "pass@8": 0.9961116676864055, |
| "pass@16": 0.9978235394203914, |
| "pass@32": 0.9991057439094873, |
| "pass@64": 0.999825416636273, |
| "pass@128": 1.0 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9300469708588958, |
| "pass@2": 0.9466099916670692, |
| "pass@4": 0.9608256378803874, |
| "pass@8": 0.9735479075977559, |
| "pass@16": 0.9852696934862603, |
| "pass@32": 0.9940972426092773, |
| "pass@64": 0.9987733649475956, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9961647727272728, |
| "pass@2": 0.9972918157957528, |
| "pass@4": 0.9983365499767074, |
| "pass@8": 0.9991928917832018, |
| "pass@16": 0.9998063298472625, |
| "pass@32": 0.9999922499316101, |
| "pass@64": 0.9999999987964725, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.997965976331361, |
| "pass@2": 0.9991187596095608, |
| "pass@4": 0.9997857705841503, |
| "pass@8": 0.9999865024173374, |
| "pass@16": 0.9999999643806659, |
| "pass@32": 0.9999999999999731, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99375, |
| "pass@2": 0.9967833504507589, |
| "pass@4": 0.9986635988436225, |
| "pass@8": 0.9996831882834758, |
| "pass@16": 0.9999789012150662, |
| "pass@32": 0.9999999417280491, |
| "pass@64": 0.9999999999999963, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9955676934097422, |
| "pass@2": 0.9973236243034089, |
| "pass@4": 0.9983310997300121, |
| "pass@8": 0.9991142096348365, |
| "pass@16": 0.9997496114237934, |
| "pass@32": 0.9999854792844564, |
| "pass@64": 0.999999993573057, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9948067196531792, |
| "pass@2": 0.9959908089071959, |
| "pass@4": 0.9969885587820309, |
| "pass@8": 0.9979235613025731, |
| "pass@16": 0.9987206336864413, |
| "pass@32": 0.9995059858135572, |
| "pass@64": 0.9999600471131015, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9922975352112676, |
| "pass@2": 0.9943263419097261, |
| "pass@4": 0.99571091852955, |
| "pass@8": 0.9969972148105722, |
| "pass@16": 0.99814919003857, |
| "pass@32": 0.9991126715814442, |
| "pass@64": 0.9998322253384277, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.995158121257485, |
| "pass@2": 0.9969157403696542, |
| "pass@4": 0.997358424601865, |
| "pass@8": 0.9976945425380946, |
| "pass@16": 0.9982569113741001, |
| "pass@32": 0.9990676801627342, |
| "pass@64": 0.9998216794756942, |
| "pass@128": 1.0 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 382336, |
| "correct": 379017, |
| "answer_accuracy": 0.9913191538332775, |
| "avg_response_len": 211.89908875962504, |
| "resp_tokens_sum": 81016650, |
| "pass_at_k": { |
| "pass@1": 0.9913191538332775, |
| "pass@2": 0.9945729394304457, |
| "pass@4": 0.9966948852162644, |
| "pass@8": 0.9981444327241598, |
| "pass@16": 0.9991034066792451, |
| "pass@32": 0.9996647220162015, |
| "pass@64": 0.9999114903663415, |
| "pass@128": 1.0 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9534048507462687, |
| "pass@2": 0.970000514161476, |
| "pass@4": 0.9828655804218498, |
| "pass@8": 0.9913297716090343, |
| "pass@16": 0.9958306599914305, |
| "pass@32": 0.9979223480778879, |
| "pass@64": 0.9992399818731803, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9987928206997084, |
| "pass@2": 0.99955952572255, |
| "pass@4": 0.9999370614387487, |
| "pass@8": 0.9999989051962135, |
| "pass@16": 0.9999999998462147, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986979166666666, |
| "pass@2": 0.9994841310954552, |
| "pass@4": 0.9999133192232549, |
| "pass@8": 0.9999978720766682, |
| "pass@16": 0.9999999993392, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9966681985294118, |
| "pass@2": 0.9992585543075498, |
| "pass@4": 0.999930265610181, |
| "pass@8": 0.9999993599537156, |
| "pass@16": 0.9999999999763772, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.996352523659306, |
| "pass@2": 0.9972878735189645, |
| "pass@4": 0.9978065682444268, |
| "pass@8": 0.9984953578892186, |
| "pass@16": 0.9993197040358152, |
| "pass@32": 0.9998856774168334, |
| "pass@64": 0.999999036044663, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9953273338607594, |
| "pass@2": 0.9967836607943786, |
| "pass@4": 0.9975761380302146, |
| "pass@8": 0.998262935853195, |
| "pass@16": 0.9990878595282221, |
| "pass@32": 0.9997847637219769, |
| "pass@64": 0.9999954280485364, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9900525525525525, |
| "pass@2": 0.9931176254995152, |
| "pass@4": 0.9950761472946511, |
| "pass@8": 0.996932633631575, |
| "pass@16": 0.9985345359444987, |
| "pass@32": 0.9996000242336316, |
| "pass@64": 0.9999802487805222, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9923382234726688, |
| "pass@2": 0.9954557466389852, |
| "pass@4": 0.9969244255440737, |
| "pass@8": 0.9981047741525653, |
| "pass@16": 0.9990691147291293, |
| "pass@32": 0.9997813019589095, |
| "pass@64": 0.9999953545444935, |
| "pass@128": 1.0 |
| } |
| } |
| } |
| } |
| }, |
| "ood": { |
| "answer_accuracy": 0.8540453125, |
| "avg_response_len": 381.87561171875, |
| "avg_loss": 0.0, |
| "count": 1280000, |
| "validation_metrics": {}, |
| "pass_at_k": { |
| "pass@1": 0.8540453125, |
| "pass@2": 0.9031663385826517, |
| "pass@4": 0.9325282339801222, |
| "pass@8": 0.9512891821432544, |
| "pass@16": 0.9638628298980004, |
| "pass@32": 0.9727622294964876, |
| "pass@64": 0.97954231019947, |
| "pass@128": 0.9848 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9624921875, |
| "pass@2": 0.9759582923228339, |
| "pass@4": 0.9853311848518935, |
| "pass@8": 0.9919277954226844, |
| "pass@16": 0.9960295972437585, |
| "pass@32": 0.9978717406808197, |
| "pass@64": 0.9984840527828055, |
| "pass@128": 0.999 |
| }, |
| "12": { |
| "pass@1": 0.951375, |
| "pass@2": 0.9708375984251973, |
| "pass@4": 0.9831443614548185, |
| "pass@8": 0.9908391481052844, |
| "pass@16": 0.9954637287723308, |
| "pass@32": 0.9979827905163924, |
| "pass@64": 0.9992459838187541, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.927671875, |
| "pass@2": 0.9560797244094491, |
| "pass@4": 0.9735508835770535, |
| "pass@8": 0.9845974255568427, |
| "pass@16": 0.9910754289987089, |
| "pass@32": 0.9948196851214032, |
| "pass@64": 0.9974767526798574, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9342578125, |
| "pass@2": 0.9600164862204726, |
| "pass@4": 0.9764483093363321, |
| "pass@8": 0.9877587695211665, |
| "pass@16": 0.9944378788673935, |
| "pass@32": 0.9976130640940937, |
| "pass@64": 0.9991646203883312, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.930546875, |
| "pass@2": 0.9632146899606311, |
| "pass@4": 0.979843436070491, |
| "pass@8": 0.9879005123945149, |
| "pass@16": 0.9915401089295047, |
| "pass@32": 0.9936956420221912, |
| "pass@64": 0.9951344064115004, |
| "pass@128": 0.996 |
| }, |
| "16": { |
| "pass@1": 0.866078125, |
| "pass@2": 0.9150172244094495, |
| "pass@4": 0.9411895326209221, |
| "pass@8": 0.9579022839054648, |
| "pass@16": 0.969572659701787, |
| "pass@32": 0.9778052014424473, |
| "pass@64": 0.9839818197350004, |
| "pass@128": 0.989 |
| }, |
| "17": { |
| "pass@1": 0.8524375, |
| "pass@2": 0.9029194143700803, |
| "pass@4": 0.9286221111736035, |
| "pass@8": 0.9443386208689557, |
| "pass@16": 0.955734081883812, |
| "pass@32": 0.9648144384726397, |
| "pass@64": 0.9726182428749347, |
| "pass@128": 0.98 |
| }, |
| "18": { |
| "pass@1": 0.8109296875, |
| "pass@2": 0.8804515255905518, |
| "pass@4": 0.9190456437945261, |
| "pass@8": 0.9411653142560411, |
| "pass@16": 0.954709171238404, |
| "pass@32": 0.9636073849642399, |
| "pass@64": 0.9701930633795666, |
| "pass@128": 0.975 |
| }, |
| "19": { |
| "pass@1": 0.6708515625, |
| "pass@2": 0.7712925688976383, |
| "pass@4": 0.8335043385826767, |
| "pass@8": 0.8737365678515328, |
| "pass@16": 0.9019963268627569, |
| "pass@32": 0.9241395020391616, |
| "pass@64": 0.9433327950305856, |
| "pass@128": 0.96 |
| }, |
| "20": { |
| "pass@1": 0.6338125, |
| "pass@2": 0.7358758612204728, |
| "pass@4": 0.8046025383389569, |
| "pass@8": 0.8527253835500371, |
| "pass@16": 0.8880693164814899, |
| "pass@32": 0.915272845611458, |
| "pass@64": 0.9357913648933431, |
| "pass@128": 0.949 |
| } |
| }, |
| "per_op_accuracy": { |
| "11": 0.9624921875, |
| "12": 0.951375, |
| "13": 0.927671875, |
| "14": 0.9342578125, |
| "15": 0.930546875, |
| "16": 0.866078125, |
| "17": 0.8524375, |
| "18": 0.8109296875, |
| "19": 0.6708515625, |
| "20": 0.6338125 |
| }, |
| "per_op_avg_response_len": { |
| "11": 300.8788359375, |
| "12": 313.412328125, |
| "13": 334.19246875, |
| "14": 361.4050703125, |
| "15": 381.8040859375, |
| "16": 392.90459375, |
| "17": 414.05021875, |
| "18": 432.1644609375, |
| "19": 434.3923984375, |
| "20": 453.55165625 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "11": 300.8788359375, |
| "12": 313.412328125, |
| "13": 334.19246875, |
| "14": 361.4050703125, |
| "15": 381.8040859375, |
| "16": 392.90459375, |
| "17": 414.05021875, |
| "18": 432.1644609375, |
| "19": 434.3923984375, |
| "20": 453.55165625 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 430720, |
| "correct": 366800, |
| "answer_accuracy": 0.8515973254086181, |
| "avg_response_len": 363.4109096396731, |
| "resp_tokens_sum": 156528347, |
| "pass_at_k": { |
| "pass@1": 0.8515973254086181, |
| "pass@2": 0.9000831422353838, |
| "pass@4": 0.9288547789252976, |
| "pass@8": 0.9476257278411426, |
| "pass@16": 0.961010872591785, |
| "pass@32": 0.9707065629086032, |
| "pass@64": 0.9777220682162787, |
| "pass@128": 0.9830609212481426 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9567767351274787, |
| "pass@2": 0.9729954579420491, |
| "pass@4": 0.9837077224412106, |
| "pass@8": 0.9915053244533025, |
| "pass@16": 0.9967604348473688, |
| "pass@32": 0.9993037377604537, |
| "pass@64": 0.999959962274875, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9463497528248588, |
| "pass@2": 0.9650545370123225, |
| "pass@4": 0.9766829093397222, |
| "pass@8": 0.9848453467690281, |
| "pass@16": 0.9914138441913528, |
| "pass@32": 0.9958525342115812, |
| "pass@64": 0.9982239261012225, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9212627351097179, |
| "pass@2": 0.953259601856194, |
| "pass@4": 0.974031526035734, |
| "pass@8": 0.9872172251386195, |
| "pass@16": 0.9946975877158775, |
| "pass@32": 0.9979212628228779, |
| "pass@64": 0.9992200434190286, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9226781542056075, |
| "pass@2": 0.9488315457355219, |
| "pass@4": 0.966660107907072, |
| "pass@8": 0.9807327520370567, |
| "pass@16": 0.9900033030501925, |
| "pass@32": 0.994705328348, |
| "pass@64": 0.997677205060592, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9338872832369942, |
| "pass@2": 0.9658665826544082, |
| "pass@4": 0.9800264382595235, |
| "pass@8": 0.9852710411507393, |
| "pass@16": 0.9867086223947849, |
| "pass@32": 0.9875391269726735, |
| "pass@64": 0.9882671703608444, |
| "pass@128": 0.9884393063583815 |
| }, |
| "16": { |
| "pass@1": 0.8443296370967742, |
| "pass@2": 0.8949314992379985, |
| "pass@4": 0.9236545057875828, |
| "pass@8": 0.9436194889710628, |
| "pass@16": 0.9597137566119098, |
| "pass@32": 0.9731177144426382, |
| "pass@64": 0.9836290640887158, |
| "pass@128": 0.9935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.8775063222543352, |
| "pass@2": 0.9258678343043105, |
| "pass@4": 0.9494584514544068, |
| "pass@8": 0.9630582535369274, |
| "pass@16": 0.9726016060789089, |
| "pass@32": 0.9786961371782323, |
| "pass@64": 0.9822631761264358, |
| "pass@128": 0.9855491329479769 |
| }, |
| "18": { |
| "pass@1": 0.8074330357142857, |
| "pass@2": 0.8726863048368955, |
| "pass@4": 0.9079920633135147, |
| "pass@8": 0.9290310338171512, |
| "pass@16": 0.9431022516433624, |
| "pass@32": 0.9531360758782476, |
| "pass@64": 0.9606210100169015, |
| "pass@128": 0.9657142857142857 |
| }, |
| "19": { |
| "pass@1": 0.6631240615615616, |
| "pass@2": 0.7647959228913951, |
| "pass@4": 0.8269798164306035, |
| "pass@8": 0.8654212985928951, |
| "pass@16": 0.8938579593227545, |
| "pass@32": 0.9182400396910518, |
| "pass@64": 0.9393446013378842, |
| "pass@128": 0.9579579579579579 |
| }, |
| "20": { |
| "pass@1": 0.6333521021021021, |
| "pass@2": 0.7295261550920993, |
| "pass@4": 0.7942091112484806, |
| "pass@8": 0.8420309365275893, |
| "pass@16": 0.8790149220263148, |
| "pass@32": 0.9073538190959931, |
| "pass@64": 0.9277009935323411, |
| "pass@128": 0.93993993993994 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 430464, |
| "correct": 371427, |
| "answer_accuracy": 0.8628526427297056, |
| "avg_response_len": 377.68319534269995, |
| "resp_tokens_sum": 162579019, |
| "pass_at_k": { |
| "pass@1": 0.8628526427297056, |
| "pass@2": 0.9105499562749843, |
| "pass@4": 0.9382578365260149, |
| "pass@8": 0.9553304746020284, |
| "pass@16": 0.9664182673619955, |
| "pass@32": 0.974095193480023, |
| "pass@64": 0.9799923229829369, |
| "pass@128": 0.9845376152245019 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.97265625, |
| "pass@2": 0.9817142586092406, |
| "pass@4": 0.9872070927128084, |
| "pass@8": 0.9907918466392928, |
| "pass@16": 0.9932876981463241, |
| "pass@32": 0.9946033623345881, |
| "pass@64": 0.9954810039531555, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9539086990595611, |
| "pass@2": 0.9742443783476912, |
| "pass@4": 0.9873180110910899, |
| "pass@8": 0.9939946254019711, |
| "pass@16": 0.9967163926023948, |
| "pass@32": 0.9983956273931865, |
| "pass@64": 0.9996077018477316, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9435836834733894, |
| "pass@2": 0.9670878686120119, |
| "pass@4": 0.9795068925207874, |
| "pass@8": 0.9867661154939795, |
| "pass@16": 0.9912037146425846, |
| "pass@32": 0.9943074150090554, |
| "pass@64": 0.9971910964276763, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9339820906432749, |
| "pass@2": 0.961808433485288, |
| "pass@4": 0.9791213154276766, |
| "pass@8": 0.9903633423119838, |
| "pass@16": 0.9965715597469695, |
| "pass@32": 0.998978060543375, |
| "pass@64": 0.9998257129593244, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.936032196969697, |
| "pass@2": 0.9698241022429014, |
| "pass@4": 0.986383716808126, |
| "pass@8": 0.993887020431712, |
| "pass@16": 0.997341288437486, |
| "pass@32": 0.9992550901802849, |
| "pass@64": 0.999953724719699, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8875979344729344, |
| "pass@2": 0.9302362344931245, |
| "pass@4": 0.9518918415112638, |
| "pass@8": 0.9653093003879017, |
| "pass@16": 0.97478416069728, |
| "pass@32": 0.9809651521660178, |
| "pass@64": 0.986492391819359, |
| "pass@128": 0.9914529914529915 |
| }, |
| "17": { |
| "pass@1": 0.8626746894409938, |
| "pass@2": 0.9124707322590117, |
| "pass@4": 0.9368205890024611, |
| "pass@8": 0.9508498521267209, |
| "pass@16": 0.960456726487503, |
| "pass@32": 0.9678057412486193, |
| "pass@64": 0.9734389189717876, |
| "pass@128": 0.9782608695652174 |
| }, |
| "18": { |
| "pass@1": 0.8121553308823529, |
| "pass@2": 0.8809753502779064, |
| "pass@4": 0.9216372758552233, |
| "pass@8": 0.9462221099752758, |
| "pass@16": 0.9606949776103674, |
| "pass@32": 0.9685754572068952, |
| "pass@64": 0.9731564892213497, |
| "pass@128": 0.9764705882352941 |
| }, |
| "19": { |
| "pass@1": 0.66943359375, |
| "pass@2": 0.7666238985751784, |
| "pass@4": 0.8258763982627161, |
| "pass@8": 0.8655574931646675, |
| "pass@16": 0.8941625232220163, |
| "pass@32": 0.9161714667742615, |
| "pass@64": 0.9350029611318369, |
| "pass@128": 0.9523809523809523 |
| }, |
| "20": { |
| "pass@1": 0.6556652320359282, |
| "pass@2": 0.7598498868404915, |
| "pass@4": 0.8262946623625637, |
| "pass@8": 0.8690020135890967, |
| "pass@16": 0.8983220264807161, |
| "pass@32": 0.9212925306509724, |
| "pass@64": 0.9391524696479382, |
| "pass@128": 0.9491017964071856 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 418816, |
| "correct": 354951, |
| "answer_accuracy": 0.8475106013141809, |
| "avg_response_len": 405.1741504622555, |
| "resp_tokens_sum": 169693417, |
| "pass_at_k": { |
| "pass@1": 0.8475106013141809, |
| "pass@2": 0.8987481996491417, |
| "pass@4": 0.9304171468461809, |
| "pass@8": 0.9509030749267878, |
| "pass@16": 0.9641693397219542, |
| "pass@32": 0.9735062882347418, |
| "pass@64": 0.9809517604692125, |
| "pass@128": 0.9868581907090465 |
| }, |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.9581845238095238, |
| "pass@2": 0.973211942257218, |
| "pass@4": 0.9851733461888689, |
| "pass@8": 0.9935984850997558, |
| "pass@16": 0.9981004697081209, |
| "pass@32": 0.9997117617660194, |
| "pass@64": 0.9999952151978636, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9543434633027523, |
| "pass@2": 0.9737747266970067, |
| "pass@4": 0.9860677859648275, |
| "pass@8": 0.994249571393947, |
| "pass@16": 0.9986259898728691, |
| "pass@32": 0.9998862026515789, |
| "pass@64": 0.9999995629648154, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9164496527777778, |
| "pass@2": 0.9467269824778841, |
| "pass@4": 0.966515018955963, |
| "pass@8": 0.9796284799576312, |
| "pass@16": 0.9873678160183381, |
| "pass@32": 0.9923304170452849, |
| "pass@64": 0.9960751154460078, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9455675074183977, |
| "pass@2": 0.9688518569359099, |
| "pass@4": 0.9830591241005852, |
| "pass@8": 0.9918079912361211, |
| "pass@16": 0.9964965731596999, |
| "pass@32": 0.9989974984823478, |
| "pass@64": 0.999910515524606, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9213927469135802, |
| "pass@2": 0.9536509216729855, |
| "pass@4": 0.9729866107593029, |
| "pass@8": 0.9846111586848604, |
| "pass@16": 0.9907910506066029, |
| "pass@32": 0.9946077911424446, |
| "pass@64": 0.9975593713245301, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8636845501474927, |
| "pass@2": 0.9176269656005392, |
| "pass@4": 0.9461433612280757, |
| "pass@8": 0.9632940350686785, |
| "pass@16": 0.9731921968948374, |
| "pass@32": 0.9788198865927932, |
| "pass@64": 0.9817049567519268, |
| "pass@128": 0.9823008849557522 |
| }, |
| "17": { |
| "pass@1": 0.816382718373494, |
| "pass@2": 0.8697396623897163, |
| "pass@4": 0.8989555943120052, |
| "pass@8": 0.918514496266187, |
| "pass@16": 0.9335748802757035, |
| "pass@32": 0.9474461636352892, |
| "pass@64": 0.9617706386333493, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8135332661290322, |
| "pass@2": 0.8886441929133857, |
| "pass@4": 0.9286830575613528, |
| "pass@8": 0.9493191452530526, |
| "pass@16": 0.9612486799216202, |
| "pass@32": 0.9699809772790603, |
| "pass@64": 0.9777500114141688, |
| "pass@128": 0.9838709677419355 |
| }, |
| "19": { |
| "pass@1": 0.6800651435045317, |
| "pass@2": 0.7825676636058709, |
| "pass@4": 0.8478114498338757, |
| "pass@8": 0.890404706092961, |
| "pass@16": 0.9181360078721577, |
| "pass@32": 0.9381630090208142, |
| "pass@64": 0.9558007487757464, |
| "pass@128": 0.9697885196374623 |
| }, |
| "20": { |
| "pass@1": 0.6123545420420421, |
| "pass@2": 0.7181795476578936, |
| "pass@4": 0.7932386998922434, |
| "pass@8": 0.8470943217645397, |
| "pass@16": 0.8868402120185244, |
| "pass@32": 0.9171541099551569, |
| "pass@64": 0.9405105380920781, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| }, |
| "total": { |
| "answer_accuracy": 0.9184313322368421, |
| "avg_loss": 0.0, |
| "count": 2432000, |
| "validation_metrics": { |
| "id": {}, |
| "ood": {} |
| }, |
| "pass_at_k": { |
| "pass@1": 0.9184313322368421, |
| "pass@2": 0.9456795223787686, |
| "pass@4": 0.9621411416730752, |
| "pass@8": 0.9728259288743495, |
| "pass@16": 0.9800925632428983, |
| "pass@32": 0.9852384521713364, |
| "pass@64": 0.989074467007528, |
| "pass@128": 0.9919473684210526 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.941265625, |
| "pass@2": 0.9563416584645673, |
| "pass@4": 0.9692801073303342, |
| "pass@8": 0.9799107461407804, |
| "pass@16": 0.9882794827881477, |
| "pass@32": 0.9940941877802166, |
| "pass@64": 0.9975055528695351, |
| "pass@128": 0.999 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.998203125, |
| "pass@2": 0.9989484498031497, |
| "pass@4": 0.999429442632171, |
| "pass@8": 0.9997332787706423, |
| "pass@16": 0.9999360887968483, |
| "pass@32": 0.9999974424774314, |
| "pass@64": 0.9999999996028359, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99871875, |
| "pass@2": 0.9995188238188978, |
| "pass@4": 0.9998979254780653, |
| "pass@8": 0.9999947100672492, |
| "pass@16": 0.9999999877346716, |
| "pass@32": 0.9999999999999909, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9939296875, |
| "pass@2": 0.9966125738188978, |
| "pass@4": 0.9980609976565431, |
| "pass@8": 0.9988772917617279, |
| "pass@16": 0.9992186179280644, |
| "pass@32": 0.9994389261114691, |
| "pass@64": 0.9997519685039321, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9972734375, |
| "pass@2": 0.9982062007874017, |
| "pass@4": 0.9987222359392577, |
| "pass@8": 0.9992138876134402, |
| "pass@16": 0.9996969605662573, |
| "pass@32": 0.9999586920114116, |
| "pass@64": 0.9999996921831551, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9960859375, |
| "pass@2": 0.9974864665354329, |
| "pass@4": 0.9981825993625796, |
| "pass@8": 0.9987325731094482, |
| "pass@16": 0.9992691028653226, |
| "pass@32": 0.9997610564276355, |
| "pass@64": 0.9999847315644707, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.991609375, |
| "pass@2": 0.994200910433071, |
| "pass@4": 0.9958966556055493, |
| "pass@8": 0.9973900285847406, |
| "pass@16": 0.9986377579744303, |
| "pass@32": 0.9995155655514678, |
| "pass@64": 0.9999335572652138, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.99265625, |
| "pass@2": 0.9949324557086615, |
| "pass@4": 0.9959293879827522, |
| "pass@8": 0.9969483111319597, |
| "pass@16": 0.9980924039815611, |
| "pass@32": 0.9991424259308008, |
| "pass@64": 0.9998162691591999, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9624921875, |
| "pass@2": 0.9759582923228339, |
| "pass@4": 0.9853311848518935, |
| "pass@8": 0.9919277954226844, |
| "pass@16": 0.9960295972437585, |
| "pass@32": 0.9978717406808197, |
| "pass@64": 0.9984840527828055, |
| "pass@128": 0.999 |
| }, |
| "12": { |
| "pass@1": 0.951375, |
| "pass@2": 0.9708375984251973, |
| "pass@4": 0.9831443614548185, |
| "pass@8": 0.9908391481052844, |
| "pass@16": 0.9954637287723308, |
| "pass@32": 0.9979827905163924, |
| "pass@64": 0.9992459838187541, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.927671875, |
| "pass@2": 0.9560797244094491, |
| "pass@4": 0.9735508835770535, |
| "pass@8": 0.9845974255568427, |
| "pass@16": 0.9910754289987089, |
| "pass@32": 0.9948196851214032, |
| "pass@64": 0.9974767526798574, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9342578125, |
| "pass@2": 0.9600164862204726, |
| "pass@4": 0.9764483093363321, |
| "pass@8": 0.9877587695211665, |
| "pass@16": 0.9944378788673935, |
| "pass@32": 0.9976130640940937, |
| "pass@64": 0.9991646203883312, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.930546875, |
| "pass@2": 0.9632146899606311, |
| "pass@4": 0.979843436070491, |
| "pass@8": 0.9879005123945149, |
| "pass@16": 0.9915401089295047, |
| "pass@32": 0.9936956420221912, |
| "pass@64": 0.9951344064115004, |
| "pass@128": 0.996 |
| }, |
| "16": { |
| "pass@1": 0.866078125, |
| "pass@2": 0.9150172244094495, |
| "pass@4": 0.9411895326209221, |
| "pass@8": 0.9579022839054648, |
| "pass@16": 0.969572659701787, |
| "pass@32": 0.9778052014424473, |
| "pass@64": 0.9839818197350004, |
| "pass@128": 0.989 |
| }, |
| "17": { |
| "pass@1": 0.8524375, |
| "pass@2": 0.9029194143700803, |
| "pass@4": 0.9286221111736035, |
| "pass@8": 0.9443386208689557, |
| "pass@16": 0.955734081883812, |
| "pass@32": 0.9648144384726397, |
| "pass@64": 0.9726182428749347, |
| "pass@128": 0.98 |
| }, |
| "18": { |
| "pass@1": 0.8109296875, |
| "pass@2": 0.8804515255905518, |
| "pass@4": 0.9190456437945261, |
| "pass@8": 0.9411653142560411, |
| "pass@16": 0.954709171238404, |
| "pass@32": 0.9636073849642399, |
| "pass@64": 0.9701930633795666, |
| "pass@128": 0.975 |
| }, |
| "19": { |
| "pass@1": 0.6708515625, |
| "pass@2": 0.7712925688976383, |
| "pass@4": 0.8335043385826767, |
| "pass@8": 0.8737365678515328, |
| "pass@16": 0.9019963268627569, |
| "pass@32": 0.9241395020391616, |
| "pass@64": 0.9433327950305856, |
| "pass@128": 0.96 |
| }, |
| "20": { |
| "pass@1": 0.6338125, |
| "pass@2": 0.7358758612204728, |
| "pass@4": 0.8046025383389569, |
| "pass@8": 0.8527253835500371, |
| "pass@16": 0.8880693164814899, |
| "pass@32": 0.915272845611458, |
| "pass@64": 0.9357913648933431, |
| "pass@128": 0.949 |
| } |
| }, |
| "per_op_accuracy": { |
| "10": 0.941265625, |
| "2": 1.0, |
| "3": 0.998203125, |
| "4": 0.99871875, |
| "5": 0.9939296875, |
| "6": 0.9972734375, |
| "7": 0.9960859375, |
| "8": 0.991609375, |
| "9": 0.99265625, |
| "11": 0.9624921875, |
| "12": 0.951375, |
| "13": 0.927671875, |
| "14": 0.9342578125, |
| "15": 0.930546875, |
| "16": 0.866078125, |
| "17": 0.8524375, |
| "18": 0.8109296875, |
| "19": 0.6708515625, |
| "20": 0.6338125 |
| }, |
| "per_op_avg_response_len": { |
| "10": 281.453609375, |
| "2": 115.61171875, |
| "3": 145.2217734375, |
| "4": 160.020953125, |
| "5": 173.8887578125, |
| "6": 199.223328125, |
| "7": 224.244828125, |
| "8": 243.6493359375, |
| "9": 259.7151640625, |
| "11": 300.8788359375, |
| "12": 313.412328125, |
| "13": 334.19246875, |
| "14": 361.4050703125, |
| "15": 381.8040859375, |
| "16": 392.90459375, |
| "17": 414.05021875, |
| "18": 432.1644609375, |
| "19": 434.3923984375, |
| "20": 453.55165625 |
| }, |
| "per_op_avg_loss": {}, |
| "per_op_length": { |
| "10": 281.453609375, |
| "2": 115.61171875, |
| "3": 145.2217734375, |
| "4": 160.020953125, |
| "5": 173.8887578125, |
| "6": 199.223328125, |
| "7": 224.244828125, |
| "8": 243.6493359375, |
| "9": 259.7151640625, |
| "11": 300.8788359375, |
| "12": 313.412328125, |
| "13": 334.19246875, |
| "14": 361.4050703125, |
| "15": 381.8040859375, |
| "16": 392.90459375, |
| "17": 414.05021875, |
| "18": 432.1644609375, |
| "19": 434.3923984375, |
| "20": 453.55165625 |
| }, |
| "per_op_loss": {}, |
| "per_template": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 742004, |
| "answer_accuracy": 0.9163620376225102, |
| "avg_response_len": 282.5668360239488, |
| "resp_tokens_sum": 228802279, |
| "pass_at_k": { |
| "pass@1": 0.9163620376225102, |
| "pass@2": 0.9432990699238839, |
| "pass@4": 0.9594926690468993, |
| "pass@8": 0.9702768212202704, |
| "pass@16": 0.978067131021929, |
| "pass@32": 0.9837285212462428, |
| "pass@64": 0.9878001481817852, |
| "pass@128": 0.9908314890926335 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.940058075221239, |
| "pass@2": 0.952202445241911, |
| "pass@4": 0.9639851915634436, |
| "pass@8": 0.9747452943211968, |
| "pass@16": 0.9837117746740343, |
| "pass@32": 0.9903082598333327, |
| "pass@64": 0.9945723951890973, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999641628440367, |
| "pass@2": 0.9999793066291025, |
| "pass@4": 0.999999905401733, |
| "pass@8": 0.9999999999996472, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9995361328125, |
| "pass@2": 0.9999784694881889, |
| "pass@4": 0.9999999370195912, |
| "pass@8": 0.9999999999999016, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.991170634920635, |
| "pass@2": 0.9935695538057743, |
| "pass@4": 0.9953833833270841, |
| "pass@8": 0.996783521967192, |
| "pass@16": 0.9975425302123753, |
| "pass@32": 0.9982188768739435, |
| "pass@64": 0.9992125984251853, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999298278443114, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9981046597633136, |
| "pass@2": 0.9996745853328983, |
| "pass@4": 0.9999718887764769, |
| "pass@8": 0.9999998022755866, |
| "pass@16": 0.9999999999967328, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9924879807692307, |
| "pass@2": 0.995214390268524, |
| "pass@4": 0.9969837258924363, |
| "pass@8": 0.9983251613066434, |
| "pass@16": 0.9993038302603201, |
| "pass@32": 0.9998838431738969, |
| "pass@64": 0.9999990205966608, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9905809859154929, |
| "pass@2": 0.992608059775979, |
| "pass@4": 0.9937131769620348, |
| "pass@8": 0.9952330962332059, |
| "pass@16": 0.9970819743714158, |
| "pass@32": 0.9986530587245823, |
| "pass@64": 0.9996542900027615, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9567767351274787, |
| "pass@2": 0.9729954579420491, |
| "pass@4": 0.9837077224412106, |
| "pass@8": 0.9915053244533025, |
| "pass@16": 0.9967604348473688, |
| "pass@32": 0.9993037377604537, |
| "pass@64": 0.999959962274875, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9463497528248588, |
| "pass@2": 0.9650545370123225, |
| "pass@4": 0.9766829093397222, |
| "pass@8": 0.9848453467690281, |
| "pass@16": 0.9914138441913528, |
| "pass@32": 0.9958525342115812, |
| "pass@64": 0.9982239261012225, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9212627351097179, |
| "pass@2": 0.953259601856194, |
| "pass@4": 0.974031526035734, |
| "pass@8": 0.9872172251386195, |
| "pass@16": 0.9946975877158775, |
| "pass@32": 0.9979212628228779, |
| "pass@64": 0.9992200434190286, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9226781542056075, |
| "pass@2": 0.9488315457355219, |
| "pass@4": 0.966660107907072, |
| "pass@8": 0.9807327520370567, |
| "pass@16": 0.9900033030501925, |
| "pass@32": 0.994705328348, |
| "pass@64": 0.997677205060592, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9338872832369942, |
| "pass@2": 0.9658665826544082, |
| "pass@4": 0.9800264382595235, |
| "pass@8": 0.9852710411507393, |
| "pass@16": 0.9867086223947849, |
| "pass@32": 0.9875391269726735, |
| "pass@64": 0.9882671703608444, |
| "pass@128": 0.9884393063583815 |
| }, |
| "16": { |
| "pass@1": 0.8443296370967742, |
| "pass@2": 0.8949314992379985, |
| "pass@4": 0.9236545057875828, |
| "pass@8": 0.9436194889710628, |
| "pass@16": 0.9597137566119098, |
| "pass@32": 0.9731177144426382, |
| "pass@64": 0.9836290640887158, |
| "pass@128": 0.9935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.8775063222543352, |
| "pass@2": 0.9258678343043105, |
| "pass@4": 0.9494584514544068, |
| "pass@8": 0.9630582535369274, |
| "pass@16": 0.9726016060789089, |
| "pass@32": 0.9786961371782323, |
| "pass@64": 0.9822631761264358, |
| "pass@128": 0.9855491329479769 |
| }, |
| "18": { |
| "pass@1": 0.8074330357142857, |
| "pass@2": 0.8726863048368955, |
| "pass@4": 0.9079920633135147, |
| "pass@8": 0.9290310338171512, |
| "pass@16": 0.9431022516433624, |
| "pass@32": 0.9531360758782476, |
| "pass@64": 0.9606210100169015, |
| "pass@128": 0.9657142857142857 |
| }, |
| "19": { |
| "pass@1": 0.6631240615615616, |
| "pass@2": 0.7647959228913951, |
| "pass@4": 0.8269798164306035, |
| "pass@8": 0.8654212985928951, |
| "pass@16": 0.8938579593227545, |
| "pass@32": 0.9182400396910518, |
| "pass@64": 0.9393446013378842, |
| "pass@128": 0.9579579579579579 |
| }, |
| "20": { |
| "pass@1": 0.6333521021021021, |
| "pass@2": 0.7295261550920993, |
| "pass@4": 0.7942091112484806, |
| "pass@8": 0.8420309365275893, |
| "pass@16": 0.8790149220263148, |
| "pass@32": 0.9073538190959931, |
| "pass@64": 0.9277009935323411, |
| "pass@128": 0.93993993993994 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 757653, |
| "answer_accuracy": 0.9227067907248636, |
| "avg_response_len": 292.3765210931411, |
| "resp_tokens_sum": 240076209, |
| "pass_at_k": { |
| "pass@1": 0.9227067907248636, |
| "pass@2": 0.9491983678141017, |
| "pass@4": 0.9648446673072715, |
| "pass@8": 0.974732532480983, |
| "pass@16": 0.9813596376382586, |
| "pass@32": 0.9859942113928409, |
| "pass@64": 0.9894281143827801, |
| "pass@128": 0.9918939984411536 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9300469708588958, |
| "pass@2": 0.9466099916670692, |
| "pass@4": 0.9608256378803874, |
| "pass@8": 0.9735479075977559, |
| "pass@16": 0.9852696934862603, |
| "pass@32": 0.9940972426092773, |
| "pass@64": 0.9987733649475956, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9961647727272728, |
| "pass@2": 0.9972918157957528, |
| "pass@4": 0.9983365499767074, |
| "pass@8": 0.9991928917832018, |
| "pass@16": 0.9998063298472625, |
| "pass@32": 0.9999922499316101, |
| "pass@64": 0.9999999987964725, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.997965976331361, |
| "pass@2": 0.9991187596095608, |
| "pass@4": 0.9997857705841503, |
| "pass@8": 0.9999865024173374, |
| "pass@16": 0.9999999643806659, |
| "pass@32": 0.9999999999999731, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99375, |
| "pass@2": 0.9967833504507589, |
| "pass@4": 0.9986635988436225, |
| "pass@8": 0.9996831882834758, |
| "pass@16": 0.9999789012150662, |
| "pass@32": 0.9999999417280491, |
| "pass@64": 0.9999999999999963, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9955676934097422, |
| "pass@2": 0.9973236243034089, |
| "pass@4": 0.9983310997300121, |
| "pass@8": 0.9991142096348365, |
| "pass@16": 0.9997496114237934, |
| "pass@32": 0.9999854792844564, |
| "pass@64": 0.999999993573057, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9948067196531792, |
| "pass@2": 0.9959908089071959, |
| "pass@4": 0.9969885587820309, |
| "pass@8": 0.9979235613025731, |
| "pass@16": 0.9987206336864413, |
| "pass@32": 0.9995059858135572, |
| "pass@64": 0.9999600471131015, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9922975352112676, |
| "pass@2": 0.9943263419097261, |
| "pass@4": 0.99571091852955, |
| "pass@8": 0.9969972148105722, |
| "pass@16": 0.99814919003857, |
| "pass@32": 0.9991126715814442, |
| "pass@64": 0.9998322253384277, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.995158121257485, |
| "pass@2": 0.9969157403696542, |
| "pass@4": 0.997358424601865, |
| "pass@8": 0.9976945425380946, |
| "pass@16": 0.9982569113741001, |
| "pass@32": 0.9990676801627342, |
| "pass@64": 0.9998216794756942, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.97265625, |
| "pass@2": 0.9817142586092406, |
| "pass@4": 0.9872070927128084, |
| "pass@8": 0.9907918466392928, |
| "pass@16": 0.9932876981463241, |
| "pass@32": 0.9946033623345881, |
| "pass@64": 0.9954810039531555, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9539086990595611, |
| "pass@2": 0.9742443783476912, |
| "pass@4": 0.9873180110910899, |
| "pass@8": 0.9939946254019711, |
| "pass@16": 0.9967163926023948, |
| "pass@32": 0.9983956273931865, |
| "pass@64": 0.9996077018477316, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9435836834733894, |
| "pass@2": 0.9670878686120119, |
| "pass@4": 0.9795068925207874, |
| "pass@8": 0.9867661154939795, |
| "pass@16": 0.9912037146425846, |
| "pass@32": 0.9943074150090554, |
| "pass@64": 0.9971910964276763, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9339820906432749, |
| "pass@2": 0.961808433485288, |
| "pass@4": 0.9791213154276766, |
| "pass@8": 0.9903633423119838, |
| "pass@16": 0.9965715597469695, |
| "pass@32": 0.998978060543375, |
| "pass@64": 0.9998257129593244, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.936032196969697, |
| "pass@2": 0.9698241022429014, |
| "pass@4": 0.986383716808126, |
| "pass@8": 0.993887020431712, |
| "pass@16": 0.997341288437486, |
| "pass@32": 0.9992550901802849, |
| "pass@64": 0.999953724719699, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8875979344729344, |
| "pass@2": 0.9302362344931245, |
| "pass@4": 0.9518918415112638, |
| "pass@8": 0.9653093003879017, |
| "pass@16": 0.97478416069728, |
| "pass@32": 0.9809651521660178, |
| "pass@64": 0.986492391819359, |
| "pass@128": 0.9914529914529915 |
| }, |
| "17": { |
| "pass@1": 0.8626746894409938, |
| "pass@2": 0.9124707322590117, |
| "pass@4": 0.9368205890024611, |
| "pass@8": 0.9508498521267209, |
| "pass@16": 0.960456726487503, |
| "pass@32": 0.9678057412486193, |
| "pass@64": 0.9734389189717876, |
| "pass@128": 0.9782608695652174 |
| }, |
| "18": { |
| "pass@1": 0.8121553308823529, |
| "pass@2": 0.8809753502779064, |
| "pass@4": 0.9216372758552233, |
| "pass@8": 0.9462221099752758, |
| "pass@16": 0.9606949776103674, |
| "pass@32": 0.9685754572068952, |
| "pass@64": 0.9731564892213497, |
| "pass@128": 0.9764705882352941 |
| }, |
| "19": { |
| "pass@1": 0.66943359375, |
| "pass@2": 0.7666238985751784, |
| "pass@4": 0.8258763982627161, |
| "pass@8": 0.8655574931646675, |
| "pass@16": 0.8941625232220163, |
| "pass@32": 0.9161714667742615, |
| "pass@64": 0.9350029611318369, |
| "pass@128": 0.9523809523809523 |
| }, |
| "20": { |
| "pass@1": 0.6556652320359282, |
| "pass@2": 0.7598498868404915, |
| "pass@4": 0.8262946623625637, |
| "pass@8": 0.8690020135890967, |
| "pass@16": 0.8983220264807161, |
| "pass@32": 0.9212925306509724, |
| "pass@64": 0.9391524696479382, |
| "pass@128": 0.9491017964071856 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 733968, |
| "answer_accuracy": 0.9161407573094743, |
| "avg_response_len": 312.93695453546894, |
| "resp_tokens_sum": 250710067, |
| "pass_at_k": { |
| "pass@1": 0.9161407573094743, |
| "pass@2": 0.9444789070667229, |
| "pass@4": 0.9620470564981104, |
| "pass@8": 0.9734481996656786, |
| "pass@16": 0.9808410217800196, |
| "pass@32": 0.985989950434011, |
| "pass@64": 0.9899999651668862, |
| "pass@128": 0.993129892954146 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9534048507462687, |
| "pass@2": 0.970000514161476, |
| "pass@4": 0.9828655804218498, |
| "pass@8": 0.9913297716090343, |
| "pass@16": 0.9958306599914305, |
| "pass@32": 0.9979223480778879, |
| "pass@64": 0.9992399818731803, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9987928206997084, |
| "pass@2": 0.99955952572255, |
| "pass@4": 0.9999370614387487, |
| "pass@8": 0.9999989051962135, |
| "pass@16": 0.9999999998462147, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986979166666666, |
| "pass@2": 0.9994841310954552, |
| "pass@4": 0.9999133192232549, |
| "pass@8": 0.9999978720766682, |
| "pass@16": 0.9999999993392, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9966681985294118, |
| "pass@2": 0.9992585543075498, |
| "pass@4": 0.999930265610181, |
| "pass@8": 0.9999993599537156, |
| "pass@16": 0.9999999999763772, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.996352523659306, |
| "pass@2": 0.9972878735189645, |
| "pass@4": 0.9978065682444268, |
| "pass@8": 0.9984953578892186, |
| "pass@16": 0.9993197040358152, |
| "pass@32": 0.9998856774168334, |
| "pass@64": 0.999999036044663, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9953273338607594, |
| "pass@2": 0.9967836607943786, |
| "pass@4": 0.9975761380302146, |
| "pass@8": 0.998262935853195, |
| "pass@16": 0.9990878595282221, |
| "pass@32": 0.9997847637219769, |
| "pass@64": 0.9999954280485364, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9900525525525525, |
| "pass@2": 0.9931176254995152, |
| "pass@4": 0.9950761472946511, |
| "pass@8": 0.996932633631575, |
| "pass@16": 0.9985345359444987, |
| "pass@32": 0.9996000242336316, |
| "pass@64": 0.9999802487805222, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9923382234726688, |
| "pass@2": 0.9954557466389852, |
| "pass@4": 0.9969244255440737, |
| "pass@8": 0.9981047741525653, |
| "pass@16": 0.9990691147291293, |
| "pass@32": 0.9997813019589095, |
| "pass@64": 0.9999953545444935, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9581845238095238, |
| "pass@2": 0.973211942257218, |
| "pass@4": 0.9851733461888689, |
| "pass@8": 0.9935984850997558, |
| "pass@16": 0.9981004697081209, |
| "pass@32": 0.9997117617660194, |
| "pass@64": 0.9999952151978636, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9543434633027523, |
| "pass@2": 0.9737747266970067, |
| "pass@4": 0.9860677859648275, |
| "pass@8": 0.994249571393947, |
| "pass@16": 0.9986259898728691, |
| "pass@32": 0.9998862026515789, |
| "pass@64": 0.9999995629648154, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9164496527777778, |
| "pass@2": 0.9467269824778841, |
| "pass@4": 0.966515018955963, |
| "pass@8": 0.9796284799576312, |
| "pass@16": 0.9873678160183381, |
| "pass@32": 0.9923304170452849, |
| "pass@64": 0.9960751154460078, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9455675074183977, |
| "pass@2": 0.9688518569359099, |
| "pass@4": 0.9830591241005852, |
| "pass@8": 0.9918079912361211, |
| "pass@16": 0.9964965731596999, |
| "pass@32": 0.9989974984823478, |
| "pass@64": 0.999910515524606, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9213927469135802, |
| "pass@2": 0.9536509216729855, |
| "pass@4": 0.9729866107593029, |
| "pass@8": 0.9846111586848604, |
| "pass@16": 0.9907910506066029, |
| "pass@32": 0.9946077911424446, |
| "pass@64": 0.9975593713245301, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8636845501474927, |
| "pass@2": 0.9176269656005392, |
| "pass@4": 0.9461433612280757, |
| "pass@8": 0.9632940350686785, |
| "pass@16": 0.9731921968948374, |
| "pass@32": 0.9788198865927932, |
| "pass@64": 0.9817049567519268, |
| "pass@128": 0.9823008849557522 |
| }, |
| "17": { |
| "pass@1": 0.816382718373494, |
| "pass@2": 0.8697396623897163, |
| "pass@4": 0.8989555943120052, |
| "pass@8": 0.918514496266187, |
| "pass@16": 0.9335748802757035, |
| "pass@32": 0.9474461636352892, |
| "pass@64": 0.9617706386333493, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8135332661290322, |
| "pass@2": 0.8886441929133857, |
| "pass@4": 0.9286830575613528, |
| "pass@8": 0.9493191452530526, |
| "pass@16": 0.9612486799216202, |
| "pass@32": 0.9699809772790603, |
| "pass@64": 0.9777500114141688, |
| "pass@128": 0.9838709677419355 |
| }, |
| "19": { |
| "pass@1": 0.6800651435045317, |
| "pass@2": 0.7825676636058709, |
| "pass@4": 0.8478114498338757, |
| "pass@8": 0.890404706092961, |
| "pass@16": 0.9181360078721577, |
| "pass@32": 0.9381630090208142, |
| "pass@64": 0.9558007487757464, |
| "pass@128": 0.9697885196374623 |
| }, |
| "20": { |
| "pass@1": 0.6123545420420421, |
| "pass@2": 0.7181795476578936, |
| "pass@4": 0.7932386998922434, |
| "pass@8": 0.8470943217645397, |
| "pass@16": 0.8868402120185244, |
| "pass@32": 0.9171541099551569, |
| "pass@64": 0.9405105380920781, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| } |
| } |
