[ { "checkpoint": "checkpoint-38796", "id_acc": 0.9916449652777778, "ood_acc": 0.88128359375, "total_acc": 0.9335600328947369, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.28180034722223, "ood_avg_resp_len": 381.877265625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 754568, "answer_accuracy": 0.9318783591527031, "avg_response_len": 282.8403031635315, "resp_tokens_sum": 229023713, "pass_at_k": { "pass@1": 0.9318783591527031, "pass@2": 0.9517385910166891, "pass@4": 0.964830907829337, "pass@8": 0.9741300581165051, "pass@16": 0.9808451032894651, "pass@32": 0.9856391594646161, "pass@64": 0.9891717827349858, "pass@128": 0.9916218779639583 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9496450958702065, "pass@2": 0.9618148706245786, "pass@4": 0.9709195504323019, "pass@8": 0.9791190874933853, "pass@16": 0.9868342218489051, "pass@32": 0.993298633445245, "pass@64": 0.9981549989323758, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997849770642202, "pass@2": 0.9999864552481399, "pass@4": 0.9999999638806617, "pass@8": 0.9999999999999809, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999755859375, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9954117063492064, "pass@2": 0.9981478877640296, "pass@4": 0.9995556263800359, "pass@8": 0.9999615639259287, "pass@16": 0.9999997091342684, "pass@32": 0.9999999999966475, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9997427020958084, "pass@2": 0.999995948064501, "pass@4": 0.9999999985967324, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9956083579881657, "pass@2": 0.9976401976657503, "pass@4": 0.9985032811172272, "pass@8": 0.9992715079029393, "pass@16": 0.9998386699365346, "pass@32": 0.9999946547625592, "pass@64": 0.9999999995148912, "pass@128": 1.0 }, "8": { "pass@1": 0.9960186298076923, "pass@2": 0.9985326916767614, "pass@4": 0.9996494007960544, "pass@8": 0.9999753031040152, "pass@16": 0.9999998899899415, "pass@32": 0.9999999999996564, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9917033450704226, "pass@2": 0.9929716091826549, "pass@4": 0.9939165605179635, "pass@8": 0.9949288254759624, "pass@16": 0.9958752495136213, "pass@32": 0.9970100988051338, "pass@64": 0.9984237773095228, "pass@128": 1.0 }, "11": { "pass@1": 0.9635047804532578, "pass@2": 0.977436093328277, "pass@4": 0.9865923250386901, "pass@8": 0.9927103321039095, "pass@16": 0.9970124444229089, "pass@32": 0.9993847243796702, "pass@64": 0.9999852421618102, "pass@128": 1.0 }, "12": { "pass@1": 0.9450918079096046, "pass@2": 0.9655720338983056, "pass@4": 0.9781990243804268, "pass@8": 0.9861440250122284, "pass@16": 0.9923112374071664, "pass@32": 0.9969355017981401, "pass@64": 0.9992276493263522, "pass@128": 1.0 }, "13": { "pass@1": 0.9179075235109718, "pass@2": 0.9502324099671706, "pass@4": 0.9729623692218725, "pass@8": 0.9865352063542692, "pass@16": 0.9929416594162586, "pass@32": 0.9950239922483407, "pass@64": 0.9960875780112477, "pass@128": 0.9968652037617555 }, "14": { "pass@1": 0.9201956775700935, "pass@2": 0.9446224121225502, "pass@4": 0.9626010729383125, "pass@8": 0.9762203698107813, "pass@16": 0.986550455857625, "pass@32": 0.9938274615772782, "pass@64": 0.9980046339203555, "pass@128": 1.0 }, "15": { "pass@1": 0.9622696893063584, "pass@2": 0.9790736226161763, "pass@4": 0.9878219928975642, "pass@8": 0.9938863424829786, "pass@16": 0.9974854612936562, "pass@32": 0.9990553826707531, "pass@64": 0.9998278336678724, "pass@128": 1.0 }, "16": { "pass@1": 0.8738155241935484, "pass@2": 0.9108982092964184, "pass@4": 0.9368475640141757, "pass@8": 0.9526611840974472, "pass@16": 0.9624771254488251, "pass@32": 0.9706463781509711, "pass@64": 0.97815086262173, "pass@128": 0.9838709677419355 }, "17": { "pass@1": 0.8999954841040463, "pass@2": 0.9269427597742481, "pass@4": 0.9442069857777889, "pass@8": 0.9584880211486807, "pass@16": 0.9699099356325508, "pass@32": 0.977956592718777, "pass@64": 0.9832164176739884, "pass@128": 0.9884393063583815 }, "18": { "pass@1": 0.8340625, "pass@2": 0.875173298650169, "pass@4": 0.9003785883014621, "pass@8": 0.9191113837595276, "pass@16": 0.9347451165816725, "pass@32": 0.9484697556158473, "pass@64": 0.9606376972907202, "pass@128": 0.9685714285714285 }, "19": { "pass@1": 0.7546218093093093, "pass@2": 0.8168980988862881, "pass@4": 0.8594574675913256, "pass@8": 0.8903657440259747, "pass@16": 0.9127998493831533, "pass@32": 0.9301485120186141, "pass@64": 0.9444138248825593, "pass@128": 0.954954954954955 }, "20": { "pass@1": 0.7120167042042042, "pass@2": 0.7915602610484499, "pass@4": 0.8442564102347557, "pass@8": 0.8824961305644767, "pass@16": 0.9097409929460565, "pass@32": 0.927053074881089, "pass@64": 0.9392787772648254, "pass@128": 0.948948948948949 } } }, "teachers_in_school": { "count": 821120, "correct": 769502, "answer_accuracy": 0.9371370810600156, "avg_response_len": 292.1462916504287, "resp_tokens_sum": 239887163, "pass_at_k": { "pass@1": 0.9371370810600156, "pass@2": 0.9563403509552427, "pass@4": 0.9692129536808676, "pass@8": 0.9783119434671995, "pass@16": 0.984564498023125, "pass@32": 0.9886527260310264, "pass@64": 0.9913362234234722, "pass@128": 0.9934528448947778 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9329706671779141, "pass@2": 0.9490174085792956, "pass@4": 0.9641241520423445, "pass@8": 0.976390759682789, "pass@16": 0.9846418367918528, "pass@32": 0.9893313200761069, "pass@64": 0.9929169376529189, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9995265151515151, "pass@2": 0.9999742752326415, "pass@4": 0.9999999304064264, "pass@8": 0.9999999999999045, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9985207100591716, "pass@2": 0.9996389134790105, "pass@4": 0.9999800542683642, "pass@8": 0.9999999564675484, "pass@16": 0.999999999999962, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9973505434782609, "pass@2": 0.9985510812507132, "pass@4": 0.9993644259141521, "pass@8": 0.9998696856114555, "pass@16": 0.9999955044084027, "pass@32": 0.9999999980554193, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9993508237822349, "pass@2": 0.999962984793448, "pass@4": 0.9999998469030911, "pass@8": 0.9999999999990058, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9994806719653179, "pass@2": 0.9999896881116016, "pass@4": 0.9999999953943619, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.993155809859155, "pass@2": 0.9950745813463459, "pass@4": 0.9964881184746273, "pass@8": 0.9976291893269761, "pass@16": 0.9985375133295249, "pass@32": 0.9993490121785924, "pass@64": 0.9999188200065704, "pass@128": 1.0 }, "9": { "pass@1": 0.9948774326347305, "pass@2": 0.9968984275543402, "pass@4": 0.9971880869868812, "pass@8": 0.9973699249827601, "pass@16": 0.9977102880852468, "pass@32": 0.9983202885567448, "pass@64": 0.9992573907303504, "pass@128": 1.0 }, "11": { "pass@1": 0.9722797439759037, "pass@2": 0.9806206864861019, "pass@4": 0.987148461675724, "pass@8": 0.992097089748973, "pass@16": 0.9952356342084575, "pass@32": 0.996694117094797, "pass@64": 0.9969831820276938, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9480309561128527, "pass@2": 0.9714501363759779, "pass@4": 0.9865298393343465, "pass@8": 0.9942286337372813, "pass@16": 0.9978628049514008, "pass@32": 0.9996281644114471, "pass@64": 0.9999956190522251, "pass@128": 1.0 }, "13": { "pass@1": 0.9407169117647058, "pass@2": 0.9625553469419262, "pass@4": 0.9774801687079033, "pass@8": 0.9874219685475781, "pass@16": 0.9934829131966962, "pass@32": 0.9968338780653557, "pass@64": 0.9985126025262858, "pass@128": 1.0 }, "14": { "pass@1": 0.9497669956140351, "pass@2": 0.9702634312980611, "pass@4": 0.9843248648195289, "pass@8": 0.9936793355130432, "pass@16": 0.9984837564707297, "pass@32": 0.999880078009628, "pass@64": 0.999999559195243, "pass@128": 1.0 }, "15": { "pass@1": 0.9554214015151515, "pass@2": 0.9778524665950848, "pass@4": 0.990172612230289, "pass@8": 0.9965689152787764, "pass@16": 0.9990389896488282, "pass@32": 0.9998467083150427, "pass@64": 0.9999979765261304, "pass@128": 1.0 }, "16": { "pass@1": 0.8984820156695157, "pass@2": 0.9263237628149055, "pass@4": 0.9466511985147156, "pass@8": 0.9628916229912796, "pass@16": 0.9756373410907596, "pass@32": 0.9843113854696494, "pass@64": 0.9896611343512275, "pass@128": 0.9943019943019943 }, "17": { "pass@1": 0.8989955357142857, "pass@2": 0.9258540372670809, "pass@4": 0.9430381279009376, "pass@8": 0.9567724437885808, "pass@16": 0.9678264048731167, "pass@32": 0.975789443612367, "pass@64": 0.9804645618033802, "pass@128": 0.984472049689441 }, "18": { "pass@1": 0.8785386029411765, "pass@2": 0.9263135421491432, "pass@4": 0.9540938742032239, "pass@8": 0.970755896754488, "pass@16": 0.9808158406179858, "pass@32": 0.9865832611536812, "pass@64": 0.989914086501943, "pass@128": 0.9911764705882353 }, "19": { "pass@1": 0.7576497395833334, "pass@2": 0.8213040752718409, "pass@4": 0.8629211197930617, "pass@8": 0.8939620372022703, "pass@16": 0.9169075579630253, "pass@32": 0.9325509025737705, "pass@64": 0.94412425958293, "pass@128": 0.9553571428571429 }, "20": { "pass@1": 0.6838978293413174, "pass@2": 0.7640517439530387, "pass@4": 0.8220046661457729, "pass@8": 0.8655627671390692, "pass@16": 0.8983598878087751, "pass@32": 0.9234805889172719, "pass@64": 0.9421120203018302, "pass@128": 0.9550898203592815 } } }, "movie_festival_awards": { "count": 801152, "correct": 746348, "answer_accuracy": 0.9315935053522927, "avg_response_len": 312.82036117990094, "resp_tokens_sum": 250616658, "pass_at_k": { "pass@1": 0.9315935053522927, "pass@2": 0.9525782282961234, "pass@4": 0.966021727662297, "pass@8": 0.975328068054194, "pass@16": 0.9821933995365425, "pass@32": 0.9873189930516977, "pass@64": 0.9910883906699982, "pass@128": 0.9936092027480428 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9567863805970149, "pass@2": 0.971851495475379, "pass@4": 0.9827990326768856, "pass@8": 0.9908164062722622, "pass@16": 0.9960200237300442, "pass@32": 0.9982169750293176, "pass@64": 0.9992594797136242, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9989720394736842, "pass@2": 0.9996438561956071, "pass@4": 0.9999591621770964, "pass@8": 0.9999995591583335, "pass@16": 0.9999999999797444, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.99453125, "pass@2": 0.9965507758221399, "pass@4": 0.9972536858444163, "pass@8": 0.9975790660148303, "pass@16": 0.9980363206014791, "pass@32": 0.9987689957872913, "pass@64": 0.9996410375173692, "pass@128": 1.0 }, "6": { "pass@1": 0.9999014195583596, "pass@2": 0.999999611888025, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9945361946202531, "pass@2": 0.9958628712747932, "pass@4": 0.9969610167399962, "pass@8": 0.997929570922725, "pass@16": 0.9987857390759178, "pass@32": 0.9996009663236596, "pass@64": 0.9999792027771768, "pass@128": 1.0 }, "8": { "pass@1": 0.9935013138138138, "pass@2": 0.9976191742924027, "pass@4": 0.9995169683181494, "pass@8": 0.9999713902369773, "pass@16": 0.9999998669378484, "pass@32": 0.9999999999994204, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9927150321543409, "pass@2": 0.9953299459452619, "pass@4": 0.9969358378032327, "pass@8": 0.997933512235023, "pass@16": 0.9987673566627887, "pass@32": 0.9995945511170884, "pass@64": 0.9999788684166812, "pass@128": 1.0 }, "11": { "pass@1": 0.9630704365079366, "pass@2": 0.9776848206474192, "pass@4": 0.9883574401414106, "pass@8": 0.9946295131968437, "pass@16": 0.997513997070458, "pass@32": 0.9989501368011887, "pass@64": 0.9998107266450182, "pass@128": 1.0 }, "12": { "pass@1": 0.9505208333333334, "pass@2": 0.9706150370825212, "pass@4": 0.9850789382405178, "pass@8": 0.994133402138561, "pass@16": 0.9983903509266632, "pass@32": 0.9997473498922584, "pass@64": 0.9999954976826797, "pass@128": 1.0 }, "13": { "pass@1": 0.9122058256172839, "pass@2": 0.9404311114756491, "pass@4": 0.9606684896448127, "pass@8": 0.9762644289391496, "pass@16": 0.9877580918111676, "pass@32": 0.9948566354295425, "pass@64": 0.9986678916722411, "pass@128": 1.0 }, "14": { "pass@1": 0.955628709198813, "pass@2": 0.9747088132900301, "pass@4": 0.9841755971156422, "pass@8": 0.9898190798079135, "pass@16": 0.9938872827202818, "pass@32": 0.9970027263205429, "pass@64": 0.99907716563192, "pass@128": 1.0 }, "15": { "pass@1": 0.9384645061728395, "pass@2": 0.9601393141829493, "pass@4": 0.9720348755595365, "pass@8": 0.981293627330329, "pass@16": 0.9892875060551926, "pass@32": 0.9937818556587116, "pass@64": 0.995354403432346, "pass@128": 0.9969135802469136 }, "16": { "pass@1": 0.9030927359882006, "pass@2": 0.9313183024411773, "pass@4": 0.949399185168225, "pass@8": 0.9638089071611805, "pass@16": 0.9753050149460608, "pass@32": 0.9843716916440536, "pass@64": 0.9918881750232778, "pass@128": 0.9970501474926253 }, "17": { "pass@1": 0.8378200301204819, "pass@2": 0.8722340147993546, "pass@4": 0.8994627833606642, "pass@8": 0.921389194949213, "pass@16": 0.9387280323416873, "pass@32": 0.953114829081259, "pass@64": 0.9652418490909267, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8834929435483871, "pass@2": 0.9286580041910086, "pass@4": 0.9520407616386656, "pass@8": 0.966049497344724, "pass@16": 0.9765995183921604, "pass@32": 0.9846246388700052, "pass@64": 0.989084716853413, "pass@128": 0.9903225806451613 }, "19": { "pass@1": 0.7401812688821753, "pass@2": 0.8118499030615883, "pass@4": 0.8589915267389155, "pass@8": 0.8922240120160714, "pass@16": 0.9169443378799137, "pass@32": 0.9360698853549126, "pass@64": 0.9515090838923458, "pass@128": 0.9607250755287009 }, "20": { "pass@1": 0.6837227852852853, "pass@2": 0.7745815451278046, "pass@4": 0.8314440877885758, "pass@8": 0.8682970775356633, "pass@16": 0.8966446199160057, "pass@32": 0.92130107609107, "pass@64": 0.9418564963876911, "pass@128": 0.9579579579579579 } } } } } ]