[ { "checkpoint": "checkpoint-21350", "id_acc": 0.9463585069444445, "ood_acc": 0.3788546875, "total_acc": 0.6476722861842106, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.93385069444443, "ood_avg_resp_len": 316.5756515625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 525646, "answer_accuracy": 0.6491636697755295, "avg_response_len": 248.8114650845716, "resp_tokens_sum": 201469610, "pass_at_k": { "pass@1": 0.6491636697755295, "pass@2": 0.6918507562216686, "pass@4": 0.7265823497333529, "pass@8": 0.7552785357582993, "pass@16": 0.779610731297473, "pass@32": 0.8004989331569241, "pass@64": 0.8181713588791755, "pass@128": 0.8330698703762252 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8773276179941003, "pass@2": 0.9119181590133094, "pass@4": 0.9311756113450421, "pass@8": 0.9420426122301978, "pass@16": 0.9502590673394334, "pass@32": 0.9584931653088372, "pass@64": 0.9669784816191926, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9973241590214067, "pass@2": 0.9977444225721784, "pass@4": 0.9983450506186725, "pass@8": 0.9991236482754671, "pass@16": 0.9997712860612012, "pass@32": 0.9999889701449637, "pass@64": 0.9999999970955962, "pass@128": 1.0 }, "4": { "pass@1": 0.9881591796875, "pass@2": 0.9902486005167324, "pass@4": 0.9916557953693287, "pass@8": 0.9928006480587112, "pass@16": 0.9937693424773156, "pass@32": 0.994500319642821, "pass@64": 0.9953124622828344, "pass@128": 0.996875 }, "5": { "pass@1": 0.9253224206349207, "pass@2": 0.9414018560179978, "pass@4": 0.9528689018039411, "pass@8": 0.9625002308833911, "pass@16": 0.9709636195225438, "pass@32": 0.9777918358496315, "pass@64": 0.9838997800481912, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.9671594311377245, "pass@2": 0.9752820883822906, "pass@4": 0.9815461927663233, "pass@8": 0.9861414925202345, "pass@16": 0.9886085746496264, "pass@32": 0.9900551906526195, "pass@64": 0.9917723246221735, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.9505593565088757, "pass@2": 0.9665288810977031, "pass@4": 0.976226311115696, "pass@8": 0.9825814576886557, "pass@16": 0.9870875795976058, "pass@32": 0.9910536770529155, "pass@64": 0.9947454835702054, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.9407301682692307, "pass@2": 0.9576164382697357, "pass@4": 0.9672972564126601, "pass@8": 0.9735731917238386, "pass@16": 0.977776925424273, "pass@32": 0.980567025073276, "pass@64": 0.9823063355516893, "pass@128": 0.9839743589743589 }, "9": { "pass@1": 0.8992077464788732, "pass@2": 0.9294131917489195, "pass@4": 0.9485870498582041, "pass@8": 0.9623563850919464, "pass@16": 0.9729279398669448, "pass@32": 0.9810885217868446, "pass@64": 0.9864213997414775, "pass@128": 0.9887323943661972 }, "11": { "pass@1": 0.8281471317280453, "pass@2": 0.8772393126408069, "pass@4": 0.9072971293779494, "pass@8": 0.9263560752155664, "pass@16": 0.9417339425971359, "pass@32": 0.9566709407386268, "pass@64": 0.9699104428716377, "pass@128": 0.9801699716713881 }, "12": { "pass@1": 0.7567973163841808, "pass@2": 0.8258520452422264, "pass@4": 0.8681805065680348, "pass@8": 0.8951861291592782, "pass@16": 0.9141290331047311, "pass@32": 0.9301895220722955, "pass@64": 0.9461462846873937, "pass@128": 0.96045197740113 }, "13": { "pass@1": 0.5737656739811913, "pass@2": 0.6663702083281907, "pass@4": 0.7377947493044555, "pass@8": 0.7931345691897921, "pass@16": 0.8379584677978166, "pass@32": 0.8751047840970735, "pass@64": 0.9032302694399705, "pass@128": 0.9216300940438872 }, "14": { "pass@1": 0.45030179127725856, "pass@2": 0.5457960789363944, "pass@4": 0.6231636244067621, "pass@8": 0.6869051629447177, "pass@16": 0.7392870707923059, "pass@32": 0.7804201059407638, "pass@64": 0.8141525302007103, "pass@128": 0.8473520249221184 }, "15": { "pass@1": 0.3126354768786127, "pass@2": 0.3937431017022439, "pass@4": 0.4662351267688358, "pass@8": 0.5297946130837053, "pass@16": 0.586091346781155, "pass@32": 0.6351826501037516, "pass@64": 0.6761407868785784, "pass@128": 0.7109826589595376 }, "16": { "pass@1": 0.25042842741935484, "pass@2": 0.32574811087122146, "pass@4": 0.39351368578927626, "pass@8": 0.45303587701396264, "pass@16": 0.5062086173550845, "pass@32": 0.5539039754509739, "pass@64": 0.5955508562334999, "pass@128": 0.632258064516129 }, "17": { "pass@1": 0.16848807803468208, "pass@2": 0.22524364503208774, "pass@4": 0.2802942645175134, "pass@8": 0.330783163112241, "pass@16": 0.37907349032212845, "pass@32": 0.42469351653658793, "pass@64": 0.4638600016363443, "pass@128": 0.4913294797687861 }, "18": { "pass@1": 0.1693080357142857, "pass@2": 0.22570620078740164, "pass@4": 0.28333418795864795, "pass@8": 0.33963163946315195, "pass@16": 0.3904539856568932, "pass@32": 0.4325615854877157, "pass@64": 0.464574906335151, "pass@128": 0.49142857142857144 }, "19": { "pass@1": 0.16042605105105104, "pass@2": 0.21579022427939754, "pass@4": 0.2713904286738932, "pass@8": 0.320673574822643, "pass@16": 0.36145502547813574, "pass@32": 0.3960321250148641, "pass@64": 0.42531332447168435, "pass@128": 0.45045045045045046 }, "20": { "pass@1": 0.15836148648648649, "pass@2": 0.20984591875339906, "pass@4": 0.25997215860404826, "pass@8": 0.3057199370131288, "pass@16": 0.34552328217296413, "pass@32": 0.38000113840819033, "pass@64": 0.41232314729781633, "pass@128": 0.4444444444444444 } } }, "teachers_in_school": { "count": 821120, "correct": 532584, "answer_accuracy": 0.6486067809820732, "avg_response_len": 257.71719967848793, "resp_tokens_sum": 211616747, "pass_at_k": { "pass@1": 0.6486067809820732, "pass@2": 0.6908541473600844, "pass@4": 0.7264055504607114, "pass@8": 0.7563388816938654, "pass@16": 0.7814218557464435, "pass@32": 0.8029980862811904, "pass@64": 0.821935053724575, "pass@128": 0.8388152766952455 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8393884202453987, "pass@2": 0.8698545662045313, "pass@4": 0.8904956702114691, "pass@8": 0.9055833604644317, "pass@16": 0.9173558658878211, "pass@32": 0.9274155966730994, "pass@64": 0.9375057974027622, "pass@128": 0.9478527607361963 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9979876893939394, "pass@2": 0.9995783375089478, "pass@4": 0.9999699469384509, "pass@8": 0.9999997974031156, "pass@16": 0.9999999999966537, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9895062869822485, "pass@2": 0.9918151295252295, "pass@4": 0.9931505192768064, "pass@8": 0.9945047565814593, "pass@16": 0.9959301126815046, "pass@32": 0.9968252000189942, "pass@64": 0.9970371391134153, "pass@128": 0.9970414201183432 }, "5": { "pass@1": 0.9276947463768116, "pass@2": 0.941486933698505, "pass@4": 0.9521146527879665, "pass@8": 0.9596253376638073, "pass@16": 0.9648710403519182, "pass@32": 0.9689627240711924, "pass@64": 0.9727993021286935, "pass@128": 0.9768115942028985 }, "6": { "pass@1": 0.973047994269341, "pass@2": 0.9776248364280394, "pass@4": 0.9810814086097404, "pass@8": 0.9838584328812273, "pass@16": 0.9865986488348304, "pass@32": 0.9898077163259399, "pass@64": 0.9933483043882753, "pass@128": 0.997134670487106 }, "7": { "pass@1": 0.949534862716763, "pass@2": 0.9634724466341997, "pass@4": 0.9741216728328035, "pass@8": 0.9818345949858238, "pass@16": 0.9860409160803892, "pass@32": 0.9882156768256282, "pass@64": 0.9906125875539492, "pass@128": 0.9942196531791907 }, "8": { "pass@1": 0.9240096830985915, "pass@2": 0.94706491904181, "pass@4": 0.9618155671562175, "pass@8": 0.972741330632731, "pass@16": 0.9807618398300412, "pass@32": 0.9868507906518273, "pass@64": 0.9914611399873605, "pass@128": 0.9943661971830986 }, "9": { "pass@1": 0.8909758607784432, "pass@2": 0.918286458578905, "pass@4": 0.9358988911939895, "pass@8": 0.9502432948194586, "pass@16": 0.9635661425413976, "pass@32": 0.9753201066438478, "pass@64": 0.9822986698173887, "pass@128": 0.9850299401197605 }, "11": { "pass@1": 0.8322195030120482, "pass@2": 0.8740064836116118, "pass@4": 0.9007651367562485, "pass@8": 0.9202482006306049, "pass@16": 0.9352773543837686, "pass@32": 0.9477013646227804, "pass@64": 0.9572351213137299, "pass@128": 0.963855421686747 }, "12": { "pass@1": 0.7488489420062696, "pass@2": 0.8215862038111226, "pass@4": 0.8709659145271413, "pass@8": 0.9032511169652427, "pass@16": 0.9243224931700824, "pass@32": 0.9391710642481247, "pass@64": 0.9510478149135736, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.5842086834733894, "pass@2": 0.676471966739452, "pass@4": 0.741411255840919, "pass@8": 0.7904271208733067, "pass@16": 0.8313294364350797, "pass@32": 0.8672863371588099, "pass@64": 0.8973304964641204, "pass@128": 0.9215686274509803 }, "14": { "pass@1": 0.45575201023391815, "pass@2": 0.5523883938849747, "pass@4": 0.6318049690060672, "pass@8": 0.6928149400614, "pass@16": 0.7359096694993563, "pass@32": 0.7666387497632563, "pass@64": 0.790515190000481, "pass@128": 0.8099415204678363 }, "15": { "pass@1": 0.3084280303030303, "pass@2": 0.3932630935337627, "pass@4": 0.471590225085501, "pass@8": 0.5436062297147697, "pass@16": 0.6081874040818871, "pass@32": 0.6620940150440326, "pass@64": 0.7063833432601488, "pass@128": 0.7454545454545455 }, "16": { "pass@1": 0.22760861823361822, "pass@2": 0.3023122209884019, "pass@4": 0.3737860908412092, "pass@8": 0.43597528456928797, "pass@16": 0.4874427513369775, "pass@32": 0.532987304252585, "pass@64": 0.5746519986523746, "pass@128": 0.6096866096866097 }, "17": { "pass@1": 0.19157608695652173, "pass@2": 0.2501818726463538, "pass@4": 0.3103001718147032, "pass@8": 0.36859091329527444, "pass@16": 0.4233694246594647, "pass@32": 0.47383565180387, "pass@64": 0.5219014440208865, "pass@128": 0.5714285714285714 }, "18": { "pass@1": 0.1758501838235294, "pass@2": 0.2325841680176007, "pass@4": 0.2908814795209421, "pass@8": 0.3483809368179013, "pass@16": 0.4025550346930342, "pass@32": 0.45392796747630026, "pass@64": 0.5037675550562966, "pass@128": 0.5529411764705883 }, "19": { "pass@1": 0.14306640625, "pass@2": 0.19530993684383194, "pass@4": 0.2483651784040386, "pass@8": 0.29777947891872014, "pass@16": 0.34304739154710734, "pass@32": 0.3863642388713654, "pass@64": 0.4263632707832371, "pass@128": 0.46130952380952384 }, "20": { "pass@1": 0.14897548652694612, "pass@2": 0.20376631088217265, "pass@4": 0.25812340644919385, "pass@8": 0.30629608870839276, "pass@16": 0.3463328241220228, "pass@32": 0.3798774817729502, "pass@64": 0.40926661845536116, "pass@128": 0.437125748502994 } } }, "movie_festival_awards": { "count": 801152, "correct": 516909, "answer_accuracy": 0.6452071517015497, "avg_response_len": 274.7921405675827, "resp_tokens_sum": 220150273, "pass_at_k": { "pass@1": 0.6452071517015497, "pass@2": 0.6865093792497867, "pass@4": 0.7209135034376871, "pass@8": 0.7497234395452865, "pass@16": 0.7733847971497279, "pass@32": 0.7930687594794426, "pass@64": 0.810047360600001, "pass@128": 0.8256910049528678 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8871035447761194, "pass@2": 0.917110559407686, "pass@4": 0.9365289956292769, "pass@8": 0.95085081510218, "pass@16": 0.9616873600439593, "pass@32": 0.9704475586213182, "pass@64": 0.9780345237152435, "pass@128": 0.982089552238806 }, "2": { "pass@1": 0.9999776785714286, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9995444606413995, "pass@2": 0.9999508591400565, "pass@4": 0.9999993495710995, "pass@8": 0.9999999999504271, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9823419225146199, "pass@2": 0.9854621379564396, "pass@4": 0.9880769901021142, "pass@8": 0.9902890222952649, "pass@16": 0.9922729187814656, "pass@32": 0.9941339389256002, "pass@64": 0.9959935170640539, "pass@128": 0.9970760233918129 }, "5": { "pass@1": 0.9192095588235294, "pass@2": 0.9368833950903197, "pass@4": 0.9486726628840507, "pass@8": 0.957187841731995, "pass@16": 0.963640097381688, "pass@32": 0.9699201940179724, "pass@64": 0.9774210870046359, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9850897082018928, "pass@2": 0.9905075573163766, "pass@4": 0.9929501176311952, "pass@8": 0.9936089564592303, "pass@16": 0.9936894407460112, "pass@32": 0.9936908516082206, "pass@64": 0.9936908517350158, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.9513202136075949, "pass@2": 0.9653273105003486, "pass@4": 0.9741560914458472, "pass@8": 0.9809422762648561, "pass@16": 0.9865958409360319, "pass@32": 0.9910348161676964, "pass@64": 0.9942570067837525, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9035285285285285, "pass@2": 0.9250335473268545, "pass@4": 0.9401216576531538, "pass@8": 0.9530895360198267, "pass@16": 0.9647251870543512, "pass@32": 0.9739953866287852, "pass@64": 0.9797902253746495, "pass@128": 0.9819819819819819 }, "9": { "pass@1": 0.8857013665594855, "pass@2": 0.9113160493202016, "pass@4": 0.9282795167083213, "pass@8": 0.9412904877973327, "pass@16": 0.9506791980302753, "pass@32": 0.9574335414534275, "pass@64": 0.962500672127828, "pass@128": 0.9678456591639871 }, "11": { "pass@1": 0.8270337301587302, "pass@2": 0.8687472659667539, "pass@4": 0.8942530689616175, "pass@8": 0.913000742277685, "pass@16": 0.9287522778885744, "pass@32": 0.9423010463849473, "pass@64": 0.9536259185529427, "pass@128": 0.9619047619047619 }, "12": { "pass@1": 0.7741303516819572, "pass@2": 0.844897917720147, "pass@4": 0.8911836662619007, "pass@8": 0.9227577810625305, "pass@16": 0.9434655328339684, "pass@32": 0.9580731530824094, "pass@64": 0.9686499242431075, "pass@128": 0.9755351681957186 }, "13": { "pass@1": 0.5628134645061729, "pass@2": 0.6580126919898899, "pass@4": 0.7298308666277817, "pass@8": 0.7849102761957668, "pass@16": 0.8272916207877943, "pass@32": 0.8597796191468667, "pass@64": 0.8834876179969318, "pass@128": 0.9012345679012346 }, "14": { "pass@1": 0.42303412462908013, "pass@2": 0.5254594879553262, "pass@4": 0.612024243167675, "pass@8": 0.6795870113254974, "pass@16": 0.7299641829436971, "pass@32": 0.769675569678991, "pass@64": 0.802551172129445, "pass@128": 0.8308605341246291 }, "15": { "pass@1": 0.2721113040123457, "pass@2": 0.3505919181977249, "pass@4": 0.42352521617668165, "pass@8": 0.49062896925691024, "pass@16": 0.5509084623861222, "pass@32": 0.6026686433028146, "pass@64": 0.648546132335972, "pass@128": 0.6975308641975309 }, "16": { "pass@1": 0.1982853982300885, "pass@2": 0.25889056802081145, "pass@4": 0.3179625279140993, "pass@8": 0.3723615276522692, "pass@16": 0.41992990025116966, "pass@32": 0.46214547052895755, "pass@64": 0.5066410316005072, "pass@128": 0.5634218289085545 }, "17": { "pass@1": 0.17211031626506024, "pass@2": 0.2284550356939569, "pass@4": 0.2824050603426078, "pass@8": 0.3306790083684093, "pass@16": 0.3695724116618554, "pass@32": 0.40174054863361564, "pass@64": 0.4304838558018472, "pass@128": 0.4578313253012048 }, "18": { "pass@1": 0.18351814516129034, "pass@2": 0.2443234855219709, "pass@4": 0.3019939807927236, "pass@8": 0.35022866682181714, "pass@16": 0.38934737936610586, "pass@32": 0.4241763498467773, "pass@64": 0.457008988079855, "pass@128": 0.4870967741935484 }, "19": { "pass@1": 0.15853946374622357, "pass@2": 0.21041210719128384, "pass@4": 0.26191707786904267, "pass@8": 0.3102665481682076, "pass@16": 0.35281055902223907, "pass@32": 0.38815261030610226, "pass@64": 0.4175298687617645, "pass@128": 0.44410876132930516 }, "20": { "pass@1": 0.15064283033033032, "pass@2": 0.20356281478328725, "pass@4": 0.2574440165700009, "pass@8": 0.30892386841405955, "pass@16": 0.35627459223670677, "pass@32": 0.3974007494071951, "pass@64": 0.4299470015533774, "pass@128": 0.45345345345345345 } } } } } ]