[ { "checkpoint": "SDAR-100M-block8-epoch4-op7-10", "id_acc": 0.994, "ood_acc": 0.48264140625, "total_acc": 0.7248638980263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.33833854166667, "ood_avg_resp_len": 336.96297421875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 587026, "answer_accuracy": 0.7249669024660133, "avg_response_len": 259.7559662009959, "resp_tokens_sum": 210331679, "pass_at_k": { "pass@1": 0.7249669024660133, "pass@2": 0.7604557128622972, "pass@4": 0.789882500277682, "pass@8": 0.8140953337410525, "pass@16": 0.8340288729588218, "pass@32": 0.85067531035173, "pass@64": 0.8654940988509915, "pass@128": 0.879386658235852 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9625737463126843, "pass@2": 0.9730252537569976, "pass@4": 0.9790491030988384, "pass@8": 0.9823822861395556, "pass@16": 0.9851782276001185, "pass@32": 0.9887806819385732, "pass@64": 0.9929144734452152, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9999503968253968, "pass@2": 0.999999609423822, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9978246631736527, "pass@2": 0.9984241654486302, "pass@4": 0.9991805660332878, "pass@8": 0.9997867287338961, "pass@16": 0.9999877852663126, "pass@32": 0.9999999824528186, "pass@64": 0.9999999999999998, "pass@128": 1.0 }, "7": { "pass@1": 0.9999537721893491, "pass@2": 0.999999636001491, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.995292467948718, "pass@2": 0.9984723589238845, "pass@4": 0.9997657519372578, "pass@8": 0.9999935417446474, "pass@16": 0.9999999969902831, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9949603873239437, "pass@2": 0.99670414217589, "pass@4": 0.9974577013260665, "pass@8": 0.997830234782101, "pass@16": 0.9983600236132614, "pass@32": 0.9991228314770513, "pass@64": 0.9998322280137517, "pass@128": 1.0 }, "11": { "pass@1": 0.9385844546742209, "pass@2": 0.9551942294394504, "pass@4": 0.967026604677248, "pass@8": 0.9754982084288127, "pass@16": 0.9808253815147637, "pass@32": 0.9850623210750536, "pass@64": 0.9884474194474718, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.8624426200564972, "pass@2": 0.9017520519151211, "pass@4": 0.9232137743904894, "pass@8": 0.9364155828181312, "pass@16": 0.9446959074267556, "pass@32": 0.9501805451185891, "pass@64": 0.9558795802602567, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.734203565830721, "pass@2": 0.801667443166391, "pass@4": 0.8452445452547275, "pass@8": 0.8750841253152706, "pass@16": 0.8976730768256509, "pass@32": 0.916616734947578, "pass@64": 0.932356215696857, "pass@128": 0.9404388714733543 }, "14": { "pass@1": 0.6615313473520249, "pass@2": 0.741312673240611, "pass@4": 0.7987274321434115, "pass@8": 0.8374862263892824, "pass@16": 0.8636665344499674, "pass@32": 0.883744442134681, "pass@64": 0.8996549245285982, "pass@128": 0.9127725856697819 }, "15": { "pass@1": 0.5426752167630058, "pass@2": 0.6423150829502521, "pass@4": 0.7232531651571007, "pass@8": 0.7860465523211205, "pass@16": 0.8328430235526624, "pass@32": 0.8664772705101282, "pass@64": 0.8907010774062084, "pass@128": 0.9104046242774566 }, "16": { "pass@1": 0.3149193548387097, "pass@2": 0.39902527305054625, "pass@4": 0.4697059656857409, "pass@8": 0.5302840910372171, "pass@16": 0.5838085174066465, "pass@32": 0.6282172083334884, "pass@64": 0.6665010235625493, "pass@128": 0.7032258064516129 }, "17": { "pass@1": 0.25410946531791906, "pass@2": 0.32993918119339133, "pass@4": 0.40069865038546476, "pass@8": 0.46448555649405504, "pass@16": 0.5182130045168546, "pass@32": 0.5623529995882736, "pass@64": 0.6040674100873197, "pass@128": 0.6473988439306358 }, "18": { "pass@1": 0.20060267857142858, "pass@2": 0.2679288526434197, "pass@4": 0.3344746652204189, "pass@8": 0.39710824121270083, "pass@16": 0.4566949168331932, "pass@32": 0.5120978334164232, "pass@64": 0.5656579621163826, "pass@128": 0.6228571428571429 }, "19": { "pass@1": 0.18132976726726727, "pass@2": 0.2433429985103213, "pass@4": 0.30306926977596277, "pass@8": 0.3572313687285545, "pass@16": 0.40418699091341775, "pass@32": 0.44416817763081634, "pass@64": 0.47951343317052403, "pass@128": 0.5135135135135135 }, "20": { "pass@1": 0.16441441441441443, "pass@2": 0.22810002423683523, "pass@4": 0.29267682502525005, "pass@8": 0.35116514670673493, "pass@16": 0.40103786340810277, "pass@32": 0.4445894691442985, "pass@64": 0.48528400560602686, "pass@128": 0.5195195195195195 } } }, "teachers_in_school": { "count": 821120, "correct": 594313, "answer_accuracy": 0.7237833690568979, "avg_response_len": 268.6203125, "resp_tokens_sum": 220569511, "pass_at_k": { "pass@1": 0.7237833690568979, "pass@2": 0.7596792895894745, "pass@4": 0.7891490582919107, "pass@8": 0.8134931073775019, "pass@16": 0.8341315095635516, "pass@32": 0.8522173762624881, "pass@64": 0.8682187241550315, "pass@128": 0.8819953234606391 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9544190950920245, "pass@2": 0.9655979783585334, "pass@4": 0.9755531901495438, "pass@8": 0.9849001130563421, "pass@16": 0.9914179568700926, "pass@32": 0.9953896244003085, "pass@64": 0.9980909065259532, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999791974852071, "pass@2": 0.9999868960536739, "pass@4": 0.9999999650561431, "pass@8": 0.9999999999999815, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9999094202898551, "pass@2": 0.9999992867739359, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9998433022922636, "pass@2": 0.9999925969586897, "pass@4": 0.9999999905993125, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9957086267605634, "pass@2": 0.9968146972385495, "pass@4": 0.9972236687843596, "pass@8": 0.997358535350474, "pass@16": 0.9975352112222411, "pass@32": 0.997887323943662, "pass@64": 0.9985915492957746, "pass@128": 1.0 }, "9": { "pass@1": 0.9925383607784432, "pass@2": 0.994316239568108, "pass@4": 0.9953136485746168, "pass@8": 0.9962981752872685, "pass@16": 0.9975205508838629, "pass@32": 0.99868979993864, "pass@64": 0.9996344033069231, "pass@128": 1.0 }, "11": { "pass@1": 0.9485363328313253, "pass@2": 0.9614666836637891, "pass@4": 0.9690401578191278, "pass@8": 0.9745595660303529, "pass@16": 0.9776023172016188, "pass@32": 0.979402643004502, "pass@64": 0.981192486457403, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8519298589341693, "pass@2": 0.882412744057463, "pass@4": 0.8987941407402442, "pass@8": 0.9122516853738211, "pass@16": 0.9248490185450918, "pass@32": 0.9363708212156535, "pass@64": 0.9462338759817461, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.6947435224089635, "pass@2": 0.777705521184852, "pass@4": 0.8304399998844679, "pass@8": 0.8625365131047817, "pass@16": 0.8834984921646472, "pass@32": 0.8995238580003326, "pass@64": 0.9129394155831055, "pass@128": 0.927170868347339 }, "14": { "pass@1": 0.6441429093567251, "pass@2": 0.7267150302758209, "pass@4": 0.7861907489085788, "pass@8": 0.8273056582111475, "pass@16": 0.8566846397040585, "pass@32": 0.8804814970414178, "pass@64": 0.9011153985940141, "pass@128": 0.9181286549707602 }, "15": { "pass@1": 0.5651751893939394, "pass@2": 0.6739184413027917, "pass@4": 0.7489947995136972, "pass@8": 0.7974960489060056, "pass@16": 0.8302977072796687, "pass@32": 0.8556934809010615, "pass@64": 0.876922393623797, "pass@128": 0.8939393939393939 }, "16": { "pass@1": 0.3212250712250712, "pass@2": 0.3989094712519908, "pass@4": 0.46772314010962307, "pass@8": 0.5297939095149375, "pass@16": 0.5862086467853515, "pass@32": 0.6380627166781673, "pass@64": 0.6854883475998351, "pass@128": 0.7264957264957265 }, "17": { "pass@1": 0.2698466614906832, "pass@2": 0.35929246955543587, "pass@4": 0.4414016571880376, "pass@8": 0.5100927321509832, "pass@16": 0.5673176419200605, "pass@32": 0.6157075990502674, "pass@64": 0.6567645873486623, "pass@128": 0.6925465838509317 }, "18": { "pass@1": 0.20048253676470587, "pass@2": 0.2711053294349235, "pass@4": 0.34233683252828667, "pass@8": 0.4105063409952905, "pass@16": 0.4747878989634802, "pass@32": 0.5336599706135036, "pass@64": 0.5865359016158412, "pass@128": 0.6352941176470588 }, "19": { "pass@1": 0.15308779761904762, "pass@2": 0.20727072201912244, "pass@4": 0.2626874246745942, "pass@8": 0.3177729661617527, "pass@16": 0.3739255500583787, "pass@32": 0.42984186349128706, "pass@64": 0.482475527616294, "pass@128": 0.5238095238095238 }, "20": { "pass@1": 0.14775916916167664, "pass@2": 0.20598456138903293, "pass@4": 0.26647546779332226, "pass@8": 0.32486153243092897, "pass@16": 0.37714961566942967, "pass@32": 0.4223905208474426, "pass@64": 0.46155466688637004, "pass@128": 0.49700598802395207 } } }, "movie_festival_awards": { "count": 801152, "correct": 581530, "answer_accuracy": 0.725867251158332, "avg_response_len": 288.5859150323534, "resp_tokens_sum": 231201183, "pass_at_k": { "pass@1": 0.725867251158332, "pass@2": 0.7624292475528114, "pass@4": 0.792774769656026, "pass@8": 0.8177036241468999, "pass@16": 0.8379595685217712, "pass@32": 0.8546269957433247, "pass@64": 0.8689409068065663, "pass@128": 0.8819300207700911 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9559235074626866, "pass@2": 0.9656980843812436, "pass@4": 0.9746656671647387, "pass@8": 0.9820459603607242, "pass@16": 0.9863468733654595, "pass@32": 0.9886201472854002, "pass@64": 0.9902182679940894, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9996783088235294, "pass@2": 0.9999815452755905, "pass@4": 0.9999999418271246, "pass@8": 0.9999999999999074, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.999482452681388, "pass@2": 0.9999336328522814, "pass@4": 0.9999988538498933, "pass@8": 0.9999999998332318, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9999752768987342, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.995823948948949, "pass@2": 0.9981944296658863, "pass@4": 0.9995453380827396, "pass@8": 0.9999589276607602, "pass@16": 0.9999996513465985, "pass@32": 0.9999999999945224, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9941217845659164, "pass@2": 0.9957378092006989, "pass@4": 0.997233856820952, "pass@8": 0.998791655494803, "pass@16": 0.9997618486572567, "pass@32": 0.9999916332542949, "pass@64": 0.9999999987229116, "pass@128": 1.0 }, "11": { "pass@1": 0.9454117063492063, "pass@2": 0.9644895950506187, "pass@4": 0.9755416611614021, "pass@8": 0.9823663944356187, "pass@16": 0.9867982914476112, "pass@32": 0.9892215283268877, "pass@64": 0.9902824942007619, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.8600917431192661, "pass@2": 0.9047514237039184, "pass@4": 0.930534307351489, "pass@8": 0.9465814909296013, "pass@16": 0.9574575537927447, "pass@32": 0.9651714867408759, "pass@64": 0.9696943208397095, "pass@128": 0.9724770642201835 }, "13": { "pass@1": 0.7000144675925926, "pass@2": 0.7741369568387282, "pass@4": 0.8237057225485699, "pass@8": 0.8586535597961961, "pass@16": 0.8835374150406778, "pass@32": 0.9020236050675741, "pass@64": 0.9171006250087037, "pass@128": 0.9320987654320988 }, "14": { "pass@1": 0.6755841988130564, "pass@2": 0.7566729070772676, "pass@4": 0.8109844756052377, "pass@8": 0.846805617430967, "pass@16": 0.8723752791841834, "pass@32": 0.8926627247553403, "pass@64": 0.9091311345903748, "pass@128": 0.9198813056379822 }, "15": { "pass@1": 0.5425829475308642, "pass@2": 0.639068879410907, "pass@4": 0.7103934736745863, "pass@8": 0.7632807597637843, "pass@16": 0.803510714937235, "pass@32": 0.8338604893791952, "pass@64": 0.8571668944592924, "pass@128": 0.8796296296296297 }, "16": { "pass@1": 0.2883941740412979, "pass@2": 0.379378324390867, "pass@4": 0.4647851607044694, "pass@8": 0.5395945603525556, "pass@16": 0.6013971674931184, "pass@32": 0.6518804161642361, "pass@64": 0.6972943624071537, "pass@128": 0.7404129793510325 }, "17": { "pass@1": 0.24458772590361447, "pass@2": 0.3179293206289724, "pass@4": 0.3873656151603038, "pass@8": 0.4529060893572603, "pass@16": 0.5148842285661349, "pass@32": 0.5732756836389475, "pass@64": 0.6286638481690365, "pass@128": 0.6807228915662651 }, "18": { "pass@1": 0.24243951612903225, "pass@2": 0.30982981965963907, "pass@4": 0.37429037156645767, "pass@8": 0.4342826991634681, "pass@16": 0.4870580721094836, "pass@32": 0.5307268008704006, "pass@64": 0.5660957126821192, "pass@128": 0.5967741935483871 }, "19": { "pass@1": 0.1842428247734139, "pass@2": 0.25546765349572986, "pass@4": 0.3234768755302869, "pass@8": 0.3800818937670737, "pass@16": 0.42391733987699637, "pass@32": 0.4613442746078676, "pass@64": 0.49752024916745513, "pass@128": 0.5347432024169184 }, "20": { "pass@1": 0.150243993993994, "pass@2": 0.2137408077368706, "pass@4": 0.2801131426702293, "pass@8": 0.3418353258288631, "pass@16": 0.3957745566306108, "pass@32": 0.44117214844812946, "pass@64": 0.4788019796245884, "pass@128": 0.5105105105105106 } } } } } ]