DatPySci's picture
Add files using upload-large-folder tool
53f581b verified
Raw History Blame Contribute Delete
21.5 kB
[
{
"checkpoint": "SDAR-100M-block8-epoch4-op7-10",
"id_acc": 0.994,
"ood_acc": 0.48264140625,
"total_acc": 0.7248638980263158,
"id_avg_loss": 0.0,
"ood_avg_loss": 0.0,
"total_avg_loss": 0.0,
"id_avg_resp_len": 200.33833854166667,
"ood_avg_resp_len": 336.96297421875,
"count_id": 1152000,
"count_ood": 1280000,
"count_total": 2432000,
"template_metrics": {
"crazy_zootopia": {
"count": 809728,
"correct": 587026,
"answer_accuracy": 0.7249669024660133,
"avg_response_len": 259.7559662009959,
"resp_tokens_sum": 210331679,
"pass_at_k": {
"pass@1": 0.7249669024660133,
"pass@2": 0.7604557128622972,
"pass@4": 0.789882500277682,
"pass@8": 0.8140953337410525,
"pass@16": 0.8340288729588218,
"pass@32": 0.85067531035173,
"pass@64": 0.8654940988509915,
"pass@128": 0.879386658235852
},
"per_op_pass_at_k": {
"10": {
"pass@1": 0.9625737463126843,
"pass@2": 0.9730252537569976,
"pass@4": 0.9790491030988384,
"pass@8": 0.9823822861395556,
"pass@16": 0.9851782276001185,
"pass@32": 0.9887806819385732,
"pass@64": 0.9929144734452152,
"pass@128": 0.9970501474926253
},
"2": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"3": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"4": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"5": {
"pass@1": 0.9999503968253968,
"pass@2": 0.999999609423822,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"6": {
"pass@1": 0.9978246631736527,
"pass@2": 0.9984241654486302,
"pass@4": 0.9991805660332878,
"pass@8": 0.9997867287338961,
"pass@16": 0.9999877852663126,
"pass@32": 0.9999999824528186,
"pass@64": 0.9999999999999998,
"pass@128": 1.0
},
"7": {
"pass@1": 0.9999537721893491,
"pass@2": 0.999999636001491,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"8": {
"pass@1": 0.995292467948718,
"pass@2": 0.9984723589238845,
"pass@4": 0.9997657519372578,
"pass@8": 0.9999935417446474,
"pass@16": 0.9999999969902831,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"9": {
"pass@1": 0.9949603873239437,
"pass@2": 0.99670414217589,
"pass@4": 0.9974577013260665,
"pass@8": 0.997830234782101,
"pass@16": 0.9983600236132614,
"pass@32": 0.9991228314770513,
"pass@64": 0.9998322280137517,
"pass@128": 1.0
},
"11": {
"pass@1": 0.9385844546742209,
"pass@2": 0.9551942294394504,
"pass@4": 0.967026604677248,
"pass@8": 0.9754982084288127,
"pass@16": 0.9808253815147637,
"pass@32": 0.9850623210750536,
"pass@64": 0.9884474194474718,
"pass@128": 0.9915014164305949
},
"12": {
"pass@1": 0.8624426200564972,
"pass@2": 0.9017520519151211,
"pass@4": 0.9232137743904894,
"pass@8": 0.9364155828181312,
"pass@16": 0.9446959074267556,
"pass@32": 0.9501805451185891,
"pass@64": 0.9558795802602567,
"pass@128": 0.963276836158192
},
"13": {
"pass@1": 0.734203565830721,
"pass@2": 0.801667443166391,
"pass@4": 0.8452445452547275,
"pass@8": 0.8750841253152706,
"pass@16": 0.8976730768256509,
"pass@32": 0.916616734947578,
"pass@64": 0.932356215696857,
"pass@128": 0.9404388714733543
},
"14": {
"pass@1": 0.6615313473520249,
"pass@2": 0.741312673240611,
"pass@4": 0.7987274321434115,
"pass@8": 0.8374862263892824,
"pass@16": 0.8636665344499674,
"pass@32": 0.883744442134681,
"pass@64": 0.8996549245285982,
"pass@128": 0.9127725856697819
},
"15": {
"pass@1": 0.5426752167630058,
"pass@2": 0.6423150829502521,
"pass@4": 0.7232531651571007,
"pass@8": 0.7860465523211205,
"pass@16": 0.8328430235526624,
"pass@32": 0.8664772705101282,
"pass@64": 0.8907010774062084,
"pass@128": 0.9104046242774566
},
"16": {
"pass@1": 0.3149193548387097,
"pass@2": 0.39902527305054625,
"pass@4": 0.4697059656857409,
"pass@8": 0.5302840910372171,
"pass@16": 0.5838085174066465,
"pass@32": 0.6282172083334884,
"pass@64": 0.6665010235625493,
"pass@128": 0.7032258064516129
},
"17": {
"pass@1": 0.25410946531791906,
"pass@2": 0.32993918119339133,
"pass@4": 0.40069865038546476,
"pass@8": 0.46448555649405504,
"pass@16": 0.5182130045168546,
"pass@32": 0.5623529995882736,
"pass@64": 0.6040674100873197,
"pass@128": 0.6473988439306358
},
"18": {
"pass@1": 0.20060267857142858,
"pass@2": 0.2679288526434197,
"pass@4": 0.3344746652204189,
"pass@8": 0.39710824121270083,
"pass@16": 0.4566949168331932,
"pass@32": 0.5120978334164232,
"pass@64": 0.5656579621163826,
"pass@128": 0.6228571428571429
},
"19": {
"pass@1": 0.18132976726726727,
"pass@2": 0.2433429985103213,
"pass@4": 0.30306926977596277,
"pass@8": 0.3572313687285545,
"pass@16": 0.40418699091341775,
"pass@32": 0.44416817763081634,
"pass@64": 0.47951343317052403,
"pass@128": 0.5135135135135135
},
"20": {
"pass@1": 0.16441441441441443,
"pass@2": 0.22810002423683523,
"pass@4": 0.29267682502525005,
"pass@8": 0.35116514670673493,
"pass@16": 0.40103786340810277,
"pass@32": 0.4445894691442985,
"pass@64": 0.48528400560602686,
"pass@128": 0.5195195195195195
}
}
},
"teachers_in_school": {
"count": 821120,
"correct": 594313,
"answer_accuracy": 0.7237833690568979,
"avg_response_len": 268.6203125,
"resp_tokens_sum": 220569511,
"pass_at_k": {
"pass@1": 0.7237833690568979,
"pass@2": 0.7596792895894745,
"pass@4": 0.7891490582919107,
"pass@8": 0.8134931073775019,
"pass@16": 0.8341315095635516,
"pass@32": 0.8522173762624881,
"pass@64": 0.8682187241550315,
"pass@128": 0.8819953234606391
},
"per_op_pass_at_k": {
"10": {
"pass@1": 0.9544190950920245,
"pass@2": 0.9655979783585334,
"pass@4": 0.9755531901495438,
"pass@8": 0.9849001130563421,
"pass@16": 0.9914179568700926,
"pass@32": 0.9953896244003085,
"pass@64": 0.9980909065259532,
"pass@128": 1.0
},
"2": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"3": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"4": {
"pass@1": 0.999791974852071,
"pass@2": 0.9999868960536739,
"pass@4": 0.9999999650561431,
"pass@8": 0.9999999999999815,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"5": {
"pass@1": 0.9999094202898551,
"pass@2": 0.9999992867739359,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"6": {
"pass@1": 0.9998433022922636,
"pass@2": 0.9999925969586897,
"pass@4": 0.9999999905993125,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"7": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"8": {
"pass@1": 0.9957086267605634,
"pass@2": 0.9968146972385495,
"pass@4": 0.9972236687843596,
"pass@8": 0.997358535350474,
"pass@16": 0.9975352112222411,
"pass@32": 0.997887323943662,
"pass@64": 0.9985915492957746,
"pass@128": 1.0
},
"9": {
"pass@1": 0.9925383607784432,
"pass@2": 0.994316239568108,
"pass@4": 0.9953136485746168,
"pass@8": 0.9962981752872685,
"pass@16": 0.9975205508838629,
"pass@32": 0.99868979993864,
"pass@64": 0.9996344033069231,
"pass@128": 1.0
},
"11": {
"pass@1": 0.9485363328313253,
"pass@2": 0.9614666836637891,
"pass@4": 0.9690401578191278,
"pass@8": 0.9745595660303529,
"pass@16": 0.9776023172016188,
"pass@32": 0.979402643004502,
"pass@64": 0.981192486457403,
"pass@128": 0.9819277108433735
},
"12": {
"pass@1": 0.8519298589341693,
"pass@2": 0.882412744057463,
"pass@4": 0.8987941407402442,
"pass@8": 0.9122516853738211,
"pass@16": 0.9248490185450918,
"pass@32": 0.9363708212156535,
"pass@64": 0.9462338759817461,
"pass@128": 0.9529780564263323
},
"13": {
"pass@1": 0.6947435224089635,
"pass@2": 0.777705521184852,
"pass@4": 0.8304399998844679,
"pass@8": 0.8625365131047817,
"pass@16": 0.8834984921646472,
"pass@32": 0.8995238580003326,
"pass@64": 0.9129394155831055,
"pass@128": 0.927170868347339
},
"14": {
"pass@1": 0.6441429093567251,
"pass@2": 0.7267150302758209,
"pass@4": 0.7861907489085788,
"pass@8": 0.8273056582111475,
"pass@16": 0.8566846397040585,
"pass@32": 0.8804814970414178,
"pass@64": 0.9011153985940141,
"pass@128": 0.9181286549707602
},
"15": {
"pass@1": 0.5651751893939394,
"pass@2": 0.6739184413027917,
"pass@4": 0.7489947995136972,
"pass@8": 0.7974960489060056,
"pass@16": 0.8302977072796687,
"pass@32": 0.8556934809010615,
"pass@64": 0.876922393623797,
"pass@128": 0.8939393939393939
},
"16": {
"pass@1": 0.3212250712250712,
"pass@2": 0.3989094712519908,
"pass@4": 0.46772314010962307,
"pass@8": 0.5297939095149375,
"pass@16": 0.5862086467853515,
"pass@32": 0.6380627166781673,
"pass@64": 0.6854883475998351,
"pass@128": 0.7264957264957265
},
"17": {
"pass@1": 0.2698466614906832,
"pass@2": 0.35929246955543587,
"pass@4": 0.4414016571880376,
"pass@8": 0.5100927321509832,
"pass@16": 0.5673176419200605,
"pass@32": 0.6157075990502674,
"pass@64": 0.6567645873486623,
"pass@128": 0.6925465838509317
},
"18": {
"pass@1": 0.20048253676470587,
"pass@2": 0.2711053294349235,
"pass@4": 0.34233683252828667,
"pass@8": 0.4105063409952905,
"pass@16": 0.4747878989634802,
"pass@32": 0.5336599706135036,
"pass@64": 0.5865359016158412,
"pass@128": 0.6352941176470588
},
"19": {
"pass@1": 0.15308779761904762,
"pass@2": 0.20727072201912244,
"pass@4": 0.2626874246745942,
"pass@8": 0.3177729661617527,
"pass@16": 0.3739255500583787,
"pass@32": 0.42984186349128706,
"pass@64": 0.482475527616294,
"pass@128": 0.5238095238095238
},
"20": {
"pass@1": 0.14775916916167664,
"pass@2": 0.20598456138903293,
"pass@4": 0.26647546779332226,
"pass@8": 0.32486153243092897,
"pass@16": 0.37714961566942967,
"pass@32": 0.4223905208474426,
"pass@64": 0.46155466688637004,
"pass@128": 0.49700598802395207
}
}
},
"movie_festival_awards": {
"count": 801152,
"correct": 581530,
"answer_accuracy": 0.725867251158332,
"avg_response_len": 288.5859150323534,
"resp_tokens_sum": 231201183,
"pass_at_k": {
"pass@1": 0.725867251158332,
"pass@2": 0.7624292475528114,
"pass@4": 0.792774769656026,
"pass@8": 0.8177036241468999,
"pass@16": 0.8379595685217712,
"pass@32": 0.8546269957433247,
"pass@64": 0.8689409068065663,
"pass@128": 0.8819300207700911
},
"per_op_pass_at_k": {
"10": {
"pass@1": 0.9559235074626866,
"pass@2": 0.9656980843812436,
"pass@4": 0.9746656671647387,
"pass@8": 0.9820459603607242,
"pass@16": 0.9863468733654595,
"pass@32": 0.9886201472854002,
"pass@64": 0.9902182679940894,
"pass@128": 0.991044776119403
},
"2": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"3": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"4": {
"pass@1": 1.0,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"5": {
"pass@1": 0.9996783088235294,
"pass@2": 0.9999815452755905,
"pass@4": 0.9999999418271246,
"pass@8": 0.9999999999999074,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"6": {
"pass@1": 0.999482452681388,
"pass@2": 0.9999336328522814,
"pass@4": 0.9999988538498933,
"pass@8": 0.9999999998332318,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"7": {
"pass@1": 0.9999752768987342,
"pass@2": 1.0,
"pass@4": 1.0,
"pass@8": 1.0,
"pass@16": 1.0,
"pass@32": 1.0,
"pass@64": 1.0,
"pass@128": 1.0
},
"8": {
"pass@1": 0.995823948948949,
"pass@2": 0.9981944296658863,
"pass@4": 0.9995453380827396,
"pass@8": 0.9999589276607602,
"pass@16": 0.9999996513465985,
"pass@32": 0.9999999999945224,
"pass@64": 1.0,
"pass@128": 1.0
},
"9": {
"pass@1": 0.9941217845659164,
"pass@2": 0.9957378092006989,
"pass@4": 0.997233856820952,
"pass@8": 0.998791655494803,
"pass@16": 0.9997618486572567,
"pass@32": 0.9999916332542949,
"pass@64": 0.9999999987229116,
"pass@128": 1.0
},
"11": {
"pass@1": 0.9454117063492063,
"pass@2": 0.9644895950506187,
"pass@4": 0.9755416611614021,
"pass@8": 0.9823663944356187,
"pass@16": 0.9867982914476112,
"pass@32": 0.9892215283268877,
"pass@64": 0.9902824942007619,
"pass@128": 0.9904761904761905
},
"12": {
"pass@1": 0.8600917431192661,
"pass@2": 0.9047514237039184,
"pass@4": 0.930534307351489,
"pass@8": 0.9465814909296013,
"pass@16": 0.9574575537927447,
"pass@32": 0.9651714867408759,
"pass@64": 0.9696943208397095,
"pass@128": 0.9724770642201835
},
"13": {
"pass@1": 0.7000144675925926,
"pass@2": 0.7741369568387282,
"pass@4": 0.8237057225485699,
"pass@8": 0.8586535597961961,
"pass@16": 0.8835374150406778,
"pass@32": 0.9020236050675741,
"pass@64": 0.9171006250087037,
"pass@128": 0.9320987654320988
},
"14": {
"pass@1": 0.6755841988130564,
"pass@2": 0.7566729070772676,
"pass@4": 0.8109844756052377,
"pass@8": 0.846805617430967,
"pass@16": 0.8723752791841834,
"pass@32": 0.8926627247553403,
"pass@64": 0.9091311345903748,
"pass@128": 0.9198813056379822
},
"15": {
"pass@1": 0.5425829475308642,
"pass@2": 0.639068879410907,
"pass@4": 0.7103934736745863,
"pass@8": 0.7632807597637843,
"pass@16": 0.803510714937235,
"pass@32": 0.8338604893791952,
"pass@64": 0.8571668944592924,
"pass@128": 0.8796296296296297
},
"16": {
"pass@1": 0.2883941740412979,
"pass@2": 0.379378324390867,
"pass@4": 0.4647851607044694,
"pass@8": 0.5395945603525556,
"pass@16": 0.6013971674931184,
"pass@32": 0.6518804161642361,
"pass@64": 0.6972943624071537,
"pass@128": 0.7404129793510325
},
"17": {
"pass@1": 0.24458772590361447,
"pass@2": 0.3179293206289724,
"pass@4": 0.3873656151603038,
"pass@8": 0.4529060893572603,
"pass@16": 0.5148842285661349,
"pass@32": 0.5732756836389475,
"pass@64": 0.6286638481690365,
"pass@128": 0.6807228915662651
},
"18": {
"pass@1": 0.24243951612903225,
"pass@2": 0.30982981965963907,
"pass@4": 0.37429037156645767,
"pass@8": 0.4342826991634681,
"pass@16": 0.4870580721094836,
"pass@32": 0.5307268008704006,
"pass@64": 0.5660957126821192,
"pass@128": 0.5967741935483871
},
"19": {
"pass@1": 0.1842428247734139,
"pass@2": 0.25546765349572986,
"pass@4": 0.3234768755302869,
"pass@8": 0.3800818937670737,
"pass@16": 0.42391733987699637,
"pass@32": 0.4613442746078676,
"pass@64": 0.49752024916745513,
"pass@128": 0.5347432024169184
},
"20": {
"pass@1": 0.150243993993994,
"pass@2": 0.2137408077368706,
"pass@4": 0.2801131426702293,
"pass@8": 0.3418353258288631,
"pass@16": 0.3957745566306108,
"pass@32": 0.44117214844812946,
"pass@64": 0.4788019796245884,
"pass@128": 0.5105105105105106
}
}
}
}
}
]