[ { "checkpoint": "checkpoint-42828", "id_acc": 0.5884739583333334, "ood_acc": 0.1697921875, "total_acc": 0.3681151315789474, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.50156336805554, "ood_avg_resp_len": 322.145346875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 297510, "answer_accuracy": 0.3674196767309516, "avg_response_len": 249.58559417483403, "resp_tokens_sum": 202096444, "pass_at_k": { "pass@1": 0.3674196767309516, "pass@2": 0.4686072282307062, "pass@4": 0.5724512478394008, "pass@8": 0.6657046444352267, "pass@16": 0.7403374473959198, "pass@32": 0.7965784096114769, "pass@64": 0.8389524590454404, "pass@128": 0.8721150806196649 }, "per_op_pass_at_k": { "10": { "pass@1": 0.3160951327433628, "pass@2": 0.45978539532669027, "pass@4": 0.6087440955942451, "pass@8": 0.7389912618384603, "pass@16": 0.8347761227732956, "pass@32": 0.8974145781361296, "pass@64": 0.9361195557565894, "pass@128": 0.9587020648967551 }, "2": { "pass@1": 0.9389846183800623, "pass@2": 0.9547148275566021, "pass@4": 0.964830180398011, "pass@8": 0.9716501249659655, "pass@16": 0.9762703677512582, "pass@32": 0.9794593284358751, "pass@64": 0.9826803268884213, "pass@128": 0.9875389408099688 }, "3": { "pass@1": 0.93623375382263, "pass@2": 0.9602559356112597, "pass@4": 0.971118965347221, "pass@8": 0.9775815501353607, "pass@16": 0.9825540775856214, "pass@32": 0.9868925189336957, "pass@64": 0.9907274512321347, "pass@128": 0.9938837920489296 }, "4": { "pass@1": 0.782666015625, "pass@2": 0.8574395607775591, "pass@4": 0.9108361469464749, "pass@8": 0.9460693707480754, "pass@16": 0.9668682851949002, "pass@32": 0.9775023168825377, "pass@64": 0.9821243598725908, "pass@128": 0.984375 }, "5": { "pass@1": 0.6209325396825397, "pass@2": 0.7310359642544682, "pass@4": 0.8163355446640592, "pass@8": 0.8783539103209079, "pass@16": 0.922046305462945, "pass@32": 0.9516874532916285, "pass@64": 0.971409019971419, "pass@128": 0.9841269841269841 }, "6": { "pass@1": 0.5845808383233533, "pass@2": 0.723798343274082, "pass@4": 0.834097648067195, "pass@8": 0.908153197796081, "pass@16": 0.9503107359238322, "pass@32": 0.9694990410164247, "pass@64": 0.9774287469166411, "pass@128": 0.9820359281437125 }, "7": { "pass@1": 0.45092917899408286, "pass@2": 0.596248121767693, "pass@4": 0.7315876857848386, "pass@8": 0.8403357342652344, "pass@16": 0.9115535229290597, "pass@32": 0.9505698802282272, "pass@64": 0.9706700595276606, "pass@128": 0.9822485207100592 }, "8": { "pass@1": 0.40384615384615385, "pass@2": 0.5536141290631941, "pass@4": 0.698270090036822, "pass@8": 0.8140536634874682, "pass@16": 0.8919098750755075, "pass@32": 0.9385195819134277, "pass@64": 0.9639372810558046, "pass@128": 0.9743589743589743 }, "9": { "pass@1": 0.29634683098591547, "pass@2": 0.44307973827215225, "pass@4": 0.5996262900236059, "pass@8": 0.7377739613933347, "pass@16": 0.8412027330164041, "pass@32": 0.9081616296352407, "pass@64": 0.9463322026470657, "pass@128": 0.9690140845070423 }, "11": { "pass@1": 0.27554001416430596, "pass@2": 0.4100284262006203, "pass@4": 0.5497061987294078, "pass@8": 0.6745030060620455, "pass@16": 0.7762085776756984, "pass@32": 0.8539760738175253, "pass@64": 0.9082416689117168, "pass@128": 0.9405099150141643 }, "12": { "pass@1": 0.223583156779661, "pass@2": 0.34553187141331937, "pass@4": 0.48248001150279896, "pass@8": 0.6133563555890079, "pass@16": 0.7254988286208823, "pass@32": 0.8126269158410524, "pass@64": 0.8750599826992427, "pass@128": 0.9180790960451978 }, "13": { "pass@1": 0.21137833072100312, "pass@2": 0.32505769752918806, "pass@4": 0.44905429150901616, "pass@8": 0.5689478723786576, "pass@16": 0.6776168936361864, "pass@32": 0.7707110571183069, "pass@64": 0.8439511414301138, "pass@128": 0.8934169278996865 }, "14": { "pass@1": 0.17978485202492211, "pass@2": 0.2873168708759536, "pass@4": 0.4125795875982794, "pass@8": 0.5333225333050318, "pass@16": 0.6392014624846123, "pass@32": 0.7319459986189009, "pass@64": 0.8117486314877205, "pass@128": 0.8785046728971962 }, "15": { "pass@1": 0.14981484826589594, "pass@2": 0.23933430715488574, "pass@4": 0.34942827206209043, "pass@8": 0.46691425776949924, "pass@16": 0.577039877850708, "pass@32": 0.6734734398150024, "pass@64": 0.7581490937591103, "pass@128": 0.8352601156069365 }, "16": { "pass@1": 0.1690272177419355, "pass@2": 0.26243729362458734, "pass@4": 0.3701715785526809, "pass@8": 0.4754923682022537, "pass@16": 0.566165648570034, "pass@32": 0.6434866743114298, "pass@64": 0.715818663351052, "pass@128": 0.7870967741935484 }, "17": { "pass@1": 0.14773753612716764, "pass@2": 0.22860994322060887, "pass@4": 0.3213330353214517, "pass@8": 0.4127466446487047, "pass@16": 0.4945081321294641, "pass@32": 0.5670904861099084, "pass@64": 0.6320432166581963, "pass@128": 0.684971098265896 }, "18": { "pass@1": 0.12301339285714286, "pass@2": 0.19675583520809908, "pass@4": 0.2861904290535112, "pass@8": 0.37906513974828127, "pass@16": 0.46258529242199625, "pass@32": 0.531878886537751, "pass@64": 0.5906816065671889, "pass@128": 0.6457142857142857 }, "19": { "pass@1": 0.13278903903903905, "pass@2": 0.21219681788087297, "pass@4": 0.30740728129704514, "pass@8": 0.3992844439193679, "pass@16": 0.47047243675247624, "pass@32": 0.522112980009493, "pass@64": 0.5637430472466359, "pass@128": 0.6066066066066066 }, "20": { "pass@1": 0.1215746996996997, "pass@2": 0.19323038589770877, "pass@4": 0.27804619692808696, "pass@8": 0.36386612424102277, "pass@16": 0.4397676932029267, "pass@32": 0.4992069295218254, "pass@64": 0.5447303107088249, "pass@128": 0.5855855855855856 } } }, "teachers_in_school": { "count": 821120, "correct": 303408, "answer_accuracy": 0.3695050662509743, "avg_response_len": 260.1671899356976, "resp_tokens_sum": 213628483, "pass_at_k": { "pass@1": 0.3695050662509743, "pass@2": 0.47036956628473514, "pass@4": 0.573653824112794, "pass@8": 0.6669001838874253, "pass@16": 0.7418401866442392, "pass@32": 0.7981261746577575, "pass@64": 0.8402704965092456, "pass@128": 0.873265783320343 }, "per_op_pass_at_k": { "10": { "pass@1": 0.305382476993865, "pass@2": 0.4384706656683249, "pass@4": 0.5788130030525324, "pass@8": 0.7079080797271436, "pass@16": 0.809303783504082, "pass@32": 0.8795306867277652, "pass@64": 0.9246891544210146, "pass@128": 0.950920245398773 }, "2": { "pass@1": 0.9775598404255319, "pass@2": 0.9882914402986861, "pass@4": 0.9931788543529323, "pass@8": 0.9950785758504687, "pass@16": 0.9963275595235965, "pass@32": 0.9977752173693791, "pass@64": 0.9992040875534045, "pass@128": 1.0 }, "3": { "pass@1": 0.9306344696969697, "pass@2": 0.9533270699117159, "pass@4": 0.9638342272556834, "pass@8": 0.9706880776824892, "pass@16": 0.9762976075504467, "pass@32": 0.981515194409324, "pass@64": 0.9865150313128073, "pass@128": 0.990909090909091 }, "4": { "pass@1": 0.7786150147928994, "pass@2": 0.8566846870195216, "pass@4": 0.9127427596883223, "pass@8": 0.9497142611938738, "pass@16": 0.9711949174245315, "pass@32": 0.9826570262446811, "pass@64": 0.9899957012280247, "pass@128": 0.9940828402366864 }, "5": { "pass@1": 0.6168025362318841, "pass@2": 0.7120884685609948, "pass@4": 0.7810124440966606, "pass@8": 0.8333119147189844, "pass@16": 0.8745230907515112, "pass@32": 0.9086414545020511, "pass@64": 0.9394193940381265, "pass@128": 0.9652173913043478 }, "6": { "pass@1": 0.5708945200573066, "pass@2": 0.7201273182095074, "pass@4": 0.8332363168966347, "pass@8": 0.9048154542138614, "pass@16": 0.9454304796626805, "pass@32": 0.9676635934894852, "pass@64": 0.9803148386384075, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.45682803468208094, "pass@2": 0.6100648297756136, "pass@4": 0.7466018149465419, "pass@8": 0.8507837490523493, "pass@16": 0.9185662560649683, "pass@32": 0.9563321743967517, "pass@64": 0.9785556722654779, "pass@128": 0.9942196531791907 }, "8": { "pass@1": 0.3819762323943662, "pass@2": 0.5406409421093487, "pass@4": 0.693883219263085, "pass@8": 0.8154527976781428, "pass@16": 0.8940305609873201, "pass@32": 0.9388662926400119, "pass@64": 0.9641705344024881, "pass@128": 0.9802816901408451 }, "9": { "pass@1": 0.3124766092814371, "pass@2": 0.4618525007072463, "pass@4": 0.6193879476704632, "pass@8": 0.7587824319807451, "pass@16": 0.8609492491217673, "pass@32": 0.9215203946762859, "pass@64": 0.9532264564553681, "pass@128": 0.9700598802395209 }, "11": { "pass@1": 0.23966961596385541, "pass@2": 0.3581350500426905, "pass@4": 0.48620031052217855, "pass@8": 0.6102333477314572, "pass@16": 0.7208873268789171, "pass@32": 0.8102303148700188, "pass@64": 0.8733357401088951, "pass@128": 0.9156626506024096 }, "12": { "pass@1": 0.2491673197492163, "pass@2": 0.376715884407474, "pass@4": 0.5163546439532046, "pass@8": 0.6470903372450432, "pass@16": 0.7569027766764611, "pass@32": 0.8431432973967836, "pass@64": 0.904642073000038, "pass@128": 0.9467084639498433 }, "13": { "pass@1": 0.195750175070028, "pass@2": 0.3036820535300734, "pass@4": 0.4266344887036182, "pass@8": 0.5461250500217757, "pass@16": 0.6500571912155733, "pass@32": 0.7377077535289895, "pass@64": 0.8120524459657263, "pass@128": 0.876750700280112 }, "14": { "pass@1": 0.18718019005847952, "pass@2": 0.29091444778284303, "pass@4": 0.40853745721477797, "pass@8": 0.5212011661278351, "pass@16": 0.6195737300108569, "pass@32": 0.7043303525635856, "pass@64": 0.7735331168668783, "pass@128": 0.8245614035087719 }, "15": { "pass@1": 0.15082859848484848, "pass@2": 0.24180013719875898, "pass@4": 0.3517029428139666, "pass@8": 0.46704428304724294, "pass@16": 0.5760473201704117, "pass@32": 0.6708225663764403, "pass@64": 0.7498293734022303, "pass@128": 0.8121212121212121 }, "16": { "pass@1": 0.14872685185185186, "pass@2": 0.237111136348341, "pass@4": 0.34124766161708425, "pass@8": 0.44297967345870165, "pass@16": 0.5312036383809488, "pass@32": 0.6088643968988856, "pass@64": 0.6800786909386469, "pass@128": 0.7464387464387464 }, "17": { "pass@1": 0.15285326086956522, "pass@2": 0.2479397955690318, "pass@4": 0.3625688723109922, "pass@8": 0.47662920580834106, "pass@16": 0.5736009919599262, "pass@32": 0.6496201555232083, "pass@64": 0.7054333096266076, "pass@128": 0.7422360248447205 }, "18": { "pass@1": 0.13494944852941176, "pass@2": 0.21811783522464095, "pass@4": 0.3204172084188005, "pass@8": 0.4256938916457332, "pass@16": 0.5168048555096408, "pass@32": 0.5864634897053611, "pass@64": 0.6395187651433718, "pass@128": 0.6852941176470588 }, "19": { "pass@1": 0.12423270089285714, "pass@2": 0.19673212235189355, "pass@4": 0.2854409744540861, "pass@8": 0.3764069622534746, "pass@16": 0.4543999639029509, "pass@32": 0.5144732309561025, "pass@64": 0.5651312688961807, "pass@128": 0.6220238095238095 }, "20": { "pass@1": 0.1110123502994012, "pass@2": 0.18206966971568675, "pass@4": 0.27122720307416665, "pass@8": 0.364016093733274, "pass@16": 0.44317220201732704, "pass@32": 0.5001531127134062, "pass@64": 0.5416504433836719, "pass@128": 0.5808383233532934 } } }, "movie_festival_awards": { "count": 801152, "correct": 294338, "answer_accuracy": 0.36739345342706503, "avg_response_len": 278.3378410089471, "resp_tokens_sum": 222990918, "pass_at_k": { "pass@1": 0.36739345342706503, "pass@2": 0.4674166444414597, "pass@4": 0.5704824904406169, "pass@8": 0.6637894140522359, "pass@16": 0.7393907080307304, "pass@32": 0.7967258012135678, "pass@64": 0.8399129773505546, "pass@128": 0.8737817542738456 }, "per_op_pass_at_k": { "10": { "pass@1": 0.3270055970149254, "pass@2": 0.47624610706311005, "pass@4": 0.6279120981146009, "pass@8": 0.7563851107378542, "pass@16": 0.8497442890943676, "pass@32": 0.9111210388030382, "pass@64": 0.9478362643936907, "pass@128": 0.9671641791044776 }, "2": { "pass@1": 0.9769642857142857, "pass@2": 0.9830708661417321, "pass@4": 0.9866033550270502, "pass@8": 0.9893155018165151, "pass@16": 0.9916245608505572, "pass@32": 0.9939798256522657, "pass@64": 0.9961816845755459, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9469979956268222, "pass@2": 0.9713605633479483, "pass@4": 0.9806534271590681, "pass@8": 0.9854379828873127, "pass@16": 0.989555097127444, "pass@32": 0.9933855442177933, "pass@64": 0.9955484171321441, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.7869380482456141, "pass@2": 0.8646955725928998, "pass@4": 0.9180183661252861, "pass@8": 0.9531497227749841, "pass@16": 0.9741448322769339, "pass@32": 0.9852302693709742, "pass@64": 0.9917320162290298, "pass@128": 0.9941520467836257 }, "5": { "pass@1": 0.5654181985294118, "pass@2": 0.6746120889300605, "pass@4": 0.7654377985840002, "pass@8": 0.8379095206619214, "pass@16": 0.8929060765099897, "pass@32": 0.9300541740977897, "pass@64": 0.9505432949778564, "pass@128": 0.9588235294117647 }, "6": { "pass@1": 0.5254830441640379, "pass@2": 0.6867497795523986, "pass@4": 0.8163599742264075, "pass@8": 0.9003470975341901, "pass@16": 0.9486839007810618, "pass@32": 0.9743177057639598, "pass@64": 0.9871789815186821, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.4392306170886076, "pass@2": 0.5908540877603908, "pass@4": 0.7283318169089623, "pass@8": 0.8372287158063058, "pass@16": 0.9098519085185298, "pass@32": 0.9500437938189338, "pass@64": 0.971288581664221, "pass@128": 0.9841772151898734 }, "8": { "pass@1": 0.37216122372372373, "pass@2": 0.5274600387789361, "pass@4": 0.6792629347570285, "pass@8": 0.8013797775771526, "pass@16": 0.8833480015453465, "pass@32": 0.9340773849291986, "pass@64": 0.9645022213053622, "pass@128": 0.9819819819819819 }, "9": { "pass@1": 0.28336012861736337, "pass@2": 0.4180384744410967, "pass@4": 0.5628083778514825, "pass@8": 0.6970908247674539, "pass@16": 0.8052044902612292, "pass@32": 0.8800700951064251, "pass@64": 0.9274745938248696, "pass@128": 0.9646302250803859 }, "11": { "pass@1": 0.2545138888888889, "pass@2": 0.3883834833145856, "pass@4": 0.5346769011016481, "pass@8": 0.6715277489831641, "pass@16": 0.7853170286802313, "pass@32": 0.8677554185709992, "pass@64": 0.9183972865925268, "pass@128": 0.9492063492063492 }, "12": { "pass@1": 0.21502293577981652, "pass@2": 0.3330044968576177, "pass@4": 0.4687746024292833, "pass@8": 0.6034395513800196, "pass@16": 0.7221423876765495, "pass@32": 0.8183903107061395, "pass@64": 0.8911141319352575, "pass@128": 0.9418960244648318 }, "13": { "pass@1": 0.2099488811728395, "pass@2": 0.3174991038446582, "pass@4": 0.4354547793794296, "pass@8": 0.5481744928579311, "pass@16": 0.6486211003222007, "pass@32": 0.7379259580961539, "pass@64": 0.8151676418417848, "pass@128": 0.8734567901234568 }, "14": { "pass@1": 0.1815189169139466, "pass@2": 0.28346785263674384, "pass@4": 0.4032592756395063, "pass@8": 0.523281961422275, "pass@16": 0.6304113447749283, "pass@32": 0.7224702971779134, "pass@64": 0.7969991172125063, "pass@128": 0.8545994065281899 }, "15": { "pass@1": 0.1320167824074074, "pass@2": 0.2144788325070477, "pass@4": 0.3215919644882351, "pass@8": 0.44103201441754786, "pass@16": 0.5547608300838373, "pass@32": 0.651467623339146, "pass@64": 0.7279869288291854, "pass@128": 0.7901234567901234 }, "16": { "pass@1": 0.15899244100294985, "pass@2": 0.2515438819594455, "pass@4": 0.3578476877337234, "pass@8": 0.4608270452373182, "pass@16": 0.5533257358956741, "pass@32": 0.6394566913432015, "pass@64": 0.7226423547087432, "pass@128": 0.799410029498525 }, "17": { "pass@1": 0.1363187123493976, "pass@2": 0.22074222085191147, "pass@4": 0.3212653414558117, "pass@8": 0.4186067605348141, "pass@16": 0.49906082889385794, "pass@32": 0.5628644264374904, "pass@64": 0.6196725167686506, "pass@128": 0.677710843373494 }, "18": { "pass@1": 0.12885584677419354, "pass@2": 0.202006604013208, "pass@4": 0.2896082764856006, "pass@8": 0.3788062977989847, "pass@16": 0.4582978330657227, "pass@32": 0.5262400787404502, "pass@64": 0.5901966248323723, "pass@128": 0.6516129032258065 }, "19": { "pass@1": 0.13878398791540786, "pass@2": 0.2194093893474797, "pass@4": 0.31099969527978194, "pass@8": 0.3977403547926579, "pass@16": 0.4681941745583987, "pass@32": 0.5200797609470273, "pass@64": 0.5630198897420078, "pass@128": 0.6072507552870091 }, "20": { "pass@1": 0.11085304054054054, "pass@2": 0.18222220744366419, "pass@4": 0.2722583466481104, "pass@8": 0.36793721246803396, "pass@16": 0.4533363894654612, "pass@32": 0.5175601720413012, "pass@64": 0.5652306966653234, "pass@128": 0.6036036036036037 } } } } } ]