[ { "checkpoint": "checkpoint-21350", "id_acc": 0.7816371527777778, "ood_acc": 0.2690828125, "total_acc": 0.5118717105263157, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.91984982638888, "ood_avg_resp_len": 310.56489765625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 414495, "answer_accuracy": 0.5118941175308251, "avg_response_len": 245.55807629228582, "resp_tokens_sum": 198835250, "pass_at_k": { "pass@1": 0.5118941175308251, "pass@2": 0.6060234127809088, "pass@4": 0.6823425681248854, "pass@8": 0.7382585878256954, "pass@16": 0.7790123570823432, "pass@32": 0.8101523431020332, "pass@64": 0.834971805948881, "pass@128": 0.8559911476446411 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6108038348082596, "pass@2": 0.7592836155436323, "pass@4": 0.8628990543770518, "pass@8": 0.9209853506323293, "pass@16": 0.9502138983934935, "pass@32": 0.9653789425226263, "pass@64": 0.9738636888533403, "pass@128": 0.9823008849557522 }, "2": { "pass@1": 0.9687013239875389, "pass@2": 0.9735359635244192, "pass@4": 0.9774219598134347, "pass@8": 0.9811756048228187, "pass@16": 0.9857846921623187, "pass@32": 0.9912202144423357, "pass@64": 0.9954408818642898, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.9708046636085627, "pass@2": 0.9782207914950999, "pass@4": 0.9856412343525864, "pass@8": 0.9913135287729292, "pass@16": 0.9943854355053758, "pass@32": 0.9964695421928315, "pass@64": 0.9982793427520417, "pass@128": 1.0 }, "4": { "pass@1": 0.933056640625, "pass@2": 0.9640221302903547, "pass@4": 0.9808446167861831, "pass@8": 0.9889389221699499, "pass@16": 0.9928154480539686, "pass@32": 0.9944218943628333, "pass@64": 0.9953122612788613, "pass@128": 0.996875 }, "5": { "pass@1": 0.7997519841269841, "pass@2": 0.8826568553930755, "pass@4": 0.930272895352366, "pass@8": 0.9548178801748886, "pass@16": 0.9687459281478877, "pass@32": 0.9774046788113472, "pass@64": 0.983474310194418, "pass@128": 0.9873015873015873 }, "6": { "pass@1": 0.7953078218562875, "pass@2": 0.8896447999669947, "pass@4": 0.944976489933518, "pass@8": 0.973097387972258, "pass@16": 0.9870541545418998, "pass@32": 0.9947098289309901, "pass@64": 0.9987991381053104, "pass@128": 1.0 }, "7": { "pass@1": 0.7229105029585798, "pass@2": 0.8472880655080834, "pass@4": 0.9232964889189142, "pass@8": 0.9585954889402899, "pass@16": 0.9758673121900198, "pass@32": 0.987287560047621, "pass@64": 0.9952321084717118, "pass@128": 1.0 }, "8": { "pass@1": 0.6659405048076923, "pass@2": 0.8077088696244699, "pass@4": 0.9056807724755556, "pass@8": 0.9572339732355738, "pass@16": 0.9783949645645401, "pass@32": 0.986614119977316, "pass@64": 0.9907329976438498, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.5919014084507043, "pass@2": 0.7484803010979263, "pass@4": 0.8604035258796869, "pass@8": 0.9244515130120431, "pass@16": 0.958264149196147, "pass@32": 0.9771934956760493, "pass@64": 0.9883104989872438, "pass@128": 0.9943661971830986 }, "11": { "pass@1": 0.5599327195467422, "pass@2": 0.7109233844884119, "pass@4": 0.8225052578413533, "pass@8": 0.8926580349921708, "pass@16": 0.9333295570967041, "pass@32": 0.9564493119764775, "pass@64": 0.9692695987042779, "pass@128": 0.9773371104815864 }, "12": { "pass@1": 0.4427966101694915, "pass@2": 0.6038222479870099, "pass@4": 0.7427874291137334, "pass@8": 0.8402630444157572, "pass@16": 0.8997258866173762, "pass@32": 0.9344561909630209, "pass@64": 0.9530677128620351, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.3682650862068966, "pass@2": 0.4979925425172167, "pass@4": 0.619347059673967, "pass@8": 0.7220487867666285, "pass@16": 0.8013165423964927, "pass@32": 0.8591777688713896, "pass@64": 0.9028755230719813, "pass@128": 0.9341692789968652 }, "14": { "pass@1": 0.2775506230529595, "pass@2": 0.3937211010130745, "pass@4": 0.5110733629090756, "pass@8": 0.615459827824061, "pass@16": 0.7029316216951226, "pass@32": 0.7736901410312667, "pass@64": 0.8274568239163917, "pass@128": 0.8660436137071651 }, "15": { "pass@1": 0.23317828757225434, "pass@2": 0.3282679441081425, "pass@4": 0.42495304768623543, "pass@8": 0.5137240855394527, "pass@16": 0.5889845728337569, "pass@32": 0.6469725361731052, "pass@64": 0.689443755215235, "pass@128": 0.7254335260115607 }, "16": { "pass@1": 0.1934475806451613, "pass@2": 0.26473877635255283, "pass@4": 0.33628472217182526, "pass@8": 0.40642216848629104, "pass@16": 0.4755940708933289, "pass@32": 0.5403641008495248, "pass@64": 0.5982561508228122, "pass@128": 0.6483870967741936 }, "17": { "pass@1": 0.17131051300578035, "pass@2": 0.2387941775522277, "pass@4": 0.30575373544347406, "pass@8": 0.36986780070560576, "pass@16": 0.4334840695506441, "pass@32": 0.49633664095597957, "pass@64": 0.554674308041247, "pass@128": 0.6098265895953757 }, "18": { "pass@1": 0.1675669642857143, "pass@2": 0.23335911136107987, "pass@4": 0.30120476270823276, "pass@8": 0.3655080978889007, "pass@16": 0.42347287095849856, "pass@32": 0.4755511329959968, "pass@64": 0.5251996675191798, "pass@128": 0.5771428571428572 }, "19": { "pass@1": 0.1631709834834835, "pass@2": 0.22588434891584497, "pass@4": 0.286252485049729, "pass@8": 0.33920095575428966, "pass@16": 0.38482372915001195, "pass@32": 0.42880135905591316, "pass@64": 0.47486273358836156, "pass@128": 0.5195195195195195 }, "20": { "pass@1": 0.15169857357357358, "pass@2": 0.21332885543023333, "pass@4": 0.2779789730675557, "pass@8": 0.3386028033542325, "pass@16": 0.3894628828004633, "pass@32": 0.4317839018100493, "pass@64": 0.4703749719074299, "pass@128": 0.5105105105105106 } } }, "teachers_in_school": { "count": 821120, "correct": 414949, "answer_accuracy": 0.5053451383476227, "avg_response_len": 254.23482073265782, "resp_tokens_sum": 208757296, "pass_at_k": { "pass@1": 0.5053451383476227, "pass@2": 0.5979934263015504, "pass@4": 0.6754241830157071, "pass@8": 0.733934855593854, "pass@16": 0.777473519194293, "pass@32": 0.811257252919628, "pass@64": 0.8377472273022628, "pass@128": 0.8583008573655495 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5747699386503068, "pass@2": 0.7293609004395927, "pass@4": 0.842726695972819, "pass@8": 0.9087806167016811, "pass@16": 0.9435410406326107, "pass@32": 0.962740051480476, "pass@64": 0.9756515561073661, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 0.9805043693009119, "pass@2": 0.9843484491300291, "pass@4": 0.9870946238103215, "pass@8": 0.9895161087507777, "pass@16": 0.9925479984428305, "pass@32": 0.9957528327388283, "pass@64": 0.9980910863778228, "pass@128": 1.0 }, "3": { "pass@1": 0.9767282196969697, "pass@2": 0.9832457199952278, "pass@4": 0.9873704749974436, "pass@8": 0.9902525452585946, "pass@16": 0.9920598769834958, "pass@32": 0.9934207656439095, "pass@64": 0.9950847041173666, "pass@128": 0.996969696969697 }, "4": { "pass@1": 0.890625, "pass@2": 0.927779783814006, "pass@4": 0.9517953019371097, "pass@8": 0.9651948360744051, "pass@16": 0.9715446905626869, "pass@32": 0.9753629793049493, "pass@64": 0.9796651628850817, "pass@128": 0.985207100591716 }, "5": { "pass@1": 0.8036005434782608, "pass@2": 0.8880306401917152, "pass@4": 0.9356044589534999, "pass@8": 0.9566631468858801, "pass@16": 0.9677299365458314, "pass@32": 0.9762975400418303, "pass@64": 0.9827398189807199, "pass@128": 0.9884057971014493 }, "6": { "pass@1": 0.7473809097421203, "pass@2": 0.8443866615526928, "pass@4": 0.9105473168826673, "pass@8": 0.9485723437526672, "pass@16": 0.9680171909631179, "pass@32": 0.9798515662831963, "pass@64": 0.9887143445075527, "pass@128": 0.994269340974212 }, "7": { "pass@1": 0.7122245303468208, "pass@2": 0.82878353852806, "pass@4": 0.9054310449271872, "pass@8": 0.9459152039364331, "pass@16": 0.9658799168085859, "pass@32": 0.9764867382386906, "pass@64": 0.9835947173089279, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.6647887323943662, "pass@2": 0.7976325967616721, "pass@4": 0.8875285859514035, "pass@8": 0.9386763687150315, "pass@16": 0.9646758684012232, "pass@32": 0.979982181086267, "pass@64": 0.9892714095496256, "pass@128": 0.9915492957746479 }, "9": { "pass@1": 0.5772595434131736, "pass@2": 0.7188549819652033, "pass@4": 0.8316630995601596, "pass@8": 0.9051994481455924, "pass@16": 0.9467216399752423, "pass@32": 0.9702232540521561, "pass@64": 0.9833604457495104, "pass@128": 0.9910179640718563 }, "11": { "pass@1": 0.5197900978915663, "pass@2": 0.6667180533156247, "pass@4": 0.7842412309530583, "pass@8": 0.8631492332252853, "pass@16": 0.9100323170688028, "pass@32": 0.939724543524283, "pass@64": 0.9587396589529518, "pass@128": 0.9698795180722891 }, "12": { "pass@1": 0.47910952194357365, "pass@2": 0.6399678035445416, "pass@4": 0.7682812988893628, "pass@8": 0.8561805492242159, "pass@16": 0.9122820941511327, "pass@32": 0.9466209121984869, "pass@64": 0.9664428205027745, "pass@128": 0.9749216300940439 }, "13": { "pass@1": 0.3403361344537815, "pass@2": 0.47338797723813925, "pass@4": 0.599396724831665, "pass@8": 0.7050868992294267, "pass@16": 0.789229855222965, "pass@32": 0.850339928979183, "pass@64": 0.8882099270210574, "pass@128": 0.9103641456582633 }, "14": { "pass@1": 0.2902960526315789, "pass@2": 0.3985098079845278, "pass@4": 0.5051356878087606, "pass@8": 0.5994879403043188, "pass@16": 0.6772260970805483, "pass@32": 0.7417045218997341, "pass@64": 0.7956760258214428, "pass@128": 0.8391812865497076 }, "15": { "pass@1": 0.21420454545454545, "pass@2": 0.3055867483894059, "pass@4": 0.39845808222835793, "pass@8": 0.4871670652592622, "pass@16": 0.5689720699097469, "pass@32": 0.6398546008577851, "pass@64": 0.6962629638631309, "pass@128": 0.7393939393939394 }, "16": { "pass@1": 0.19827279202279202, "pass@2": 0.27872031821342824, "pass@4": 0.3629984417011977, "pass@8": 0.4408281715911366, "pass@16": 0.508026552191099, "pass@32": 0.5674690802927236, "pass@64": 0.6205853526518682, "pass@128": 0.6695156695156695 }, "17": { "pass@1": 0.18997476708074534, "pass@2": 0.2610380648750425, "pass@4": 0.3368291220973155, "pass@8": 0.4132398731867813, "pass@16": 0.4834337259752426, "pass@32": 0.5428640234414458, "pass@64": 0.5920497781538735, "pass@128": 0.6335403726708074 }, "18": { "pass@1": 0.1624310661764706, "pass@2": 0.23409311313107897, "pass@4": 0.30889321582963913, "pass@8": 0.38150896761898023, "pass@16": 0.45005568993156586, "pass@32": 0.5123031105086977, "pass@64": 0.5669113473578711, "pass@128": 0.611764705882353 }, "19": { "pass@1": 0.1396484375, "pass@2": 0.20062848589238844, "pass@4": 0.26531595074053244, "pass@8": 0.3268746347503379, "pass@16": 0.38500503796087715, "pass@32": 0.4401306384131243, "pass@64": 0.48860559547150584, "pass@128": 0.5267857142857143 }, "20": { "pass@1": 0.13585329341317365, "pass@2": 0.19414848943844587, "pass@4": 0.2557152845602383, "pass@8": 0.31399903611220836, "pass@16": 0.3664202373537723, "pass@32": 0.41394175111035364, "pass@64": 0.458630781103217, "pass@128": 0.5 } } }, "movie_festival_awards": { "count": 801152, "correct": 415428, "answer_accuracy": 0.5185383048410289, "avg_response_len": 270.5880906494648, "resp_tokens_sum": 216782190, "pass_at_k": { "pass@1": 0.5185383048410289, "pass@2": 0.6084410889264427, "pass@4": 0.6810429088149833, "pass@8": 0.7345410710265036, "pass@16": 0.7740380221037376, "pass@32": 0.8049699669862138, "pass@64": 0.830523348092998, "pass@128": 0.852532353411088 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6364972014925373, "pass@2": 0.779170951345635, "pass@4": 0.8748992114791617, "pass@8": 0.9296739769639495, "pass@16": 0.9606524838402445, "pass@32": 0.9778808056234413, "pass@64": 0.9866519715129541, "pass@128": 0.991044776119403 }, "2": { "pass@1": 0.9890178571428572, "pass@2": 0.9894386248593927, "pass@4": 0.9902087345331835, "pass@8": 0.9914985188935059, "pass@16": 0.9933107529937302, "pass@32": 0.9951512772864062, "pass@64": 0.9964259400874629, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9770863702623906, "pass@2": 0.9816582636991802, "pass@4": 0.9849404237189008, "pass@8": 0.9880356633680653, "pass@16": 0.9908833077628361, "pass@32": 0.9930378018134619, "pass@64": 0.994901347869653, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.9239309210526315, "pass@2": 0.9586214860477973, "pass@4": 0.976123211024061, "pass@8": 0.9856909444572529, "pass@16": 0.992686049786314, "pass@32": 0.9975110532244272, "pass@64": 0.9995905276774154, "pass@128": 1.0 }, "5": { "pass@1": 0.7964154411764706, "pass@2": 0.8850762795275591, "pass@4": 0.9348415889190315, "pass@8": 0.9588671697704182, "pass@16": 0.9705587733630365, "pass@32": 0.9767018962624524, "pass@64": 0.9807280377964777, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.7949526813880127, "pass@2": 0.8886277757768444, "pass@4": 0.9394307528396486, "pass@8": 0.9633346653308688, "pass@16": 0.9756900342767074, "pass@32": 0.9836834856909692, "pass@64": 0.9887716619145747, "pass@128": 0.9905362776025236 }, "7": { "pass@1": 0.7339052610759493, "pass@2": 0.858375563764577, "pass@4": 0.9326507575595763, "pass@8": 0.9665651133669365, "pass@16": 0.9816764265402234, "pass@32": 0.989944041561538, "pass@64": 0.9933136265995508, "pass@128": 0.9936708860759493 }, "8": { "pass@1": 0.676684496996997, "pass@2": 0.8082308440330094, "pass@4": 0.8958928258967639, "pass@8": 0.9414967540056054, "pass@16": 0.9641145512791177, "pass@32": 0.9783433425114311, "pass@64": 0.9884820344539859, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.5907606511254019, "pass@2": 0.740335895764235, "pass@4": 0.8499149766648948, "pass@8": 0.9154502502375335, "pass@16": 0.9532473681211369, "pass@32": 0.9756017883574458, "pass@64": 0.9872400961475196, "pass@128": 0.9935691318327974 }, "11": { "pass@1": 0.5646329365079366, "pass@2": 0.71685336207974, "pass@4": 0.82946349980062, "pass@8": 0.8989213736705283, "pass@16": 0.9382922546798307, "pass@32": 0.9607834842187218, "pass@64": 0.9747276125638802, "pass@128": 0.9841269841269841 }, "12": { "pass@1": 0.4446435397553517, "pass@2": 0.5997179681668231, "pass@4": 0.7331657046309579, "pass@8": 0.8297917311860121, "pass@16": 0.8931182146188502, "pass@32": 0.9322322978163884, "pass@64": 0.9567219108599694, "pass@128": 0.9724770642201835 }, "13": { "pass@1": 0.36060474537037035, "pass@2": 0.48475168902498333, "pass@4": 0.6023837983793693, "pass@8": 0.6990689294100764, "pass@16": 0.7706865603861656, "pass@32": 0.8230004455601055, "pass@64": 0.8626431069718627, "pass@128": 0.8950617283950617 }, "14": { "pass@1": 0.29214577151335314, "pass@2": 0.4008234713427883, "pass@4": 0.5074832745313363, "pass@8": 0.601375084942065, "pass@16": 0.6789703942326578, "pass@32": 0.7406445380246015, "pass@64": 0.789367599624807, "pass@128": 0.8308605341246291 }, "15": { "pass@1": 0.1976514274691358, "pass@2": 0.29131314097890537, "pass@4": 0.3921881697658164, "pass@8": 0.4855609918599434, "pass@16": 0.5662116470586928, "pass@32": 0.6373774462304737, "pass@64": 0.6977827741321524, "pass@128": 0.7438271604938271 }, "16": { "pass@1": 0.18706213126843657, "pass@2": 0.25359985076533575, "pass@4": 0.3209949519584388, "pass@8": 0.38554267819668736, "pass@16": 0.4486018761610928, "pass@32": 0.5138035329859271, "pass@64": 0.5784656480723253, "pass@128": 0.6371681415929203 }, "17": { "pass@1": 0.16479198042168675, "pass@2": 0.23328031614647562, "pass@4": 0.30360634128188774, "pass@8": 0.3691675256281784, "pass@16": 0.42690508197986565, "pass@32": 0.47893424285590325, "pass@64": 0.5317847410416872, "pass@128": 0.5933734939759037 }, "18": { "pass@1": 0.17295866935483872, "pass@2": 0.24062341249682495, "pass@4": 0.3112551211340518, "pass@8": 0.37637725350191814, "pass@16": 0.4330918948484191, "pass@32": 0.48455445994709084, "pass@64": 0.5325561649996549, "pass@128": 0.5709677419354838 }, "19": { "pass@1": 0.1567692598187311, "pass@2": 0.21468178330994123, "pass@4": 0.2714260056616788, "pass@8": 0.3231161448695808, "pass@16": 0.36890863896298454, "pass@32": 0.4108112826103503, "pass@64": 0.449994222417303, "pass@128": 0.486404833836858 }, "20": { "pass@1": 0.13527590090090091, "pass@2": 0.1969527352155305, "pass@4": 0.26444538830168735, "pass@8": 0.330625251893072, "pass@16": 0.3876755280995512, "pass@32": 0.43574100762264556, "pass@64": 0.48261430489771634, "pass@128": 0.5345345345345346 } } } } } ]