[ { "checkpoint": "checkpoint-85656", "id_acc": 0.9933975694444445, "ood_acc": 0.48817109375, "total_acc": 0.7274888980263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.00196440972223, "ood_avg_resp_len": 334.2508890625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 586420, "answer_accuracy": 0.7242185030034777, "avg_response_len": 256.7089454730477, "resp_tokens_sum": 207864421, "pass_at_k": { "pass@1": 0.7242185030034777, "pass@2": 0.7585805035959495, "pass@4": 0.7882152405970935, "pass@8": 0.8131007504036403, "pass@16": 0.8335299105303158, "pass@32": 0.8507363433743848, "pass@64": 0.8662116073517198, "pass@128": 0.8808093582042364 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9612140486725663, "pass@2": 0.9701686729147794, "pass@4": 0.97611373799514, "pass@8": 0.9795318507801245, "pass@16": 0.9815704330658933, "pass@32": 0.9828295616605529, "pass@64": 0.9837715482760134, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9966269841269841, "pass@2": 0.9969132764654418, "pass@4": 0.9970214452360122, "pass@8": 0.9972112860892366, "pass@16": 0.9975721784776902, "pass@32": 0.9982189726284215, "pass@64": 0.9992125984251968, "pass@128": 1.0 }, "6": { "pass@1": 0.9999766092814372, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9993296967455622, "pass@2": 0.9998813364860458, "pass@4": 0.9999958538836491, "pass@8": 0.9999999967670793, "pass@16": 0.9999999999999998, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9997495993589743, "pass@2": 0.9999948736876642, "pass@4": 0.999999998497784, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9941901408450704, "pass@2": 0.9947054868581569, "pass@4": 0.9950140287921757, "pass@8": 0.9955275550723595, "pass@16": 0.996246252427952, "pass@32": 0.9969218386791731, "pass@64": 0.9971743778471076, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.9172716005665722, "pass@2": 0.9294245332470837, "pass@4": 0.9375553278927952, "pass@8": 0.9436644091134152, "pass@16": 0.94896922068123, "pass@32": 0.9540347827409994, "pass@64": 0.9584288926634769, "pass@128": 0.9603399433427762 }, "12": { "pass@1": 0.8369085451977402, "pass@2": 0.8658925413719473, "pass@4": 0.8836768536877804, "pass@8": 0.8958159717383948, "pass@16": 0.9059099640119125, "pass@32": 0.9152839590889877, "pass@64": 0.9230974982514136, "pass@128": 0.9293785310734464 }, "13": { "pass@1": 0.7135579937304075, "pass@2": 0.7642971083355962, "pass@4": 0.7975250877731188, "pass@8": 0.8216605066521206, "pass@16": 0.8408669871052371, "pass@32": 0.8585289603051615, "pass@64": 0.8786083453443386, "pass@128": 0.9028213166144201 }, "14": { "pass@1": 0.6607038551401869, "pass@2": 0.7342098081781829, "pass@4": 0.7866001411038571, "pass@8": 0.8223529852870952, "pass@16": 0.8463869020391322, "pass@32": 0.8634275011957604, "pass@64": 0.8774430319994344, "pass@128": 0.8909657320872274 }, "15": { "pass@1": 0.5573744580924855, "pass@2": 0.6510904999772427, "pass@4": 0.723555908654481, "pass@8": 0.779664294770354, "pass@16": 0.8218733604726072, "pass@32": 0.8537991239873666, "pass@64": 0.8786629030021994, "pass@128": 0.8988439306358381 }, "16": { "pass@1": 0.32026209677419354, "pass@2": 0.4102227266954534, "pass@4": 0.4922931598267957, "pass@8": 0.5608738002824728, "pass@16": 0.6161741026713436, "pass@32": 0.6634223589324809, "pass@64": 0.7052034534598515, "pass@128": 0.7419354838709677 }, "17": { "pass@1": 0.26948609104046245, "pass@2": 0.3551862398161212, "pass@4": 0.43630361992323174, "pass@8": 0.505574472344914, "pass@16": 0.5626248447366223, "pass@32": 0.612875860834388, "pass@64": 0.6613171736457973, "pass@128": 0.7052023121387283 }, "18": { "pass@1": 0.22223214285714285, "pass@2": 0.29839848143981995, "pass@4": 0.37236364186619514, "pass@8": 0.43890352397685345, "pass@16": 0.49460411827912604, "pass@32": 0.5408992361679525, "pass@64": 0.5819298205174266, "pass@128": 0.6228571428571429 }, "19": { "pass@1": 0.1733765015015015, "pass@2": 0.24151267409141433, "pass@4": 0.3106473104150269, "pass@8": 0.3760434973464308, "pass@16": 0.4339640400717074, "pass@32": 0.48326526375921236, "pass@64": 0.5279349721453175, "pass@128": 0.5735735735735735 }, "20": { "pass@1": 0.16915352852852852, "pass@2": 0.22966285675439213, "pass@4": 0.29331480636992446, "pass@8": 0.3554372484491605, "pass@16": 0.4113463579552269, "pass@32": 0.4595926430033481, "pass@64": 0.5028409960228297, "pass@128": 0.5435435435435435 } } }, "teachers_in_school": { "count": 821120, "correct": 601723, "answer_accuracy": 0.7328076286048324, "avg_response_len": 266.3788885911925, "resp_tokens_sum": 218729033, "pass_at_k": { "pass@1": 0.7328076286048324, "pass@2": 0.7669891057192438, "pass@4": 0.7962575649205126, "pass@8": 0.8205455374066479, "pass@16": 0.8399386311082003, "pass@32": 0.8559486775608024, "pass@64": 0.8701074575696454, "pass@128": 0.883398285268901 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9406873082822086, "pass@2": 0.9511727241437608, "pass@4": 0.9582467286489494, "pass@8": 0.962884319554284, "pass@16": 0.9658664939297285, "pass@32": 0.9683840162934146, "pass@64": 0.9708284840200408, "pass@128": 0.9723926380368099 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9996866045845272, "pass@2": 0.9999725029894185, "pass@4": 0.9999998079573864, "pass@8": 0.9999999999974207, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9996387283236994, "pass@2": 0.9999765315643349, "pass@4": 0.9999998658946533, "pass@8": 0.9999999999989994, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9976012323943662, "pass@2": 0.9993276588665854, "pass@4": 0.9999400980687273, "pass@8": 0.9999995286192314, "pass@16": 0.9999999999862863, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9954622005988024, "pass@2": 0.996530438139469, "pass@4": 0.9974659918071619, "pass@8": 0.9982248854431038, "pass@16": 0.9990042031981354, "pass@32": 0.9997223135661357, "pass@64": 0.999990730945279, "pass@128": 1.0 }, "11": { "pass@1": 0.9490540286144579, "pass@2": 0.9586703111659238, "pass@4": 0.963866821334833, "pass@8": 0.9669982809187331, "pass@16": 0.969628195342412, "pass@32": 0.9726172988672541, "pass@64": 0.9751752015368585, "pass@128": 0.9759036144578314 }, "12": { "pass@1": 0.8521012931034483, "pass@2": 0.8769631044356135, "pass@4": 0.893521940635163, "pass@8": 0.9075129211754909, "pass@16": 0.919181931341527, "pass@32": 0.9290936707076372, "pass@64": 0.9385951394414912, "pass@128": 0.9498432601880877 }, "13": { "pass@1": 0.7197785364145658, "pass@2": 0.7765865204349457, "pass@4": 0.8138981214533052, "pass@8": 0.840127920923921, "pass@16": 0.8606898149555469, "pass@32": 0.8784119819890157, "pass@64": 0.8928222471865029, "pass@128": 0.9047619047619048 }, "14": { "pass@1": 0.6990131578947368, "pass@2": 0.7748933727034123, "pass@4": 0.8276604991262051, "pass@8": 0.8608615541084115, "pass@16": 0.8813606703486647, "pass@32": 0.8960207950792911, "pass@64": 0.9067741271436445, "pass@128": 0.9152046783625731 }, "15": { "pass@1": 0.5314867424242424, "pass@2": 0.6243128877356237, "pass@4": 0.6995647515651452, "pass@8": 0.75847718932693, "pass@16": 0.8041517108171061, "pass@32": 0.8403438310227098, "pass@64": 0.8678819774825477, "pass@128": 0.8878787878787879 }, "16": { "pass@1": 0.3511173433048433, "pass@2": 0.4367120375978642, "pass@4": 0.509192721636291, "pass@8": 0.567463683171276, "pass@16": 0.6138049647381483, "pass@32": 0.6543388540897418, "pass@64": 0.6920704030785279, "pass@128": 0.7236467236467237 }, "17": { "pass@1": 0.2860539596273292, "pass@2": 0.3759124199149017, "pass@4": 0.4613117808061256, "pass@8": 0.5321531449577279, "pass@16": 0.5842617561652651, "pass@32": 0.6244032611091793, "pass@64": 0.6603615858501944, "pass@128": 0.6987577639751553 }, "18": { "pass@1": 0.2511948529411765, "pass@2": 0.3300544233441406, "pass@4": 0.40649941000022055, "pass@8": 0.4779186811576755, "pass@16": 0.5402423482438927, "pass@32": 0.5939876301314176, "pass@64": 0.6458831650585033, "pass@128": 0.6970588235294117 }, "19": { "pass@1": 0.18038504464285715, "pass@2": 0.24445003456599168, "pass@4": 0.31143215635991933, "pass@8": 0.3728059388521139, "pass@16": 0.42353768253638485, "pass@32": 0.4649215111100438, "pass@64": 0.500723873663761, "pass@128": 0.5386904761904762 }, "20": { "pass@1": 0.1533261601796407, "pass@2": 0.21130733120373435, "pass@4": 0.2719651745440502, "pass@8": 0.332414175156139, "pass@16": 0.3859289673999605, "pass@32": 0.4304629843267275, "pass@64": 0.4713483469991754, "pass@128": 0.5119760479041916 } } }, "movie_festival_awards": { "count": 801152, "correct": 581110, "answer_accuracy": 0.7253430060712573, "avg_response_len": 284.832275273606, "resp_tokens_sum": 228193947, "pass_at_k": { "pass@1": 0.7253430060712573, "pass@2": 0.75873765399871, "pass@4": 0.7882019855861954, "pass@8": 0.8138588630545672, "pass@16": 0.8355427556244789, "pass@32": 0.8539066962141767, "pass@64": 0.8698343314586575, "pass@128": 0.8838472599456783 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9534048507462687, "pass@2": 0.9626968503937007, "pass@4": 0.9692538331962235, "pass@8": 0.9737725528077535, "pass@16": 0.9771025187908413, "pass@32": 0.9801357911203106, "pass@64": 0.9834547978205753, "pass@128": 0.9880597014925373 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9966452205882353, "pass@2": 0.9975343040759611, "pass@4": 0.9980915448068992, "pass@8": 0.9987831969997992, "pass@16": 0.9995279850408865, "pass@32": 0.9999439253289022, "pass@64": 0.9999998174959024, "pass@128": 1.0 }, "6": { "pass@1": 0.9999507097791798, "pass@2": 0.999999611888025, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9998022151898734, "pass@2": 0.999996106598226, "pass@4": 0.9999999985167994, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9918355855855856, "pass@2": 0.9945075488874701, "pass@4": 0.9965787761890124, "pass@8": 0.998290842921839, "pass@16": 0.9994100203212533, "pass@32": 0.9999208020070317, "pass@64": 0.99999958431727, "pass@128": 1.0 }, "9": { "pass@1": 0.9956541398713826, "pass@2": 0.9983808105678912, "pass@4": 0.999648498318739, "pass@8": 0.9999806352115692, "pass@16": 0.9999999483522362, "pass@32": 0.9999999999999448, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9447420634920635, "pass@2": 0.9581868672665916, "pass@4": 0.967509306872355, "pass@8": 0.9740486174543461, "pass@16": 0.9786847931164305, "pass@32": 0.9824150336632566, "pass@64": 0.9857183739073222, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.8612146406727829, "pass@2": 0.8891486220472444, "pass@4": 0.9075370982870257, "pass@8": 0.921338707735371, "pass@16": 0.9329246372544424, "pass@32": 0.9447859108257943, "pass@64": 0.9570330678851205, "pass@128": 0.9694189602446484 }, "13": { "pass@1": 0.6789400077160493, "pass@2": 0.73740597963449, "pass@4": 0.7799996001657193, "pass@8": 0.8137514624128047, "pass@16": 0.8405120187983943, "pass@32": 0.8611789198572768, "pass@64": 0.8775786513793102, "pass@128": 0.8919753086419753 }, "14": { "pass@1": 0.6469538204747775, "pass@2": 0.7170783049837612, "pass@4": 0.7698086949072013, "pass@8": 0.8095645184136512, "pass@16": 0.8393733146970238, "pass@32": 0.8621325130569488, "pass@64": 0.8802217668630712, "pass@128": 0.8931750741839762 }, "15": { "pass@1": 0.5352527006172839, "pass@2": 0.6160802165354328, "pass@4": 0.6814122117837121, "pass@8": 0.734581191554782, "pass@16": 0.7757085390245353, "pass@32": 0.8070609426882411, "pass@64": 0.8332631723379745, "pass@128": 0.8549382716049383 }, "16": { "pass@1": 0.3416758849557522, "pass@2": 0.42799572619794185, "pass@4": 0.5040647457784591, "pass@8": 0.5700512110067496, "pass@16": 0.6254832399008291, "pass@32": 0.6707192692761402, "pass@64": 0.7088023579005431, "pass@128": 0.7433628318584071 }, "17": { "pass@1": 0.2613422439759036, "pass@2": 0.3423844245090598, "pass@4": 0.41293712645858993, "pass@8": 0.46923112448581195, "pass@16": 0.5187456803719004, "pass@32": 0.5701175454546831, "pass@64": 0.6236549755551729, "pass@128": 0.6716867469879518 }, "18": { "pass@1": 0.2354334677419355, "pass@2": 0.3043799212598424, "pass@4": 0.37419252270885495, "pass@8": 0.4401254353269728, "pass@16": 0.4954914482495745, "pass@32": 0.5391664434428325, "pass@64": 0.5774079786759414, "pass@128": 0.6193548387096774 }, "19": { "pass@1": 0.17546261329305135, "pass@2": 0.24499771035040552, "pass@4": 0.31780488079095853, "pass@8": 0.38831339902772993, "pass@16": 0.45257478979141497, "pass@32": 0.5076430412025037, "pass@64": 0.5521427562189815, "pass@128": 0.5891238670694864 }, "20": { "pass@1": 0.1487424924924925, "pass@2": 0.2120966931498427, "pass@4": 0.28508847148610916, "pass@8": 0.3607474325394686, "pass@16": 0.42986908543734814, "pass@32": 0.48956023306096935, "pass@64": 0.5385710498391255, "pass@128": 0.5765765765765766 } } } } } ]