[ { "checkpoint": "checkpoint-87232", "id_acc": 0.9916848958333333, "ood_acc": 0.5103265625, "total_acc": 0.7383384046052631, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.09075, "ood_avg_resp_len": 331.12723203125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 594339, "answer_accuracy": 0.7339983303035094, "avg_response_len": 254.85102651754664, "resp_tokens_sum": 206360012, "pass_at_k": { "pass@1": 0.7339983303035094, "pass@2": 0.766837811270068, "pass@4": 0.7939157616165781, "pass@8": 0.8163702919498419, "pass@16": 0.8352614482430876, "pass@32": 0.8513196102786976, "pass@64": 0.8653439036553512, "pass@128": 0.8779639582674676 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9480779867256637, "pass@2": 0.9566359341973846, "pass@4": 0.9616766333411866, "pass@8": 0.9651726636249782, "pass@16": 0.9692196823524485, "pass@32": 0.97481772830985, "pass@64": 0.9812497154356057, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999761085626911, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9945932539682539, "pass@2": 0.9959176977877766, "pass@4": 0.9974196499247115, "pass@8": 0.9989851212168439, "pass@16": 0.9998591210200438, "pass@32": 0.9999983941274779, "pass@64": 0.9999999999930085, "pass@128": 1.0 }, "6": { "pass@1": 0.9989006362275449, "pass@2": 0.9997112075062474, "pass@4": 0.9999743510728822, "pass@8": 0.999999838950045, "pass@16": 0.999999999997984, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9968333949704142, "pass@2": 0.9972194153892746, "pass@4": 0.9973982767686584, "pass@8": 0.9977218352513628, "pass@16": 0.9982775396418643, "pass@32": 0.9990787135335893, "pass@64": 0.999823789777757, "pass@128": 1.0 }, "8": { "pass@1": 0.9970452724358975, "pass@2": 0.9986099806935191, "pass@4": 0.999473333261227, "pass@8": 0.9999200982642733, "pass@16": 0.9999985668344405, "pass@32": 0.9999999998711727, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9945202464788733, "pass@2": 0.9958373766219364, "pass@4": 0.9969449463183299, "pass@8": 0.9983695680131398, "pass@16": 0.9995592129533992, "pass@32": 0.9999749253186775, "pass@64": 0.9999999847953782, "pass@128": 1.0 }, "11": { "pass@1": 0.9395803824362606, "pass@2": 0.9552343105217375, "pass@4": 0.9662133374440095, "pass@8": 0.9741163225761772, "pass@16": 0.9805233498467629, "pass@32": 0.9862271955974979, "pass@64": 0.990258617147434, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.8688426906779662, "pass@2": 0.8937004398549764, "pass@4": 0.9074316775021765, "pass@8": 0.916103799384447, "pass@16": 0.9235575220707577, "pass@32": 0.9321592204063329, "pass@64": 0.9423997578620953, "pass@128": 0.9548022598870056 }, "13": { "pass@1": 0.7924422021943573, "pass@2": 0.8466476038555523, "pass@4": 0.881631973804998, "pass@8": 0.9072567938439052, "pass@16": 0.9267869637927506, "pass@32": 0.9402232771590543, "pass@64": 0.9492802250397395, "pass@128": 0.9561128526645768 }, "14": { "pass@1": 0.6981357087227414, "pass@2": 0.7716420450364267, "pass@4": 0.8249349056251142, "pass@8": 0.8621783595097423, "pass@16": 0.8881510445226661, "pass@32": 0.9062686837304577, "pass@64": 0.9198054242296082, "pass@128": 0.9314641744548287 }, "15": { "pass@1": 0.5974078757225434, "pass@2": 0.6856353261116929, "pass@4": 0.7487913491485523, "pass@8": 0.7956108185134464, "pass@16": 0.8316006329172544, "pass@32": 0.8583271731591949, "pass@64": 0.8760916709273456, "pass@128": 0.8901734104046243 }, "16": { "pass@1": 0.3366179435483871, "pass@2": 0.42653749682499353, "pass@4": 0.5031402418447689, "pass@8": 0.5672463445899628, "pass@16": 0.6209081778921159, "pass@32": 0.665086694227002, "pass@64": 0.7026113845936993, "pass@128": 0.7354838709677419 }, "17": { "pass@1": 0.25467395231213874, "pass@2": 0.33395868417459373, "pass@4": 0.4082760364853236, "pass@8": 0.47233764949345625, "pass@16": 0.5260260079789568, "pass@32": 0.571963436324117, "pass@64": 0.6134129630124927, "pass@128": 0.653179190751445 }, "18": { "pass@1": 0.21457589285714285, "pass@2": 0.2768356299212599, "pass@4": 0.3355648478761583, "pass@8": 0.38952041068387955, "pass@16": 0.43873754558431316, "pass@32": 0.48381457400582284, "pass@64": 0.5260384593414925, "pass@128": 0.5685714285714286 }, "19": { "pass@1": 0.1757225975975976, "pass@2": 0.23524324028753155, "pass@4": 0.2930079792953809, "pass@8": 0.34762229169153697, "pass@16": 0.3981864068077494, "pass@32": 0.44387262585496184, "pass@64": 0.48552201334672673, "pass@128": 0.5225225225225225 }, "20": { "pass@1": 0.17107732732732733, "pass@2": 0.2323425787992716, "pass@4": 0.292989135930081, "pass@8": 0.3479686059426612, "pass@16": 0.3961013361234629, "pass@32": 0.4388507465142246, "pass@64": 0.47856541310267475, "pass@128": 0.5135135135135135 } } }, "teachers_in_school": { "count": 821120, "correct": 608346, "answer_accuracy": 0.7408734411535464, "avg_response_len": 264.18500462782544, "resp_tokens_sum": 216927591, "pass_at_k": { "pass@1": 0.7408734411535464, "pass@2": 0.7729139688598866, "pass@4": 0.799469514030927, "pass@8": 0.8221626845488487, "pass@16": 0.8419379333989953, "pass@32": 0.8586623678609139, "pass@64": 0.8724422740633893, "pass@128": 0.884489477786438 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9309336656441718, "pass@2": 0.9446063897879331, "pass@4": 0.953863818978149, "pass@8": 0.9590589424615077, "pass@16": 0.962847844844578, "pass@32": 0.9662415542616282, "pass@64": 0.9684511377172703, "pass@128": 0.9693251533742331 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997632575757576, "pass@2": 0.9999832229778096, "pass@4": 0.9999999403483655, "pass@8": 0.9999999999999045, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9970788043478261, "pass@2": 0.9971014492753624, "pass@4": 0.9971014492753624, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9999104584527221, "pass@2": 0.9999978848453399, "pass@4": 0.9999999997314089, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9972227239884393, "pass@2": 0.9973320655636977, "pass@4": 0.9975404185387231, "pass@8": 0.9979180896078554, "pass@16": 0.9985344794678028, "pass@32": 0.9993322691599966, "pass@64": 0.9999167083883301, "pass@128": 1.0 }, "8": { "pass@1": 0.9961267605633802, "pass@2": 0.9970001109016303, "pass@4": 0.9974083912926376, "pass@8": 0.9976823323117774, "pass@16": 0.9981192931789404, "pass@32": 0.9988210100498002, "pass@64": 0.9996562049462128, "pass@128": 1.0 }, "9": { "pass@1": 0.9935909431137725, "pass@2": 0.9940053456079966, "pass@4": 0.9940119743639827, "pass@8": 0.9940119760479041, "pass@16": 0.9940119760479041, "pass@32": 0.9940119760479041, "pass@64": 0.9940119760479041, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.9602550828313253, "pass@2": 0.970971978465041, "pass@4": 0.9782139917939471, "pass@8": 0.9839004159996938, "pass@16": 0.9879086832778012, "pass@32": 0.9901380783202616, "pass@64": 0.990912883728698, "pass@128": 0.9909638554216867 }, "12": { "pass@1": 0.8763224921630094, "pass@2": 0.900271209241478, "pass@4": 0.9133548522931495, "pass@8": 0.9209842028359165, "pass@16": 0.9267801804069482, "pass@32": 0.933089345765585, "pass@64": 0.9410744604116404, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.7849921218487395, "pass@2": 0.8409747265047747, "pass@4": 0.8762234135123865, "pass@8": 0.8985125497258456, "pass@16": 0.9134010231005779, "pass@32": 0.9254236218087579, "pass@64": 0.9370460119024951, "pass@128": 0.9495798319327731 }, "14": { "pass@1": 0.7069398757309941, "pass@2": 0.7717301681862135, "pass@4": 0.8148905690516753, "pass@8": 0.8471529723976811, "pass@16": 0.874481645705777, "pass@32": 0.8968713302321746, "pass@64": 0.9119866191391205, "pass@128": 0.9210526315789473 }, "15": { "pass@1": 0.5650804924242424, "pass@2": 0.648119855046528, "pass@4": 0.705192844360364, "pass@8": 0.748218519072543, "pass@16": 0.7846452064508993, "pass@32": 0.8177111748276681, "pass@64": 0.8478250288764668, "pass@128": 0.8727272727272727 }, "16": { "pass@1": 0.36529558404558404, "pass@2": 0.44831099838481714, "pass@4": 0.522387939488333, "pass@8": 0.5885047312678048, "pass@16": 0.6456204773103222, "pass@32": 0.6925542643828992, "pass@64": 0.7322780842103859, "pass@128": 0.7720797720797721 }, "17": { "pass@1": 0.2891110248447205, "pass@2": 0.36688106751601696, "pass@4": 0.43798682569570097, "pass@8": 0.5033939886857743, "pass@16": 0.5646914107061055, "pass@32": 0.6216320496640186, "pass@64": 0.6729229318261878, "pass@128": 0.717391304347826 }, "18": { "pass@1": 0.24885110294117646, "pass@2": 0.3269091593330246, "pass@4": 0.40279429905820585, "pass@8": 0.47328846803764835, "pass@16": 0.5382715998405633, "pass@32": 0.5923264198869231, "pass@64": 0.6280874274780245, "pass@128": 0.6470588235294118 }, "19": { "pass@1": 0.18440755208333334, "pass@2": 0.24145516966629174, "pass@4": 0.29885994245139, "pass@8": 0.3529140211026931, "pass@16": 0.4005923401776416, "pass@32": 0.43950719643893654, "pass@64": 0.4732488990474139, "pass@128": 0.5089285714285714 }, "20": { "pass@1": 0.16193394461077845, "pass@2": 0.22137307569899564, "pass@4": 0.2823540183412704, "pass@8": 0.34147304486737784, "pass@16": 0.39384732852364274, "pass@32": 0.4351023741508188, "pass@64": 0.4685266363500677, "pass@128": 0.5 } } }, "movie_festival_awards": { "count": 801152, "correct": 592954, "answer_accuracy": 0.7401267175267615, "avg_response_len": 284.0956497643393, "resp_tokens_sum": 227603798, "pass_at_k": { "pass@1": 0.7401267175267615, "pass@2": 0.7722973406483588, "pass@4": 0.798888160134517, "pass@8": 0.8212680869317169, "pass@16": 0.8402065828292287, "pass@32": 0.8560490247769718, "pass@64": 0.8694195607343695, "pass@128": 0.8811311711135964 }, "per_op_pass_at_k": { "10": { "pass@1": 0.955946828358209, "pass@2": 0.9649635679868374, "pass@4": 0.9718655183027491, "pass@8": 0.976645209603533, "pass@16": 0.980130434589518, "pass@32": 0.9829127186112555, "pass@64": 0.9846494147518906, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.99995444606414, "pass@2": 0.9999996413075917, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.990625, "pass@2": 0.9919548257295043, "pass@4": 0.992871911874986, "pass@8": 0.9942384543252845, "pass@16": 0.9962103892851597, "pass@32": 0.9982961665733241, "pass@64": 0.9996276828087242, "pass@128": 1.0 }, "6": { "pass@1": 0.9998028391167192, "pass@2": 0.9999976713281503, "pass@4": 0.9999999997042957, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9985907832278481, "pass@2": 0.9996593273447623, "pass@4": 0.9999667955864694, "pass@8": 0.999999738747604, "pass@16": 0.9999999999943548, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9923986486486487, "pass@2": 0.9953048520961906, "pass@4": 0.9977257513418932, "pass@8": 0.9992899419707638, "pass@16": 0.9998973768486178, "pass@32": 0.9999981316234936, "pass@64": 0.9999999999680574, "pass@128": 1.0 }, "9": { "pass@1": 0.9932174437299035, "pass@2": 0.9962429899739221, "pass@4": 0.9975781732066449, "pass@8": 0.9985523897401913, "pass@16": 0.9994013402043442, "pass@32": 0.9999153427870624, "pass@64": 0.9999995549120945, "pass@128": 1.0 }, "11": { "pass@1": 0.9605654761904762, "pass@2": 0.975881530433696, "pass@4": 0.9833013834580199, "pass@8": 0.9872022483964916, "pass@16": 0.9891194740789849, "pass@32": 0.9905335487886743, "pass@64": 0.991972003414339, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.8935875382262997, "pass@2": 0.9220066850875291, "pass@4": 0.9387073992471122, "pass@8": 0.9510898366281239, "pass@16": 0.9615428935827477, "pass@32": 0.969691622075752, "pass@64": 0.9752066618655814, "pass@128": 0.9785932721712538 }, "13": { "pass@1": 0.7517361111111112, "pass@2": 0.8090873948916112, "pass@4": 0.8455759129645837, "pass@8": 0.8697116970197645, "pass@16": 0.8874883039902416, "pass@32": 0.900889745357289, "pass@64": 0.9101563477790442, "pass@128": 0.9166666666666666 }, "14": { "pass@1": 0.68692043768546, "pass@2": 0.7552582274118557, "pass@4": 0.802779436046013, "pass@8": 0.8374284766346397, "pass@16": 0.8637923075666653, "pass@32": 0.8852207785869997, "pass@64": 0.9041290395423973, "pass@128": 0.9169139465875371 }, "15": { "pass@1": 0.580029899691358, "pass@2": 0.6586145590308153, "pass@4": 0.7123502805783536, "pass@8": 0.7504793262867279, "pass@16": 0.7821101700595695, "pass@32": 0.8110951011884022, "pass@64": 0.8367530155250535, "pass@128": 0.8580246913580247 }, "16": { "pass@1": 0.362947087020649, "pass@2": 0.44745538928762235, "pass@4": 0.5243293916247198, "pass@8": 0.5920827100675968, "pass@16": 0.6494094758377291, "pass@32": 0.6968132030019422, "pass@64": 0.7359247606414249, "pass@128": 0.7669616519174042 }, "17": { "pass@1": 0.2875094126506024, "pass@2": 0.3558600790959113, "pass@4": 0.4198601656344163, "pass@8": 0.47922518320068247, "pass@16": 0.5319420034970082, "pass@32": 0.5775220564958283, "pass@64": 0.6191132532606295, "pass@128": 0.6566265060240963 }, "18": { "pass@1": 0.2513104838709677, "pass@2": 0.3259802832105665, "pass@4": 0.394517647592438, "pass@8": 0.45139163380199426, "pass@16": 0.4981203974693504, "pass@32": 0.5369155158447213, "pass@64": 0.5719011271454831, "pass@128": 0.6096774193548387 }, "19": { "pass@1": 0.17529739425981872, "pass@2": 0.23745636284701577, "pass@4": 0.30084155115051703, "pass@8": 0.3611008955586078, "pass@16": 0.4128868923939326, "pass@32": 0.4547154673885572, "pass@64": 0.48965423676567504, "pass@128": 0.5226586102719033 }, "20": { "pass@1": 0.1692942942942943, "pass@2": 0.22726983873637402, "pass@4": 0.2870520309398261, "pass@8": 0.34649098992377375, "pass@16": 0.4028096541887778, "pass@32": 0.4514113817669326, "pass@64": 0.49106104416255225, "pass@128": 0.5285285285285285 } } } } } ]