| [ |
| { |
| "checkpoint": "checkpoint-87232", |
| "id_acc": 0.9916848958333333, |
| "ood_acc": 0.5103265625, |
| "total_acc": 0.7383384046052631, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.09075, |
| "ood_avg_resp_len": 331.12723203125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 594339, |
| "answer_accuracy": 0.7339983303035094, |
| "avg_response_len": 254.85102651754664, |
| "resp_tokens_sum": 206360012, |
| "pass_at_k": { |
| "pass@1": 0.7339983303035094, |
| "pass@2": 0.766837811270068, |
| "pass@4": 0.7939157616165781, |
| "pass@8": 0.8163702919498419, |
| "pass@16": 0.8352614482430876, |
| "pass@32": 0.8513196102786976, |
| "pass@64": 0.8653439036553512, |
| "pass@128": 0.8779639582674676 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9480779867256637, |
| "pass@2": 0.9566359341973846, |
| "pass@4": 0.9616766333411866, |
| "pass@8": 0.9651726636249782, |
| "pass@16": 0.9692196823524485, |
| "pass@32": 0.97481772830985, |
| "pass@64": 0.9812497154356057, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999761085626911, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9945932539682539, |
| "pass@2": 0.9959176977877766, |
| "pass@4": 0.9974196499247115, |
| "pass@8": 0.9989851212168439, |
| "pass@16": 0.9998591210200438, |
| "pass@32": 0.9999983941274779, |
| "pass@64": 0.9999999999930085, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9989006362275449, |
| "pass@2": 0.9997112075062474, |
| "pass@4": 0.9999743510728822, |
| "pass@8": 0.999999838950045, |
| "pass@16": 0.999999999997984, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9968333949704142, |
| "pass@2": 0.9972194153892746, |
| "pass@4": 0.9973982767686584, |
| "pass@8": 0.9977218352513628, |
| "pass@16": 0.9982775396418643, |
| "pass@32": 0.9990787135335893, |
| "pass@64": 0.999823789777757, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9970452724358975, |
| "pass@2": 0.9986099806935191, |
| "pass@4": 0.999473333261227, |
| "pass@8": 0.9999200982642733, |
| "pass@16": 0.9999985668344405, |
| "pass@32": 0.9999999998711727, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9945202464788733, |
| "pass@2": 0.9958373766219364, |
| "pass@4": 0.9969449463183299, |
| "pass@8": 0.9983695680131398, |
| "pass@16": 0.9995592129533992, |
| "pass@32": 0.9999749253186775, |
| "pass@64": 0.9999999847953782, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9395803824362606, |
| "pass@2": 0.9552343105217375, |
| "pass@4": 0.9662133374440095, |
| "pass@8": 0.9741163225761772, |
| "pass@16": 0.9805233498467629, |
| "pass@32": 0.9862271955974979, |
| "pass@64": 0.990258617147434, |
| "pass@128": 0.9915014164305949 |
| }, |
| "12": { |
| "pass@1": 0.8688426906779662, |
| "pass@2": 0.8937004398549764, |
| "pass@4": 0.9074316775021765, |
| "pass@8": 0.916103799384447, |
| "pass@16": 0.9235575220707577, |
| "pass@32": 0.9321592204063329, |
| "pass@64": 0.9423997578620953, |
| "pass@128": 0.9548022598870056 |
| }, |
| "13": { |
| "pass@1": 0.7924422021943573, |
| "pass@2": 0.8466476038555523, |
| "pass@4": 0.881631973804998, |
| "pass@8": 0.9072567938439052, |
| "pass@16": 0.9267869637927506, |
| "pass@32": 0.9402232771590543, |
| "pass@64": 0.9492802250397395, |
| "pass@128": 0.9561128526645768 |
| }, |
| "14": { |
| "pass@1": 0.6981357087227414, |
| "pass@2": 0.7716420450364267, |
| "pass@4": 0.8249349056251142, |
| "pass@8": 0.8621783595097423, |
| "pass@16": 0.8881510445226661, |
| "pass@32": 0.9062686837304577, |
| "pass@64": 0.9198054242296082, |
| "pass@128": 0.9314641744548287 |
| }, |
| "15": { |
| "pass@1": 0.5974078757225434, |
| "pass@2": 0.6856353261116929, |
| "pass@4": 0.7487913491485523, |
| "pass@8": 0.7956108185134464, |
| "pass@16": 0.8316006329172544, |
| "pass@32": 0.8583271731591949, |
| "pass@64": 0.8760916709273456, |
| "pass@128": 0.8901734104046243 |
| }, |
| "16": { |
| "pass@1": 0.3366179435483871, |
| "pass@2": 0.42653749682499353, |
| "pass@4": 0.5031402418447689, |
| "pass@8": 0.5672463445899628, |
| "pass@16": 0.6209081778921159, |
| "pass@32": 0.665086694227002, |
| "pass@64": 0.7026113845936993, |
| "pass@128": 0.7354838709677419 |
| }, |
| "17": { |
| "pass@1": 0.25467395231213874, |
| "pass@2": 0.33395868417459373, |
| "pass@4": 0.4082760364853236, |
| "pass@8": 0.47233764949345625, |
| "pass@16": 0.5260260079789568, |
| "pass@32": 0.571963436324117, |
| "pass@64": 0.6134129630124927, |
| "pass@128": 0.653179190751445 |
| }, |
| "18": { |
| "pass@1": 0.21457589285714285, |
| "pass@2": 0.2768356299212599, |
| "pass@4": 0.3355648478761583, |
| "pass@8": 0.38952041068387955, |
| "pass@16": 0.43873754558431316, |
| "pass@32": 0.48381457400582284, |
| "pass@64": 0.5260384593414925, |
| "pass@128": 0.5685714285714286 |
| }, |
| "19": { |
| "pass@1": 0.1757225975975976, |
| "pass@2": 0.23524324028753155, |
| "pass@4": 0.2930079792953809, |
| "pass@8": 0.34762229169153697, |
| "pass@16": 0.3981864068077494, |
| "pass@32": 0.44387262585496184, |
| "pass@64": 0.48552201334672673, |
| "pass@128": 0.5225225225225225 |
| }, |
| "20": { |
| "pass@1": 0.17107732732732733, |
| "pass@2": 0.2323425787992716, |
| "pass@4": 0.292989135930081, |
| "pass@8": 0.3479686059426612, |
| "pass@16": 0.3961013361234629, |
| "pass@32": 0.4388507465142246, |
| "pass@64": 0.47856541310267475, |
| "pass@128": 0.5135135135135135 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 608346, |
| "answer_accuracy": 0.7408734411535464, |
| "avg_response_len": 264.18500462782544, |
| "resp_tokens_sum": 216927591, |
| "pass_at_k": { |
| "pass@1": 0.7408734411535464, |
| "pass@2": 0.7729139688598866, |
| "pass@4": 0.799469514030927, |
| "pass@8": 0.8221626845488487, |
| "pass@16": 0.8419379333989953, |
| "pass@32": 0.8586623678609139, |
| "pass@64": 0.8724422740633893, |
| "pass@128": 0.884489477786438 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9309336656441718, |
| "pass@2": 0.9446063897879331, |
| "pass@4": 0.953863818978149, |
| "pass@8": 0.9590589424615077, |
| "pass@16": 0.962847844844578, |
| "pass@32": 0.9662415542616282, |
| "pass@64": 0.9684511377172703, |
| "pass@128": 0.9693251533742331 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997632575757576, |
| "pass@2": 0.9999832229778096, |
| "pass@4": 0.9999999403483655, |
| "pass@8": 0.9999999999999045, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9970788043478261, |
| "pass@2": 0.9971014492753624, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9999104584527221, |
| "pass@2": 0.9999978848453399, |
| "pass@4": 0.9999999997314089, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9972227239884393, |
| "pass@2": 0.9973320655636977, |
| "pass@4": 0.9975404185387231, |
| "pass@8": 0.9979180896078554, |
| "pass@16": 0.9985344794678028, |
| "pass@32": 0.9993322691599966, |
| "pass@64": 0.9999167083883301, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9961267605633802, |
| "pass@2": 0.9970001109016303, |
| "pass@4": 0.9974083912926376, |
| "pass@8": 0.9976823323117774, |
| "pass@16": 0.9981192931789404, |
| "pass@32": 0.9988210100498002, |
| "pass@64": 0.9996562049462128, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9935909431137725, |
| "pass@2": 0.9940053456079966, |
| "pass@4": 0.9940119743639827, |
| "pass@8": 0.9940119760479041, |
| "pass@16": 0.9940119760479041, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 0.9940119760479041, |
| "pass@128": 0.9940119760479041 |
| }, |
| "11": { |
| "pass@1": 0.9602550828313253, |
| "pass@2": 0.970971978465041, |
| "pass@4": 0.9782139917939471, |
| "pass@8": 0.9839004159996938, |
| "pass@16": 0.9879086832778012, |
| "pass@32": 0.9901380783202616, |
| "pass@64": 0.990912883728698, |
| "pass@128": 0.9909638554216867 |
| }, |
| "12": { |
| "pass@1": 0.8763224921630094, |
| "pass@2": 0.900271209241478, |
| "pass@4": 0.9133548522931495, |
| "pass@8": 0.9209842028359165, |
| "pass@16": 0.9267801804069482, |
| "pass@32": 0.933089345765585, |
| "pass@64": 0.9410744604116404, |
| "pass@128": 0.9529780564263323 |
| }, |
| "13": { |
| "pass@1": 0.7849921218487395, |
| "pass@2": 0.8409747265047747, |
| "pass@4": 0.8762234135123865, |
| "pass@8": 0.8985125497258456, |
| "pass@16": 0.9134010231005779, |
| "pass@32": 0.9254236218087579, |
| "pass@64": 0.9370460119024951, |
| "pass@128": 0.9495798319327731 |
| }, |
| "14": { |
| "pass@1": 0.7069398757309941, |
| "pass@2": 0.7717301681862135, |
| "pass@4": 0.8148905690516753, |
| "pass@8": 0.8471529723976811, |
| "pass@16": 0.874481645705777, |
| "pass@32": 0.8968713302321746, |
| "pass@64": 0.9119866191391205, |
| "pass@128": 0.9210526315789473 |
| }, |
| "15": { |
| "pass@1": 0.5650804924242424, |
| "pass@2": 0.648119855046528, |
| "pass@4": 0.705192844360364, |
| "pass@8": 0.748218519072543, |
| "pass@16": 0.7846452064508993, |
| "pass@32": 0.8177111748276681, |
| "pass@64": 0.8478250288764668, |
| "pass@128": 0.8727272727272727 |
| }, |
| "16": { |
| "pass@1": 0.36529558404558404, |
| "pass@2": 0.44831099838481714, |
| "pass@4": 0.522387939488333, |
| "pass@8": 0.5885047312678048, |
| "pass@16": 0.6456204773103222, |
| "pass@32": 0.6925542643828992, |
| "pass@64": 0.7322780842103859, |
| "pass@128": 0.7720797720797721 |
| }, |
| "17": { |
| "pass@1": 0.2891110248447205, |
| "pass@2": 0.36688106751601696, |
| "pass@4": 0.43798682569570097, |
| "pass@8": 0.5033939886857743, |
| "pass@16": 0.5646914107061055, |
| "pass@32": 0.6216320496640186, |
| "pass@64": 0.6729229318261878, |
| "pass@128": 0.717391304347826 |
| }, |
| "18": { |
| "pass@1": 0.24885110294117646, |
| "pass@2": 0.3269091593330246, |
| "pass@4": 0.40279429905820585, |
| "pass@8": 0.47328846803764835, |
| "pass@16": 0.5382715998405633, |
| "pass@32": 0.5923264198869231, |
| "pass@64": 0.6280874274780245, |
| "pass@128": 0.6470588235294118 |
| }, |
| "19": { |
| "pass@1": 0.18440755208333334, |
| "pass@2": 0.24145516966629174, |
| "pass@4": 0.29885994245139, |
| "pass@8": 0.3529140211026931, |
| "pass@16": 0.4005923401776416, |
| "pass@32": 0.43950719643893654, |
| "pass@64": 0.4732488990474139, |
| "pass@128": 0.5089285714285714 |
| }, |
| "20": { |
| "pass@1": 0.16193394461077845, |
| "pass@2": 0.22137307569899564, |
| "pass@4": 0.2823540183412704, |
| "pass@8": 0.34147304486737784, |
| "pass@16": 0.39384732852364274, |
| "pass@32": 0.4351023741508188, |
| "pass@64": 0.4685266363500677, |
| "pass@128": 0.5 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 592954, |
| "answer_accuracy": 0.7401267175267615, |
| "avg_response_len": 284.0956497643393, |
| "resp_tokens_sum": 227603798, |
| "pass_at_k": { |
| "pass@1": 0.7401267175267615, |
| "pass@2": 0.7722973406483588, |
| "pass@4": 0.798888160134517, |
| "pass@8": 0.8212680869317169, |
| "pass@16": 0.8402065828292287, |
| "pass@32": 0.8560490247769718, |
| "pass@64": 0.8694195607343695, |
| "pass@128": 0.8811311711135964 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.955946828358209, |
| "pass@2": 0.9649635679868374, |
| "pass@4": 0.9718655183027491, |
| "pass@8": 0.976645209603533, |
| "pass@16": 0.980130434589518, |
| "pass@32": 0.9829127186112555, |
| "pass@64": 0.9846494147518906, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.99995444606414, |
| "pass@2": 0.9999996413075917, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.990625, |
| "pass@2": 0.9919548257295043, |
| "pass@4": 0.992871911874986, |
| "pass@8": 0.9942384543252845, |
| "pass@16": 0.9962103892851597, |
| "pass@32": 0.9982961665733241, |
| "pass@64": 0.9996276828087242, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9998028391167192, |
| "pass@2": 0.9999976713281503, |
| "pass@4": 0.9999999997042957, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9985907832278481, |
| "pass@2": 0.9996593273447623, |
| "pass@4": 0.9999667955864694, |
| "pass@8": 0.999999738747604, |
| "pass@16": 0.9999999999943548, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9923986486486487, |
| "pass@2": 0.9953048520961906, |
| "pass@4": 0.9977257513418932, |
| "pass@8": 0.9992899419707638, |
| "pass@16": 0.9998973768486178, |
| "pass@32": 0.9999981316234936, |
| "pass@64": 0.9999999999680574, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9932174437299035, |
| "pass@2": 0.9962429899739221, |
| "pass@4": 0.9975781732066449, |
| "pass@8": 0.9985523897401913, |
| "pass@16": 0.9994013402043442, |
| "pass@32": 0.9999153427870624, |
| "pass@64": 0.9999995549120945, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9605654761904762, |
| "pass@2": 0.975881530433696, |
| "pass@4": 0.9833013834580199, |
| "pass@8": 0.9872022483964916, |
| "pass@16": 0.9891194740789849, |
| "pass@32": 0.9905335487886743, |
| "pass@64": 0.991972003414339, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.8935875382262997, |
| "pass@2": 0.9220066850875291, |
| "pass@4": 0.9387073992471122, |
| "pass@8": 0.9510898366281239, |
| "pass@16": 0.9615428935827477, |
| "pass@32": 0.969691622075752, |
| "pass@64": 0.9752066618655814, |
| "pass@128": 0.9785932721712538 |
| }, |
| "13": { |
| "pass@1": 0.7517361111111112, |
| "pass@2": 0.8090873948916112, |
| "pass@4": 0.8455759129645837, |
| "pass@8": 0.8697116970197645, |
| "pass@16": 0.8874883039902416, |
| "pass@32": 0.900889745357289, |
| "pass@64": 0.9101563477790442, |
| "pass@128": 0.9166666666666666 |
| }, |
| "14": { |
| "pass@1": 0.68692043768546, |
| "pass@2": 0.7552582274118557, |
| "pass@4": 0.802779436046013, |
| "pass@8": 0.8374284766346397, |
| "pass@16": 0.8637923075666653, |
| "pass@32": 0.8852207785869997, |
| "pass@64": 0.9041290395423973, |
| "pass@128": 0.9169139465875371 |
| }, |
| "15": { |
| "pass@1": 0.580029899691358, |
| "pass@2": 0.6586145590308153, |
| "pass@4": 0.7123502805783536, |
| "pass@8": 0.7504793262867279, |
| "pass@16": 0.7821101700595695, |
| "pass@32": 0.8110951011884022, |
| "pass@64": 0.8367530155250535, |
| "pass@128": 0.8580246913580247 |
| }, |
| "16": { |
| "pass@1": 0.362947087020649, |
| "pass@2": 0.44745538928762235, |
| "pass@4": 0.5243293916247198, |
| "pass@8": 0.5920827100675968, |
| "pass@16": 0.6494094758377291, |
| "pass@32": 0.6968132030019422, |
| "pass@64": 0.7359247606414249, |
| "pass@128": 0.7669616519174042 |
| }, |
| "17": { |
| "pass@1": 0.2875094126506024, |
| "pass@2": 0.3558600790959113, |
| "pass@4": 0.4198601656344163, |
| "pass@8": 0.47922518320068247, |
| "pass@16": 0.5319420034970082, |
| "pass@32": 0.5775220564958283, |
| "pass@64": 0.6191132532606295, |
| "pass@128": 0.6566265060240963 |
| }, |
| "18": { |
| "pass@1": 0.2513104838709677, |
| "pass@2": 0.3259802832105665, |
| "pass@4": 0.394517647592438, |
| "pass@8": 0.45139163380199426, |
| "pass@16": 0.4981203974693504, |
| "pass@32": 0.5369155158447213, |
| "pass@64": 0.5719011271454831, |
| "pass@128": 0.6096774193548387 |
| }, |
| "19": { |
| "pass@1": 0.17529739425981872, |
| "pass@2": 0.23745636284701577, |
| "pass@4": 0.30084155115051703, |
| "pass@8": 0.3611008955586078, |
| "pass@16": 0.4128868923939326, |
| "pass@32": 0.4547154673885572, |
| "pass@64": 0.48965423676567504, |
| "pass@128": 0.5226586102719033 |
| }, |
| "20": { |
| "pass@1": 0.1692942942942943, |
| "pass@2": 0.22726983873637402, |
| "pass@4": 0.2870520309398261, |
| "pass@8": 0.34649098992377375, |
| "pass@16": 0.4028096541887778, |
| "pass@32": 0.4514113817669326, |
| "pass@64": 0.49106104416255225, |
| "pass@128": 0.5285285285285285 |
| } |
| } |
| } |
| } |
| } |
| ] |
