| [ |
| { |
| "checkpoint": "checkpoint-38797", |
| "id_acc": 0.9935989583333333, |
| "ood_acc": 0.87967265625, |
| "total_acc": 0.9336377467105264, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.3666579861111, |
| "ood_avg_resp_len": 381.14332578125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 753646, |
| "answer_accuracy": 0.930739705184951, |
| "avg_response_len": 282.40589062006006, |
| "resp_tokens_sum": 228671957, |
| "pass_at_k": { |
| "pass@1": 0.930739705184951, |
| "pass@2": 0.9504902861518782, |
| "pass@4": 0.96328378278217, |
| "pass@8": 0.9720240684920053, |
| "pass@16": 0.9780653667421171, |
| "pass@32": 0.982247035735046, |
| "pass@64": 0.9852938579685702, |
| "pass@128": 0.9878280113815997 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9597852138643068, |
| "pass@2": 0.9687187884700253, |
| "pass@4": 0.9758312157882919, |
| "pass@8": 0.9804771165839773, |
| "pass@16": 0.9833711817465719, |
| "pass@32": 0.9856575859333179, |
| "pass@64": 0.9880229297421079, |
| "pass@128": 0.9911504424778761 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999761085626911, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999267578125, |
| "pass@2": 0.9999996155265748, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9988591269841269, |
| "pass@2": 0.9998297087864016, |
| "pass@4": 0.99999456436993, |
| "pass@8": 0.9999999950675459, |
| "pass@16": 0.9999999999999997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997894835329342, |
| "pass@2": 0.9999896859823659, |
| "pass@4": 0.9999999803542521, |
| "pass@8": 0.999999999999998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9998150887573964, |
| "pass@2": 0.9999959960164002, |
| "pass@4": 0.9999999986133391, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9969951923076923, |
| "pass@2": 0.9986194446547546, |
| "pass@4": 0.9994739897296492, |
| "pass@8": 0.9999200984240966, |
| "pass@16": 0.9999985668344405, |
| "pass@32": 0.9999999998711727, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9940140845070422, |
| "pass@2": 0.9958311384052345, |
| "pass@4": 0.9975970291565667, |
| "pass@8": 0.999027114688632, |
| "pass@16": 0.9998061149829303, |
| "pass@32": 0.9999927706810283, |
| "pass@64": 0.9999999988812277, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9672671742209632, |
| "pass@2": 0.9771213696995386, |
| "pass@4": 0.9849074195045728, |
| "pass@8": 0.9911905328339699, |
| "pass@16": 0.9959339037180583, |
| "pass@32": 0.9988444618983644, |
| "pass@64": 0.9999333712195624, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9643140889830508, |
| "pass@2": 0.9784611470705997, |
| "pass@4": 0.9880650824367292, |
| "pass@8": 0.9938518197176022, |
| "pass@16": 0.9970047759063408, |
| "pass@32": 0.9989130783745419, |
| "pass@64": 0.9998300275998665, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9476635971786834, |
| "pass@2": 0.9685683453459383, |
| "pass@4": 0.9822692721795356, |
| "pass@8": 0.9899545292765576, |
| "pass@16": 0.9938393300559409, |
| "pass@32": 0.9961250032333974, |
| "pass@64": 0.9968437006557012, |
| "pass@128": 0.9968652037617555 |
| }, |
| "14": { |
| "pass@1": 0.9345064252336449, |
| "pass@2": 0.9570436106409598, |
| "pass@4": 0.9730763464730461, |
| "pass@8": 0.9835112588310111, |
| "pass@16": 0.9902983995459784, |
| "pass@32": 0.9947020849108975, |
| "pass@64": 0.9978331269817698, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9710531069364162, |
| "pass@2": 0.985325116062082, |
| "pass@4": 0.9926385077949068, |
| "pass@8": 0.9958626814231997, |
| "pass@16": 0.9968806051910766, |
| "pass@32": 0.9970993941946336, |
| "pass@64": 0.9971098238446818, |
| "pass@128": 0.9971098265895953 |
| }, |
| "16": { |
| "pass@1": 0.8755292338709677, |
| "pass@2": 0.9045017621285244, |
| "pass@4": 0.9260232168559573, |
| "pass@8": 0.9427607291657083, |
| "pass@16": 0.9550549739603138, |
| "pass@32": 0.9636454926423886, |
| "pass@64": 0.9701787434524745, |
| "pass@128": 0.9741935483870968 |
| }, |
| "17": { |
| "pass@1": 0.9177203757225434, |
| "pass@2": 0.9430488627053847, |
| "pass@4": 0.9555040751770186, |
| "pass@8": 0.9635777719832805, |
| "pass@16": 0.9682680044397302, |
| "pass@32": 0.9713182119815579, |
| "pass@64": 0.9747075450281047, |
| "pass@128": 0.9797687861271677 |
| }, |
| "18": { |
| "pass@1": 0.8287946428571429, |
| "pass@2": 0.8780887935883015, |
| "pass@4": 0.9042536249933039, |
| "pass@8": 0.9200238424642594, |
| "pass@16": 0.9320406770143738, |
| "pass@32": 0.9415563598161735, |
| "pass@64": 0.949386455264049, |
| "pass@128": 0.9571428571428572 |
| }, |
| "19": { |
| "pass@1": 0.7182338588588588, |
| "pass@2": 0.7958574962758032, |
| "pass@4": 0.8451148347447551, |
| "pass@8": 0.8817489132436759, |
| "pass@16": 0.9093400886784007, |
| "pass@32": 0.9284539824296693, |
| "pass@64": 0.9422262642306297, |
| "pass@128": 0.9519519519519519 |
| }, |
| "20": { |
| "pass@1": 0.6150290915915916, |
| "pass@2": 0.7114929890520442, |
| "pass@4": 0.7800601017215189, |
| "pass@8": 0.8286590174245814, |
| "pass@16": 0.8633336851559491, |
| "pass@32": 0.8881871659264902, |
| "pass@64": 0.9061995892225608, |
| "pass@128": 0.9219219219219219 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 770562, |
| "answer_accuracy": 0.9384280007794232, |
| "avg_response_len": 292.01109094894775, |
| "resp_tokens_sum": 239776147, |
| "pass_at_k": { |
| "pass@1": 0.9384280007794232, |
| "pass@2": 0.956451549180363, |
| "pass@4": 0.9678319202305449, |
| "pass@8": 0.9757838381540438, |
| "pass@16": 0.9816614862816057, |
| "pass@32": 0.9862117376062051, |
| "pass@64": 0.9895825026086122, |
| "pass@128": 0.992049883086516 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9428920628834356, |
| "pass@2": 0.9549670758659001, |
| "pass@4": 0.9651627860167783, |
| "pass@8": 0.9737301217465295, |
| "pass@16": 0.9808736696825141, |
| "pass@32": 0.9874380814030427, |
| "pass@64": 0.993214871135638, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9990767045454545, |
| "pass@2": 0.9998654109997613, |
| "pass@4": 0.9999975227528377, |
| "pass@8": 0.9999999995728038, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9996995192307693, |
| "pass@2": 0.9999861680566557, |
| "pass@4": 0.9999999650561431, |
| "pass@8": 0.9999999999999813, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9968523550724637, |
| "pass@2": 0.997473396667808, |
| "pass@4": 0.9978423669867355, |
| "pass@8": 0.998410038397661, |
| "pass@16": 0.9991645323584537, |
| "pass@32": 0.9998028560467962, |
| "pass@64": 0.9999958123575, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.999955229226361, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9997064667630058, |
| "pass@2": 0.9999911104410358, |
| "pass@4": 0.9999999905178037, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9958406690140845, |
| "pass@2": 0.9978024148829987, |
| "pass@4": 0.999183896027081, |
| "pass@8": 0.9998904310030774, |
| "pass@16": 0.9999984671786155, |
| "pass@32": 0.9999999999160536, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9956025449101796, |
| "pass@2": 0.9974796961195719, |
| "pass@4": 0.998901010689532, |
| "pass@8": 0.999749379380219, |
| "pass@16": 0.9999852183129254, |
| "pass@32": 0.9999999733961134, |
| "pass@64": 0.9999999999999997, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9789156626506024, |
| "pass@2": 0.9844413332463713, |
| "pass@4": 0.9884038818378122, |
| "pass@8": 0.991249363378623, |
| "pass@16": 0.9938099352663854, |
| "pass@32": 0.9964604084001034, |
| "pass@64": 0.998383025431589, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9606191222570533, |
| "pass@2": 0.9744391460765678, |
| "pass@4": 0.9834404432792294, |
| "pass@8": 0.9887448737629764, |
| "pass@16": 0.9917958042057292, |
| "pass@32": 0.9937347694982218, |
| "pass@64": 0.9952073750268757, |
| "pass@128": 0.9968652037617555 |
| }, |
| "13": { |
| "pass@1": 0.9641981792717087, |
| "pass@2": 0.9810507372240239, |
| "pass@4": 0.9900469389855678, |
| "pass@8": 0.9942131738406755, |
| "pass@16": 0.9966176001621115, |
| "pass@32": 0.998631916393544, |
| "pass@64": 0.9998381618554224, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9547925804093568, |
| "pass@2": 0.9742169872680388, |
| "pass@4": 0.9865623400693333, |
| "pass@8": 0.993446275946928, |
| "pass@16": 0.9968301457719871, |
| "pass@32": 0.9986083975630443, |
| "pass@64": 0.9996411765505918, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.975, |
| "pass@2": 0.9886505308995464, |
| "pass@4": 0.9948184190044426, |
| "pass@8": 0.9979302980220845, |
| "pass@16": 0.9994682095514091, |
| "pass@32": 0.999964528888759, |
| "pass@64": 0.9999999629289098, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.9117699430199431, |
| "pass@2": 0.934440336384234, |
| "pass@4": 0.948974050786386, |
| "pass@8": 0.9611631506970469, |
| "pass@16": 0.9714023648928639, |
| "pass@32": 0.9789083675308171, |
| "pass@64": 0.9838666701804571, |
| "pass@128": 0.9886039886039886 |
| }, |
| "17": { |
| "pass@1": 0.9076814829192547, |
| "pass@2": 0.9357003778060351, |
| "pass@4": 0.9505333914976464, |
| "pass@8": 0.9600694844908072, |
| "pass@16": 0.9682785149952982, |
| "pass@32": 0.9759899415602024, |
| "pass@64": 0.9817656205726502, |
| "pass@128": 0.984472049689441 |
| }, |
| "18": { |
| "pass@1": 0.877734375, |
| "pass@2": 0.9222795565076425, |
| "pass@4": 0.9457145495599809, |
| "pass@8": 0.9600628143280701, |
| "pass@16": 0.9688410118307001, |
| "pass@32": 0.9746411096104598, |
| "pass@64": 0.9800388749808259, |
| "pass@128": 0.9852941176470589 |
| }, |
| "19": { |
| "pass@1": 0.7310732886904762, |
| "pass@2": 0.801264954185414, |
| "pass@4": 0.8486722577646543, |
| "pass@8": 0.8836192387270729, |
| "pass@16": 0.9101570834003377, |
| "pass@32": 0.9306870627573393, |
| "pass@64": 0.9444572773545391, |
| "pass@128": 0.9523809523809523 |
| }, |
| "20": { |
| "pass@1": 0.6313388847305389, |
| "pass@2": 0.7228336142675276, |
| "pass@4": 0.7860283833345681, |
| "pass@8": 0.8339708116905777, |
| "pass@16": 0.8714372379839609, |
| "pass@32": 0.9009187710626614, |
| "pass@64": 0.9239962462043453, |
| "pass@128": 0.9431137724550899 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 746399, |
| "answer_accuracy": 0.9316571636842946, |
| "avg_response_len": 312.3473984961655, |
| "resp_tokens_sum": 250237743, |
| "pass_at_k": { |
| "pass@1": 0.9316571636842946, |
| "pass@2": 0.9522591208816659, |
| "pass@4": 0.9660390632689512, |
| "pass@8": 0.9752605531481255, |
| "pass@16": 0.9813263861482189, |
| "pass@32": 0.9855967910692955, |
| "pass@64": 0.9888762089231591, |
| "pass@128": 0.9913724237098578 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9671641791044776, |
| "pass@2": 0.9783556381478435, |
| "pass@4": 0.9855664534470505, |
| "pass@8": 0.990385174548389, |
| "pass@16": 0.993735456511528, |
| "pass@32": 0.9956450899133359, |
| "pass@64": 0.9966503222335554, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9993146929824561, |
| "pass@2": 0.999843512570797, |
| "pass@4": 0.9999924886033983, |
| "pass@8": 0.9999999880298076, |
| "pass@16": 0.9999999999999953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9945082720588235, |
| "pass@2": 0.9969256600277908, |
| "pass@4": 0.9985720141599949, |
| "pass@8": 0.9995595644810157, |
| "pass@16": 0.9999494805984832, |
| "pass@32": 0.9999995833702632, |
| "pass@64": 0.9999999999993373, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.999728903785489, |
| "pass@2": 0.9999786538413772, |
| "pass@4": 0.9999999024175605, |
| "pass@8": 0.999999999999636, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9974040743670886, |
| "pass@2": 0.9984099347154392, |
| "pass@4": 0.9992418159953106, |
| "pass@8": 0.9998284242624276, |
| "pass@16": 0.9999927222555876, |
| "pass@32": 0.9999999948561573, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9942286036036037, |
| "pass@2": 0.9979768153980753, |
| "pass@4": 0.9997027497125922, |
| "pass@8": 0.9999930237698637, |
| "pass@16": 0.9999999974717585, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9958551045016077, |
| "pass@2": 0.9980734296275666, |
| "pass@4": 0.9994836422175525, |
| "pass@8": 0.9999617290113357, |
| "pass@16": 0.9999998358800019, |
| "pass@32": 0.9999999999993716, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9709077380952381, |
| "pass@2": 0.9824111829771282, |
| "pass@4": 0.9899338889186468, |
| "pass@8": 0.9945690319277134, |
| "pass@16": 0.9971965257441671, |
| "pass@32": 0.998626178234906, |
| "pass@64": 0.9996125152017663, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.962753249235474, |
| "pass@2": 0.9791324285439094, |
| "pass@4": 0.9909401766292976, |
| "pass@8": 0.9965767763240273, |
| "pass@16": 0.9983801502774032, |
| "pass@32": 0.9992932401756379, |
| "pass@64": 0.9999118688144722, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9471209490740741, |
| "pass@2": 0.9679142212258193, |
| "pass@4": 0.9816974689506403, |
| "pass@8": 0.9890441820692925, |
| "pass@16": 0.9926464233456719, |
| "pass@32": 0.9954950306506924, |
| "pass@64": 0.9982849094681752, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9703727744807121, |
| "pass@2": 0.9842764287717004, |
| "pass@4": 0.9920016803352103, |
| "pass@8": 0.9959930755335086, |
| "pass@16": 0.9975902111414426, |
| "pass@32": 0.998333788020753, |
| "pass@64": 0.9992640014830058, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9490981867283951, |
| "pass@2": 0.9668741494118793, |
| "pass@4": 0.9770455270058837, |
| "pass@8": 0.984529369547853, |
| "pass@16": 0.9903754117368537, |
| "pass@32": 0.9948032445307928, |
| "pass@64": 0.9978745376446474, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8951880530973452, |
| "pass@2": 0.9193018634009246, |
| "pass@4": 0.9383454012385618, |
| "pass@8": 0.9552424407044469, |
| "pass@16": 0.9682220592151869, |
| "pass@32": 0.9760900204372654, |
| "pass@64": 0.9804815886005911, |
| "pass@128": 0.9823008849557522 |
| }, |
| "17": { |
| "pass@1": 0.8645990210843374, |
| "pass@2": 0.8966012919789398, |
| "pass@4": 0.9182864334164551, |
| "pass@8": 0.9331278277256979, |
| "pass@16": 0.9432639037778405, |
| "pass@32": 0.9518367794011443, |
| "pass@64": 0.9604398598478096, |
| "pass@128": 0.9698795180722891 |
| }, |
| "18": { |
| "pass@1": 0.8597782258064516, |
| "pass@2": 0.9142490316230631, |
| "pass@4": 0.9464857808096566, |
| "pass@8": 0.9657417071551704, |
| "pass@16": 0.9759620011314518, |
| "pass@32": 0.9808473385701938, |
| "pass@64": 0.9833911490980485, |
| "pass@128": 0.9838709677419355 |
| }, |
| "19": { |
| "pass@1": 0.7302445241691843, |
| "pass@2": 0.8114428955444009, |
| "pass@4": 0.8644937821216446, |
| "pass@8": 0.9003745765960082, |
| "pass@16": 0.9237132138775155, |
| "pass@32": 0.9401485338678736, |
| "pass@64": 0.9528595262393311, |
| "pass@128": 0.9637462235649547 |
| }, |
| "20": { |
| "pass@1": 0.6025713213213213, |
| "pass@2": 0.7020823727270578, |
| "pass@4": 0.7745611930828463, |
| "pass@8": 0.8267509783091258, |
| "pass@16": 0.8656985659860957, |
| "pass@32": 0.8964622303358734, |
| "pass@64": 0.9208544430514269, |
| "pass@128": 0.93993993993994 |
| } |
| } |
| } |
| } |
| } |
| ] |
