| [ |
| { |
| "checkpoint": "checkpoint-38796", |
| "id_acc": 0.9899713541666667, |
| "ood_acc": 0.8540453125, |
| "total_acc": 0.9184313322368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.3366076388889, |
| "ood_avg_resp_len": 381.87561171875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 742004, |
| "answer_accuracy": 0.9163620376225102, |
| "avg_response_len": 282.5668360239488, |
| "resp_tokens_sum": 228802279, |
| "pass_at_k": { |
| "pass@1": 0.9163620376225102, |
| "pass@2": 0.9432990699238839, |
| "pass@4": 0.9594926690468993, |
| "pass@8": 0.9702768212202704, |
| "pass@16": 0.978067131021929, |
| "pass@32": 0.9837285212462428, |
| "pass@64": 0.9878001481817852, |
| "pass@128": 0.9908314890926335 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.940058075221239, |
| "pass@2": 0.952202445241911, |
| "pass@4": 0.9639851915634436, |
| "pass@8": 0.9747452943211968, |
| "pass@16": 0.9837117746740343, |
| "pass@32": 0.9903082598333327, |
| "pass@64": 0.9945723951890973, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999641628440367, |
| "pass@2": 0.9999793066291025, |
| "pass@4": 0.999999905401733, |
| "pass@8": 0.9999999999996472, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9995361328125, |
| "pass@2": 0.9999784694881889, |
| "pass@4": 0.9999999370195912, |
| "pass@8": 0.9999999999999016, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.991170634920635, |
| "pass@2": 0.9935695538057743, |
| "pass@4": 0.9953833833270841, |
| "pass@8": 0.996783521967192, |
| "pass@16": 0.9975425302123753, |
| "pass@32": 0.9982188768739435, |
| "pass@64": 0.9992125984251853, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999298278443114, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9981046597633136, |
| "pass@2": 0.9996745853328983, |
| "pass@4": 0.9999718887764769, |
| "pass@8": 0.9999998022755866, |
| "pass@16": 0.9999999999967328, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9924879807692307, |
| "pass@2": 0.995214390268524, |
| "pass@4": 0.9969837258924363, |
| "pass@8": 0.9983251613066434, |
| "pass@16": 0.9993038302603201, |
| "pass@32": 0.9998838431738969, |
| "pass@64": 0.9999990205966608, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9905809859154929, |
| "pass@2": 0.992608059775979, |
| "pass@4": 0.9937131769620348, |
| "pass@8": 0.9952330962332059, |
| "pass@16": 0.9970819743714158, |
| "pass@32": 0.9986530587245823, |
| "pass@64": 0.9996542900027615, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9567767351274787, |
| "pass@2": 0.9729954579420491, |
| "pass@4": 0.9837077224412106, |
| "pass@8": 0.9915053244533025, |
| "pass@16": 0.9967604348473688, |
| "pass@32": 0.9993037377604537, |
| "pass@64": 0.999959962274875, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9463497528248588, |
| "pass@2": 0.9650545370123225, |
| "pass@4": 0.9766829093397222, |
| "pass@8": 0.9848453467690281, |
| "pass@16": 0.9914138441913528, |
| "pass@32": 0.9958525342115812, |
| "pass@64": 0.9982239261012225, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9212627351097179, |
| "pass@2": 0.953259601856194, |
| "pass@4": 0.974031526035734, |
| "pass@8": 0.9872172251386195, |
| "pass@16": 0.9946975877158775, |
| "pass@32": 0.9979212628228779, |
| "pass@64": 0.9992200434190286, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9226781542056075, |
| "pass@2": 0.9488315457355219, |
| "pass@4": 0.966660107907072, |
| "pass@8": 0.9807327520370567, |
| "pass@16": 0.9900033030501925, |
| "pass@32": 0.994705328348, |
| "pass@64": 0.997677205060592, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9338872832369942, |
| "pass@2": 0.9658665826544082, |
| "pass@4": 0.9800264382595235, |
| "pass@8": 0.9852710411507393, |
| "pass@16": 0.9867086223947849, |
| "pass@32": 0.9875391269726735, |
| "pass@64": 0.9882671703608444, |
| "pass@128": 0.9884393063583815 |
| }, |
| "16": { |
| "pass@1": 0.8443296370967742, |
| "pass@2": 0.8949314992379985, |
| "pass@4": 0.9236545057875828, |
| "pass@8": 0.9436194889710628, |
| "pass@16": 0.9597137566119098, |
| "pass@32": 0.9731177144426382, |
| "pass@64": 0.9836290640887158, |
| "pass@128": 0.9935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.8775063222543352, |
| "pass@2": 0.9258678343043105, |
| "pass@4": 0.9494584514544068, |
| "pass@8": 0.9630582535369274, |
| "pass@16": 0.9726016060789089, |
| "pass@32": 0.9786961371782323, |
| "pass@64": 0.9822631761264358, |
| "pass@128": 0.9855491329479769 |
| }, |
| "18": { |
| "pass@1": 0.8074330357142857, |
| "pass@2": 0.8726863048368955, |
| "pass@4": 0.9079920633135147, |
| "pass@8": 0.9290310338171512, |
| "pass@16": 0.9431022516433624, |
| "pass@32": 0.9531360758782476, |
| "pass@64": 0.9606210100169015, |
| "pass@128": 0.9657142857142857 |
| }, |
| "19": { |
| "pass@1": 0.6631240615615616, |
| "pass@2": 0.7647959228913951, |
| "pass@4": 0.8269798164306035, |
| "pass@8": 0.8654212985928951, |
| "pass@16": 0.8938579593227545, |
| "pass@32": 0.9182400396910518, |
| "pass@64": 0.9393446013378842, |
| "pass@128": 0.9579579579579579 |
| }, |
| "20": { |
| "pass@1": 0.6333521021021021, |
| "pass@2": 0.7295261550920993, |
| "pass@4": 0.7942091112484806, |
| "pass@8": 0.8420309365275893, |
| "pass@16": 0.8790149220263148, |
| "pass@32": 0.9073538190959931, |
| "pass@64": 0.9277009935323411, |
| "pass@128": 0.93993993993994 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 757653, |
| "answer_accuracy": 0.9227067907248636, |
| "avg_response_len": 292.3765210931411, |
| "resp_tokens_sum": 240076209, |
| "pass_at_k": { |
| "pass@1": 0.9227067907248636, |
| "pass@2": 0.9491983678141017, |
| "pass@4": 0.9648446673072715, |
| "pass@8": 0.974732532480983, |
| "pass@16": 0.9813596376382586, |
| "pass@32": 0.9859942113928409, |
| "pass@64": 0.9894281143827801, |
| "pass@128": 0.9918939984411536 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9300469708588958, |
| "pass@2": 0.9466099916670692, |
| "pass@4": 0.9608256378803874, |
| "pass@8": 0.9735479075977559, |
| "pass@16": 0.9852696934862603, |
| "pass@32": 0.9940972426092773, |
| "pass@64": 0.9987733649475956, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9961647727272728, |
| "pass@2": 0.9972918157957528, |
| "pass@4": 0.9983365499767074, |
| "pass@8": 0.9991928917832018, |
| "pass@16": 0.9998063298472625, |
| "pass@32": 0.9999922499316101, |
| "pass@64": 0.9999999987964725, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.997965976331361, |
| "pass@2": 0.9991187596095608, |
| "pass@4": 0.9997857705841503, |
| "pass@8": 0.9999865024173374, |
| "pass@16": 0.9999999643806659, |
| "pass@32": 0.9999999999999731, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99375, |
| "pass@2": 0.9967833504507589, |
| "pass@4": 0.9986635988436225, |
| "pass@8": 0.9996831882834758, |
| "pass@16": 0.9999789012150662, |
| "pass@32": 0.9999999417280491, |
| "pass@64": 0.9999999999999963, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9955676934097422, |
| "pass@2": 0.9973236243034089, |
| "pass@4": 0.9983310997300121, |
| "pass@8": 0.9991142096348365, |
| "pass@16": 0.9997496114237934, |
| "pass@32": 0.9999854792844564, |
| "pass@64": 0.999999993573057, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9948067196531792, |
| "pass@2": 0.9959908089071959, |
| "pass@4": 0.9969885587820309, |
| "pass@8": 0.9979235613025731, |
| "pass@16": 0.9987206336864413, |
| "pass@32": 0.9995059858135572, |
| "pass@64": 0.9999600471131015, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9922975352112676, |
| "pass@2": 0.9943263419097261, |
| "pass@4": 0.99571091852955, |
| "pass@8": 0.9969972148105722, |
| "pass@16": 0.99814919003857, |
| "pass@32": 0.9991126715814442, |
| "pass@64": 0.9998322253384277, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.995158121257485, |
| "pass@2": 0.9969157403696542, |
| "pass@4": 0.997358424601865, |
| "pass@8": 0.9976945425380946, |
| "pass@16": 0.9982569113741001, |
| "pass@32": 0.9990676801627342, |
| "pass@64": 0.9998216794756942, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.97265625, |
| "pass@2": 0.9817142586092406, |
| "pass@4": 0.9872070927128084, |
| "pass@8": 0.9907918466392928, |
| "pass@16": 0.9932876981463241, |
| "pass@32": 0.9946033623345881, |
| "pass@64": 0.9954810039531555, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9539086990595611, |
| "pass@2": 0.9742443783476912, |
| "pass@4": 0.9873180110910899, |
| "pass@8": 0.9939946254019711, |
| "pass@16": 0.9967163926023948, |
| "pass@32": 0.9983956273931865, |
| "pass@64": 0.9996077018477316, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9435836834733894, |
| "pass@2": 0.9670878686120119, |
| "pass@4": 0.9795068925207874, |
| "pass@8": 0.9867661154939795, |
| "pass@16": 0.9912037146425846, |
| "pass@32": 0.9943074150090554, |
| "pass@64": 0.9971910964276763, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9339820906432749, |
| "pass@2": 0.961808433485288, |
| "pass@4": 0.9791213154276766, |
| "pass@8": 0.9903633423119838, |
| "pass@16": 0.9965715597469695, |
| "pass@32": 0.998978060543375, |
| "pass@64": 0.9998257129593244, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.936032196969697, |
| "pass@2": 0.9698241022429014, |
| "pass@4": 0.986383716808126, |
| "pass@8": 0.993887020431712, |
| "pass@16": 0.997341288437486, |
| "pass@32": 0.9992550901802849, |
| "pass@64": 0.999953724719699, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8875979344729344, |
| "pass@2": 0.9302362344931245, |
| "pass@4": 0.9518918415112638, |
| "pass@8": 0.9653093003879017, |
| "pass@16": 0.97478416069728, |
| "pass@32": 0.9809651521660178, |
| "pass@64": 0.986492391819359, |
| "pass@128": 0.9914529914529915 |
| }, |
| "17": { |
| "pass@1": 0.8626746894409938, |
| "pass@2": 0.9124707322590117, |
| "pass@4": 0.9368205890024611, |
| "pass@8": 0.9508498521267209, |
| "pass@16": 0.960456726487503, |
| "pass@32": 0.9678057412486193, |
| "pass@64": 0.9734389189717876, |
| "pass@128": 0.9782608695652174 |
| }, |
| "18": { |
| "pass@1": 0.8121553308823529, |
| "pass@2": 0.8809753502779064, |
| "pass@4": 0.9216372758552233, |
| "pass@8": 0.9462221099752758, |
| "pass@16": 0.9606949776103674, |
| "pass@32": 0.9685754572068952, |
| "pass@64": 0.9731564892213497, |
| "pass@128": 0.9764705882352941 |
| }, |
| "19": { |
| "pass@1": 0.66943359375, |
| "pass@2": 0.7666238985751784, |
| "pass@4": 0.8258763982627161, |
| "pass@8": 0.8655574931646675, |
| "pass@16": 0.8941625232220163, |
| "pass@32": 0.9161714667742615, |
| "pass@64": 0.9350029611318369, |
| "pass@128": 0.9523809523809523 |
| }, |
| "20": { |
| "pass@1": 0.6556652320359282, |
| "pass@2": 0.7598498868404915, |
| "pass@4": 0.8262946623625637, |
| "pass@8": 0.8690020135890967, |
| "pass@16": 0.8983220264807161, |
| "pass@32": 0.9212925306509724, |
| "pass@64": 0.9391524696479382, |
| "pass@128": 0.9491017964071856 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 733968, |
| "answer_accuracy": 0.9161407573094743, |
| "avg_response_len": 312.93695453546894, |
| "resp_tokens_sum": 250710067, |
| "pass_at_k": { |
| "pass@1": 0.9161407573094743, |
| "pass@2": 0.9444789070667229, |
| "pass@4": 0.9620470564981104, |
| "pass@8": 0.9734481996656786, |
| "pass@16": 0.9808410217800196, |
| "pass@32": 0.985989950434011, |
| "pass@64": 0.9899999651668862, |
| "pass@128": 0.993129892954146 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9534048507462687, |
| "pass@2": 0.970000514161476, |
| "pass@4": 0.9828655804218498, |
| "pass@8": 0.9913297716090343, |
| "pass@16": 0.9958306599914305, |
| "pass@32": 0.9979223480778879, |
| "pass@64": 0.9992399818731803, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9987928206997084, |
| "pass@2": 0.99955952572255, |
| "pass@4": 0.9999370614387487, |
| "pass@8": 0.9999989051962135, |
| "pass@16": 0.9999999998462147, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986979166666666, |
| "pass@2": 0.9994841310954552, |
| "pass@4": 0.9999133192232549, |
| "pass@8": 0.9999978720766682, |
| "pass@16": 0.9999999993392, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9966681985294118, |
| "pass@2": 0.9992585543075498, |
| "pass@4": 0.999930265610181, |
| "pass@8": 0.9999993599537156, |
| "pass@16": 0.9999999999763772, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.996352523659306, |
| "pass@2": 0.9972878735189645, |
| "pass@4": 0.9978065682444268, |
| "pass@8": 0.9984953578892186, |
| "pass@16": 0.9993197040358152, |
| "pass@32": 0.9998856774168334, |
| "pass@64": 0.999999036044663, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9953273338607594, |
| "pass@2": 0.9967836607943786, |
| "pass@4": 0.9975761380302146, |
| "pass@8": 0.998262935853195, |
| "pass@16": 0.9990878595282221, |
| "pass@32": 0.9997847637219769, |
| "pass@64": 0.9999954280485364, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9900525525525525, |
| "pass@2": 0.9931176254995152, |
| "pass@4": 0.9950761472946511, |
| "pass@8": 0.996932633631575, |
| "pass@16": 0.9985345359444987, |
| "pass@32": 0.9996000242336316, |
| "pass@64": 0.9999802487805222, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9923382234726688, |
| "pass@2": 0.9954557466389852, |
| "pass@4": 0.9969244255440737, |
| "pass@8": 0.9981047741525653, |
| "pass@16": 0.9990691147291293, |
| "pass@32": 0.9997813019589095, |
| "pass@64": 0.9999953545444935, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9581845238095238, |
| "pass@2": 0.973211942257218, |
| "pass@4": 0.9851733461888689, |
| "pass@8": 0.9935984850997558, |
| "pass@16": 0.9981004697081209, |
| "pass@32": 0.9997117617660194, |
| "pass@64": 0.9999952151978636, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9543434633027523, |
| "pass@2": 0.9737747266970067, |
| "pass@4": 0.9860677859648275, |
| "pass@8": 0.994249571393947, |
| "pass@16": 0.9986259898728691, |
| "pass@32": 0.9998862026515789, |
| "pass@64": 0.9999995629648154, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9164496527777778, |
| "pass@2": 0.9467269824778841, |
| "pass@4": 0.966515018955963, |
| "pass@8": 0.9796284799576312, |
| "pass@16": 0.9873678160183381, |
| "pass@32": 0.9923304170452849, |
| "pass@64": 0.9960751154460078, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9455675074183977, |
| "pass@2": 0.9688518569359099, |
| "pass@4": 0.9830591241005852, |
| "pass@8": 0.9918079912361211, |
| "pass@16": 0.9964965731596999, |
| "pass@32": 0.9989974984823478, |
| "pass@64": 0.999910515524606, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9213927469135802, |
| "pass@2": 0.9536509216729855, |
| "pass@4": 0.9729866107593029, |
| "pass@8": 0.9846111586848604, |
| "pass@16": 0.9907910506066029, |
| "pass@32": 0.9946077911424446, |
| "pass@64": 0.9975593713245301, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8636845501474927, |
| "pass@2": 0.9176269656005392, |
| "pass@4": 0.9461433612280757, |
| "pass@8": 0.9632940350686785, |
| "pass@16": 0.9731921968948374, |
| "pass@32": 0.9788198865927932, |
| "pass@64": 0.9817049567519268, |
| "pass@128": 0.9823008849557522 |
| }, |
| "17": { |
| "pass@1": 0.816382718373494, |
| "pass@2": 0.8697396623897163, |
| "pass@4": 0.8989555943120052, |
| "pass@8": 0.918514496266187, |
| "pass@16": 0.9335748802757035, |
| "pass@32": 0.9474461636352892, |
| "pass@64": 0.9617706386333493, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8135332661290322, |
| "pass@2": 0.8886441929133857, |
| "pass@4": 0.9286830575613528, |
| "pass@8": 0.9493191452530526, |
| "pass@16": 0.9612486799216202, |
| "pass@32": 0.9699809772790603, |
| "pass@64": 0.9777500114141688, |
| "pass@128": 0.9838709677419355 |
| }, |
| "19": { |
| "pass@1": 0.6800651435045317, |
| "pass@2": 0.7825676636058709, |
| "pass@4": 0.8478114498338757, |
| "pass@8": 0.890404706092961, |
| "pass@16": 0.9181360078721577, |
| "pass@32": 0.9381630090208142, |
| "pass@64": 0.9558007487757464, |
| "pass@128": 0.9697885196374623 |
| }, |
| "20": { |
| "pass@1": 0.6123545420420421, |
| "pass@2": 0.7181795476578936, |
| "pass@4": 0.7932386998922434, |
| "pass@8": 0.8470943217645397, |
| "pass@16": 0.8868402120185244, |
| "pass@32": 0.9171541099551569, |
| "pass@64": 0.9405105380920781, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| } |
| ] |
