| [ |
| { |
| "checkpoint": "checkpoint-87232", |
| "id_acc": 0.9914244791666667, |
| "ood_acc": 0.52209296875, |
| "total_acc": 0.7444078947368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.40281510416668, |
| "ood_avg_resp_len": 332.54644921875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 598322, |
| "answer_accuracy": 0.738917266044894, |
| "avg_response_len": 254.97740722810622, |
| "resp_tokens_sum": 206462346, |
| "pass_at_k": { |
| "pass@1": 0.738917266044894, |
| "pass@2": 0.7694304579152076, |
| "pass@4": 0.794609388738074, |
| "pass@8": 0.8157616660508631, |
| "pass@16": 0.8335155078931495, |
| "pass@32": 0.8485443383411363, |
| "pass@64": 0.861710944272157, |
| "pass@128": 0.8730635472652545 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9450589970501475, |
| "pass@2": 0.9548245766845516, |
| "pass@4": 0.9600733434316282, |
| "pass@8": 0.9635012658519264, |
| "pass@16": 0.9667523755850199, |
| "pass@32": 0.9694535779841323, |
| "pass@64": 0.9719733470168963, |
| "pass@128": 0.976401179941003 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9998566513761468, |
| "pass@2": 0.9999943563533916, |
| "pass@4": 0.9999999957000787, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9946676587301587, |
| "pass@2": 0.9956427321584803, |
| "pass@4": 0.9970027764386595, |
| "pass@8": 0.9985708586291078, |
| "pass@16": 0.9996681965873846, |
| "pass@32": 0.999983462274152, |
| "pass@64": 0.9999999928786398, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9991111526946108, |
| "pass@2": 0.9998478682398981, |
| "pass@4": 0.9999933339170928, |
| "pass@8": 0.9999999910116041, |
| "pass@16": 0.9999999999999978, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9969951923076923, |
| "pass@2": 0.9970876479289941, |
| "pass@4": 0.997133875739645, |
| "pass@8": 0.9972263313609467, |
| "pass@16": 0.9974112426035503, |
| "pass@32": 0.9977810650887574, |
| "pass@64": 0.9985207100591716, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.998848157051282, |
| "pass@2": 0.999936906925096, |
| "pass@4": 0.9999996550911907, |
| "pass@8": 0.9999999999932476, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9937940140845071, |
| "pass@2": 0.9959642203615392, |
| "pass@4": 0.9971389198709315, |
| "pass@8": 0.9984679686276483, |
| "pass@16": 0.9995073140146584, |
| "pass@32": 0.9999460220798828, |
| "pass@64": 0.9999998252071107, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9419484773371105, |
| "pass@2": 0.9577259597153754, |
| "pass@4": 0.9652531661233563, |
| "pass@8": 0.9696622029637655, |
| "pass@16": 0.9735155764884466, |
| "pass@32": 0.9768388243314153, |
| "pass@64": 0.9792149003246603, |
| "pass@128": 0.9801699716713881 |
| }, |
| "12": { |
| "pass@1": 0.8730799788135594, |
| "pass@2": 0.8947448173851152, |
| "pass@4": 0.9082170043892815, |
| "pass@8": 0.9178664664910646, |
| "pass@16": 0.9251073995443978, |
| "pass@32": 0.9317042928103167, |
| "pass@64": 0.937890439820976, |
| "pass@128": 0.943502824858757 |
| }, |
| "13": { |
| "pass@1": 0.8012588166144201, |
| "pass@2": 0.8505772838348185, |
| "pass@4": 0.8815550211395988, |
| "pass@8": 0.902041422244805, |
| "pass@16": 0.9156434397761177, |
| "pass@32": 0.9251964170363804, |
| "pass@64": 0.9315964192268797, |
| "pass@128": 0.9373040752351097 |
| }, |
| "14": { |
| "pass@1": 0.7124951323987538, |
| "pass@2": 0.780948362033017, |
| "pass@4": 0.8326437676598836, |
| "pass@8": 0.8699622045929697, |
| "pass@16": 0.8963467542274597, |
| "pass@32": 0.9151630022320785, |
| "pass@64": 0.928789217051466, |
| "pass@128": 0.9376947040498442 |
| }, |
| "15": { |
| "pass@1": 0.6236000722543352, |
| "pass@2": 0.6962096342906561, |
| "pass@4": 0.7469215516232433, |
| "pass@8": 0.7861022338881439, |
| "pass@16": 0.8184478974993373, |
| "pass@32": 0.8432674392592889, |
| "pass@64": 0.8632962541021941, |
| "pass@128": 0.8815028901734104 |
| }, |
| "16": { |
| "pass@1": 0.35897177419354837, |
| "pass@2": 0.44126793878587767, |
| "pass@4": 0.5133026022956806, |
| "pass@8": 0.576692120625883, |
| "pass@16": 0.6305196758703814, |
| "pass@32": 0.6740991125135573, |
| "pass@64": 0.7084279837224753, |
| "pass@128": 0.7322580645161291 |
| }, |
| "17": { |
| "pass@1": 0.2678829479768786, |
| "pass@2": 0.34042317145327905, |
| "pass@4": 0.40929082681500045, |
| "pass@8": 0.4708638843017089, |
| "pass@16": 0.523695917831466, |
| "pass@32": 0.5694749577617649, |
| "pass@64": 0.6097211919243859, |
| "pass@128": 0.6445086705202312 |
| }, |
| "18": { |
| "pass@1": 0.2124330357142857, |
| "pass@2": 0.2710053430821146, |
| "pass@4": 0.32689066795222027, |
| "pass@8": 0.3798762015063781, |
| "pass@16": 0.43045459996921887, |
| "pass@32": 0.47929737923180654, |
| "pass@64": 0.5266289675496978, |
| "pass@128": 0.5742857142857143 |
| }, |
| "19": { |
| "pass@1": 0.1770364114114114, |
| "pass@2": 0.2389515499751718, |
| "pass@4": 0.29822699752621007, |
| "pass@8": 0.35113589354995, |
| "pass@16": 0.3973504939392282, |
| "pass@32": 0.4402537615119897, |
| "pass@64": 0.4817467299813639, |
| "pass@128": 0.5165165165165165 |
| }, |
| "20": { |
| "pass@1": 0.17623873873873874, |
| "pass@2": 0.2368385856328771, |
| "pass@4": 0.2958934598265306, |
| "pass@8": 0.34840419959660246, |
| "pass@16": 0.3917964017371619, |
| "pass@32": 0.4274189429328129, |
| "pass@64": 0.4599680186130542, |
| "pass@128": 0.4864864864864865 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 613598, |
| "answer_accuracy": 0.7472695830085736, |
| "avg_response_len": 264.4864185502728, |
| "resp_tokens_sum": 217175088, |
| "pass_at_k": { |
| "pass@1": 0.7472695830085736, |
| "pass@2": 0.7764727148170102, |
| "pass@4": 0.8001171632384544, |
| "pass@8": 0.8200480322935797, |
| "pass@16": 0.8371993607504471, |
| "pass@32": 0.8521570331482223, |
| "pass@64": 0.8651447069991107, |
| "pass@128": 0.8760717069368668 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9302626533742331, |
| "pass@2": 0.9415724934785761, |
| "pass@4": 0.9490184284863168, |
| "pass@8": 0.9545399152772625, |
| "pass@16": 0.9589845513425366, |
| "pass@32": 0.9634418335407142, |
| "pass@64": 0.9685974560641929, |
| "pass@128": 0.9723926380368099 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9970561594202898, |
| "pass@2": 0.9971010926623303, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9999104584527221, |
| "pass@2": 0.9999996474742233, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.997154985549133, |
| "pass@2": 0.9972883289335943, |
| "pass@4": 0.9974584324900139, |
| "pass@8": 0.9977745095808112, |
| "pass@16": 0.9983173653148847, |
| "pass@32": 0.9991000149547781, |
| "pass@64": 0.9998278640025487, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9962367957746479, |
| "pass@2": 0.9972250332704892, |
| "pass@4": 0.9977350557412716, |
| "pass@8": 0.9982271286415376, |
| "pass@16": 0.9989203963246213, |
| "pass@32": 0.9996448039393536, |
| "pass@64": 0.9999814875425009, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9925383607784432, |
| "pass@2": 0.9939662996840963, |
| "pass@4": 0.994011899710141, |
| "pass@8": 0.9940119760478099, |
| "pass@16": 0.9940119760479041, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 0.9940119760479041, |
| "pass@128": 0.9940119760479041 |
| }, |
| "11": { |
| "pass@1": 0.9601609563253012, |
| "pass@2": 0.9713681250592926, |
| "pass@4": 0.9778057019980937, |
| "pass@8": 0.9825910991666997, |
| "pass@16": 0.9860410600010839, |
| "pass@32": 0.9876684351156156, |
| "pass@64": 0.9879473502047925, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.880240987460815, |
| "pass@2": 0.9021062683583047, |
| "pass@4": 0.9134986309509112, |
| "pass@8": 0.9205803424546687, |
| "pass@16": 0.9264893920886731, |
| "pass@32": 0.9330255530163245, |
| "pass@64": 0.9399906987887063, |
| "pass@128": 0.9467084639498433 |
| }, |
| "13": { |
| "pass@1": 0.7943802521008403, |
| "pass@2": 0.8459166639096585, |
| "pass@4": 0.8788433969388278, |
| "pass@8": 0.8994088755249545, |
| "pass@16": 0.9124785964224158, |
| "pass@32": 0.9226384251736293, |
| "pass@64": 0.931349262027494, |
| "pass@128": 0.938375350140056 |
| }, |
| "14": { |
| "pass@1": 0.7241639254385965, |
| "pass@2": 0.7835163696643181, |
| "pass@4": 0.822974337911708, |
| "pass@8": 0.8516282800611411, |
| "pass@16": 0.8749411010307703, |
| "pass@32": 0.8942279156943614, |
| "pass@64": 0.9076990154148188, |
| "pass@128": 0.9152046783625731 |
| }, |
| "15": { |
| "pass@1": 0.6011126893939394, |
| "pass@2": 0.6735012526843237, |
| "pass@4": 0.7223410215768484, |
| "pass@8": 0.7586458316427025, |
| "pass@16": 0.7892176036668512, |
| "pass@32": 0.8164603540597916, |
| "pass@64": 0.8405001463727434, |
| "pass@128": 0.8606060606060606 |
| }, |
| "16": { |
| "pass@1": 0.38695245726495725, |
| "pass@2": 0.46460688808129746, |
| "pass@4": 0.5310301725104875, |
| "pass@8": 0.586507777431234, |
| "pass@16": 0.6342449420868452, |
| "pass@32": 0.6763208380883311, |
| "pass@64": 0.7114651639625552, |
| "pass@128": 0.7378917378917379 |
| }, |
| "17": { |
| "pass@1": 0.2949825310559006, |
| "pass@2": 0.36418660133515907, |
| "pass@4": 0.4260861967176461, |
| "pass@8": 0.4826808908534714, |
| "pass@16": 0.5355444047838134, |
| "pass@32": 0.5855881445188759, |
| "pass@64": 0.6355138158131769, |
| "pass@128": 0.6863354037267081 |
| }, |
| "18": { |
| "pass@1": 0.27095588235294116, |
| "pass@2": 0.34281387505789707, |
| "pass@4": 0.4097198835439688, |
| "pass@8": 0.4676197190391348, |
| "pass@16": 0.5144302283585803, |
| "pass@32": 0.5544524552367253, |
| "pass@64": 0.5900773367948332, |
| "pass@128": 0.6205882352941177 |
| }, |
| "19": { |
| "pass@1": 0.18643043154761904, |
| "pass@2": 0.23665071651199854, |
| "pass@4": 0.28584336751432854, |
| "pass@8": 0.3333921116126603, |
| "pass@16": 0.3781143547100361, |
| "pass@32": 0.4184069867713272, |
| "pass@64": 0.4528938053761574, |
| "pass@128": 0.48214285714285715 |
| }, |
| "20": { |
| "pass@1": 0.16635479041916168, |
| "pass@2": 0.22205343250506857, |
| "pass@4": 0.2799436228779785, |
| "pass@8": 0.33815938365313236, |
| "pass@16": 0.3910483229426106, |
| "pass@32": 0.4332796007012806, |
| "pass@64": 0.4667753594020036, |
| "pass@128": 0.4940119760479042 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 598480, |
| "answer_accuracy": 0.7470242850295574, |
| "avg_response_len": 284.9372703307238, |
| "resp_tokens_sum": 228278064, |
| "pass_at_k": { |
| "pass@1": 0.7470242850295574, |
| "pass@2": 0.7764492752483582, |
| "pass@4": 0.8006267307884991, |
| "pass@8": 0.8210757158392421, |
| "pass@16": 0.8384563691978931, |
| "pass@32": 0.8533037383582226, |
| "pass@64": 0.8661504703510957, |
| "pass@128": 0.8771369228311232 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9533582089552238, |
| "pass@2": 0.9623622781760491, |
| "pass@4": 0.9699540755166799, |
| "pass@8": 0.9764016729742381, |
| "pass@16": 0.9810715661203923, |
| "pass@32": 0.984104975073764, |
| "pass@64": 0.9863029330743918, |
| "pass@128": 0.9880597014925373 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999772230320699, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9911305147058823, |
| "pass@2": 0.9915231299212598, |
| "pass@4": 0.9918838596462206, |
| "pass@8": 0.9925205541194468, |
| "pass@16": 0.9936037616977195, |
| "pass@32": 0.9951483049912878, |
| "pass@64": 0.9966150995831404, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9998274842271293, |
| "pass@2": 0.9999988356640751, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9994808148734177, |
| "pass@2": 0.9999630126831456, |
| "pass@4": 0.9999997030632247, |
| "pass@8": 0.9999999999933531, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9897006381381381, |
| "pass@2": 0.9923240169303161, |
| "pass@4": 0.9943014580947651, |
| "pass@8": 0.996399289945866, |
| "pass@16": 0.9983737805421303, |
| "pass@32": 0.9995889385403273, |
| "pass@64": 0.9999802281719206, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9933681672025724, |
| "pass@2": 0.9962718687495252, |
| "pass@4": 0.9975119662493954, |
| "pass@8": 0.9983480988867165, |
| "pass@16": 0.9991978005026352, |
| "pass@32": 0.999840112519781, |
| "pass@64": 0.9999978529407324, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9660962301587301, |
| "pass@2": 0.9800563992000997, |
| "pass@4": 0.9856098553157043, |
| "pass@8": 0.9871547947890077, |
| "pass@16": 0.9873004694850344, |
| "pass@32": 0.9873015872839117, |
| "pass@64": 0.9873015873015873, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.8945431957186545, |
| "pass@2": 0.9204558109995422, |
| "pass@4": 0.9357386582411145, |
| "pass@8": 0.9461271755458232, |
| "pass@16": 0.9546498872896517, |
| "pass@32": 0.9623014839946717, |
| "pass@64": 0.9687700308988774, |
| "pass@128": 0.9755351681957186 |
| }, |
| "13": { |
| "pass@1": 0.7693383487654321, |
| "pass@2": 0.8177102319675312, |
| "pass@4": 0.8476258693936403, |
| "pass@8": 0.8677171963515228, |
| "pass@16": 0.8825881522978666, |
| "pass@32": 0.8956357713705437, |
| "pass@64": 0.9076377086038764, |
| "pass@128": 0.9166666666666666 |
| }, |
| "14": { |
| "pass@1": 0.7078310459940653, |
| "pass@2": 0.7641446061823876, |
| "pass@4": 0.8041013619588346, |
| "pass@8": 0.8347883330046196, |
| "pass@16": 0.8571583652568017, |
| "pass@32": 0.8747660098007389, |
| "pass@64": 0.8917936809048442, |
| "pass@128": 0.9109792284866469 |
| }, |
| "15": { |
| "pass@1": 0.5994888117283951, |
| "pass@2": 0.6626423216195194, |
| "pass@4": 0.7061492657746482, |
| "pass@8": 0.7363345882150655, |
| "pass@16": 0.762555667288486, |
| "pass@32": 0.7893522163562189, |
| "pass@64": 0.8159024934497591, |
| "pass@128": 0.8364197530864198 |
| }, |
| "16": { |
| "pass@1": 0.38262813421828906, |
| "pass@2": 0.4605439080900284, |
| "pass@4": 0.5269444925578988, |
| "pass@8": 0.5858249902885159, |
| "pass@16": 0.6374014414272734, |
| "pass@32": 0.6810340436525093, |
| "pass@64": 0.719601323264067, |
| "pass@128": 0.7492625368731564 |
| }, |
| "17": { |
| "pass@1": 0.29045086596385544, |
| "pass@2": 0.36035369331657324, |
| "pass@4": 0.4245875250156982, |
| "pass@8": 0.48251196755235387, |
| "pass@16": 0.5349999493590595, |
| "pass@32": 0.5813152369542808, |
| "pass@64": 0.6206299947132856, |
| "pass@128": 0.6596385542168675 |
| }, |
| "18": { |
| "pass@1": 0.2664818548387097, |
| "pass@2": 0.33335899796799573, |
| "pass@4": 0.39368648868488204, |
| "pass@8": 0.44560317588205367, |
| "pass@16": 0.48800080885312824, |
| "pass@32": 0.5221118670936868, |
| "pass@64": 0.551841095020944, |
| "pass@128": 0.5806451612903226 |
| }, |
| "19": { |
| "pass@1": 0.19399074773413896, |
| "pass@2": 0.2584612960011418, |
| "pass@4": 0.3207119609482348, |
| "pass@8": 0.3767541677035014, |
| "pass@16": 0.4255612780773268, |
| "pass@32": 0.4675661315565914, |
| "pass@64": 0.5024452893007847, |
| "pass@128": 0.5287009063444109 |
| }, |
| "20": { |
| "pass@1": 0.18362894144144143, |
| "pass@2": 0.24175023941264087, |
| "pass@4": 0.30319641717082657, |
| "pass@8": 0.36417141782152096, |
| "pass@16": 0.4183361652163054, |
| "pass@32": 0.4627567871215331, |
| "pass@64": 0.49807506355210696, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| } |
| } |
| } |
| ] |
