| [ |
| { |
| "checkpoint": "checkpoint-38645", |
| "id_acc": 0.9926675347222222, |
| "ood_acc": 0.844671875, |
| "total_acc": 0.9147750822368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.20069097222222, |
| "ood_avg_resp_len": 377.0967484375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 738018, |
| "answer_accuracy": 0.9114393969332911, |
| "avg_response_len": 280.1749056473285, |
| "resp_tokens_sum": 226865466, |
| "pass_at_k": { |
| "pass@1": 0.9114393969332911, |
| "pass@2": 0.9346978341788974, |
| "pass@4": 0.9510085321100552, |
| "pass@8": 0.9624914679892927, |
| "pass@16": 0.9708110810206952, |
| "pass@32": 0.9770193310184669, |
| "pass@64": 0.981604927339997, |
| "pass@128": 0.9851406892190958 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9615366887905604, |
| "pass@2": 0.970145808654449, |
| "pass@4": 0.9771259859110534, |
| "pass@8": 0.9837142589656627, |
| "pass@16": 0.9897806295670789, |
| "pass@32": 0.9934434017257641, |
| "pass@64": 0.9955847044260153, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997610856269113, |
| "pass@2": 0.9999864552481399, |
| "pass@4": 0.9999999638806617, |
| "pass@8": 0.9999999999999809, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99931640625, |
| "pass@2": 0.9999350239911416, |
| "pass@4": 0.9999994973284589, |
| "pass@8": 0.999999999990623, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9954861111111111, |
| "pass@2": 0.9972468285214349, |
| "pass@4": 0.9985486585010207, |
| "pass@8": 0.9994671341886722, |
| "pass@16": 0.9999236250321564, |
| "pass@32": 0.9999990493973, |
| "pass@64": 0.9999999999949286, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9995555763473054, |
| "pass@2": 0.9999499033429206, |
| "pass@4": 0.9999993320445721, |
| "pass@8": 0.9999999999490911, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9964404585798816, |
| "pass@2": 0.9978735207100592, |
| "pass@4": 0.9986706386183974, |
| "pass@8": 0.9994230191252826, |
| "pass@16": 0.9999007629980622, |
| "pass@32": 0.9999981593967747, |
| "pass@64": 0.99999999996853, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.997245592948718, |
| "pass@2": 0.9984009848828992, |
| "pass@4": 0.9992322474113812, |
| "pass@8": 0.9998262245762686, |
| "pass@16": 0.9999926289511722, |
| "pass@32": 0.9999999947902106, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9950704225352113, |
| "pass@2": 0.9983818759010756, |
| "pass@4": 0.999779442534472, |
| "pass@8": 0.9999935759719568, |
| "pass@16": 0.9999999938621692, |
| "pass@32": 0.9999999999999997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9698123229461756, |
| "pass@2": 0.981584311079387, |
| "pass@4": 0.9891040887948495, |
| "pass@8": 0.9937797636155625, |
| "pass@16": 0.9970789483598115, |
| "pass@32": 0.999223562425918, |
| "pass@64": 0.9999570534856826, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9659030720338984, |
| "pass@2": 0.9809874632990792, |
| "pass@4": 0.9906215364498931, |
| "pass@8": 0.9956137072056146, |
| "pass@16": 0.9978803736651605, |
| "pass@32": 0.9993246177441057, |
| "pass@64": 0.9999568688255516, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9498922413793104, |
| "pass@2": 0.9708758608347934, |
| "pass@4": 0.9840897281060871, |
| "pass@8": 0.9909108589255634, |
| "pass@16": 0.9943999521503089, |
| "pass@32": 0.9963005415376021, |
| "pass@64": 0.9980499904733999, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9233839563862928, |
| "pass@2": 0.944520460789364, |
| "pass@4": 0.9611923819102052, |
| "pass@8": 0.9751079976868544, |
| "pass@16": 0.9861293435926446, |
| "pass@32": 0.9935272719026856, |
| "pass@64": 0.9975030863628863, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9595375722543352, |
| "pass@2": 0.9784015719584908, |
| "pass@4": 0.987961396353635, |
| "pass@8": 0.9923778119086278, |
| "pass@16": 0.9938310119401556, |
| "pass@32": 0.9941908530213401, |
| "pass@64": 0.9942196182961773, |
| "pass@128": 0.9942196531791907 |
| }, |
| "16": { |
| "pass@1": 0.8793850806451613, |
| "pass@2": 0.9133687611125221, |
| "pass@4": 0.9350571322737883, |
| "pass@8": 0.9489252481008612, |
| "pass@16": 0.9594660322806725, |
| "pass@32": 0.9677192403291304, |
| "pass@64": 0.9722136540329196, |
| "pass@128": 0.9741935483870968 |
| }, |
| "17": { |
| "pass@1": 0.8896992413294798, |
| "pass@2": 0.9255197191752769, |
| "pass@4": 0.9464512051600483, |
| "pass@8": 0.9595738090472379, |
| "pass@16": 0.9673121876199731, |
| "pass@32": 0.9725731938539892, |
| "pass@64": 0.9768482847119901, |
| "pass@128": 0.9797687861271677 |
| }, |
| "18": { |
| "pass@1": 0.7444419642857143, |
| "pass@2": 0.8136916479190104, |
| "pass@4": 0.8625618206652741, |
| "pass@8": 0.8958082898451949, |
| "pass@16": 0.9174575405950798, |
| "pass@32": 0.9337781303810576, |
| "pass@64": 0.947397000627292, |
| "pass@128": 0.9571428571428572 |
| }, |
| "19": { |
| "pass@1": 0.6253753753753754, |
| "pass@2": 0.7184758577475104, |
| "pass@4": 0.7833039738275956, |
| "pass@8": 0.8296772377534075, |
| "pass@16": 0.8660549147719705, |
| "pass@32": 0.8940814351693145, |
| "pass@64": 0.9133406551952218, |
| "pass@128": 0.9279279279279279 |
| }, |
| "20": { |
| "pass@1": 0.4760932807807808, |
| "pass@2": 0.5768932071835613, |
| "pass@4": 0.6601997517765237, |
| "pass@8": 0.7265525943478015, |
| "pass@16": 0.7789762742387952, |
| "pass@32": 0.8215439955324653, |
| "pass@64": 0.8572865791166149, |
| "pass@128": 0.8888888888888888 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 753428, |
| "answer_accuracy": 0.9175613795791114, |
| "avg_response_len": 289.52658198558066, |
| "resp_tokens_sum": 237736067, |
| "pass_at_k": { |
| "pass@1": 0.9175613795791114, |
| "pass@2": 0.9393609304901669, |
| "pass@4": 0.9544253548538294, |
| "pass@8": 0.9651838128249977, |
| "pass@16": 0.9731036723828017, |
| "pass@32": 0.9792285383144879, |
| "pass@64": 0.9839183408625037, |
| "pass@128": 0.987373343725643 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9446654524539877, |
| "pass@2": 0.9550259498333415, |
| "pass@4": 0.9634138444542589, |
| "pass@8": 0.970866877957459, |
| "pass@16": 0.9782378206275941, |
| "pass@32": 0.9857900637806933, |
| "pass@64": 0.9923585282128631, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9988873106060606, |
| "pass@2": 0.9997856269386781, |
| "pass@4": 0.9999895822681255, |
| "pass@8": 0.9999999777033803, |
| "pass@16": 0.9999999999999803, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999537721893491, |
| "pass@2": 0.999999636001491, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9965579710144927, |
| "pass@2": 0.9972633515919205, |
| "pass@4": 0.9975327839454851, |
| "pass@8": 0.9979120550709301, |
| "pass@16": 0.9985302315822023, |
| "pass@32": 0.9993303337082865, |
| "pass@64": 0.9999164669633687, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997313753581661, |
| "pass@2": 0.9999915393813595, |
| "pass@4": 0.9999999905993125, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9975162572254336, |
| "pass@2": 0.9985822931364069, |
| "pass@4": 0.9993384239181086, |
| "pass@8": 0.9998572295938166, |
| "pass@16": 0.9999945349140869, |
| "pass@32": 0.9999999969725001, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9966769366197183, |
| "pass@2": 0.9978297937229678, |
| "pass@4": 0.9984739589769589, |
| "pass@8": 0.9991927685739189, |
| "pass@16": 0.9997893254704586, |
| "pass@32": 0.999989840105361, |
| "pass@64": 0.9999999973246758, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9973100673652695, |
| "pass@2": 0.9978701553585743, |
| "pass@4": 0.9984959510113631, |
| "pass@8": 0.9992628054872507, |
| "pass@16": 0.9998367378399854, |
| "pass@32": 0.9999945907477397, |
| "pass@64": 0.9999999995090815, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9770566641566265, |
| "pass@2": 0.9841648829570252, |
| "pass@4": 0.989164249235412, |
| "pass@8": 0.9922976666641117, |
| "pass@16": 0.9941890352635745, |
| "pass@32": 0.9952655846372421, |
| "pass@64": 0.9962408325530068, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9620640673981191, |
| "pass@2": 0.9773606620097254, |
| "pass@4": 0.9866140545127782, |
| "pass@8": 0.9920244775588233, |
| "pass@16": 0.9951791105256566, |
| "pass@32": 0.9972397387226369, |
| "pass@64": 0.9990357602586, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9582020308123249, |
| "pass@2": 0.9730195720020293, |
| "pass@4": 0.9829933805543213, |
| "pass@8": 0.9895737184859488, |
| "pass@16": 0.9944316023575817, |
| "pass@32": 0.9979028151511814, |
| "pass@64": 0.9996096734768417, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9545869883040936, |
| "pass@2": 0.9759829569001243, |
| "pass@4": 0.9895990122945156, |
| "pass@8": 0.9958761100329401, |
| "pass@16": 0.9982192698442767, |
| "pass@32": 0.9995253069447078, |
| "pass@64": 0.9999798903618302, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.964654356060606, |
| "pass@2": 0.9848753280839894, |
| "pass@4": 0.994307931394939, |
| "pass@8": 0.9984355730943542, |
| "pass@16": 0.9997865737709953, |
| "pass@32": 0.999996225078149, |
| "pass@64": 0.9999999999355342, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.9025106837606838, |
| "pass@2": 0.9249840865244409, |
| "pass@4": 0.9394740470475377, |
| "pass@8": 0.951603221594124, |
| "pass@16": 0.9618929894371758, |
| "pass@32": 0.9704062836352245, |
| "pass@64": 0.9772483460299233, |
| "pass@128": 0.9829059829059829 |
| }, |
| "17": { |
| "pass@1": 0.8692498059006211, |
| "pass@2": 0.9090858377757125, |
| "pass@4": 0.9353135909447651, |
| "pass@8": 0.9535516185363221, |
| "pass@16": 0.965466357509199, |
| "pass@32": 0.9717789256494813, |
| "pass@64": 0.9742932379404277, |
| "pass@128": 0.9751552795031055 |
| }, |
| "18": { |
| "pass@1": 0.8068244485294118, |
| "pass@2": 0.8750351001621121, |
| "pass@4": 0.9166487397714989, |
| "pass@8": 0.9411302798270424, |
| "pass@16": 0.9559175964520414, |
| "pass@32": 0.9659157855028336, |
| "pass@64": 0.9733500817730232, |
| "pass@128": 0.9794117647058823 |
| }, |
| "19": { |
| "pass@1": 0.6194428943452381, |
| "pass@2": 0.7078185051087363, |
| "pass@4": 0.7742094414537468, |
| "pass@8": 0.8272903079106774, |
| "pass@16": 0.8697325655416732, |
| "pass@32": 0.9032315125581434, |
| "pass@64": 0.9280735994123079, |
| "pass@128": 0.9464285714285714 |
| }, |
| "20": { |
| "pass@1": 0.47754491017964074, |
| "pass@2": 0.5815400449101793, |
| "pass@4": 0.6627141611041132, |
| "pass@8": 0.7251319869091699, |
| "pass@16": 0.7741932335037112, |
| "pass@32": 0.8160539579293532, |
| "pass@64": 0.8519449321090409, |
| "pass@128": 0.8802395209580839 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 733287, |
| "answer_accuracy": 0.9152907313468606, |
| "avg_response_len": 310.44483568661127, |
| "resp_tokens_sum": 248713501, |
| "pass_at_k": { |
| "pass@1": 0.9152907313468606, |
| "pass@2": 0.9382467396555126, |
| "pass@4": 0.9536234835709834, |
| "pass@8": 0.9645607797964179, |
| "pass@16": 0.9724503971693556, |
| "pass@32": 0.9781535877149617, |
| "pass@64": 0.9824174417387895, |
| "pass@128": 0.9854609362517974 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9575093283582089, |
| "pass@2": 0.9675553090844993, |
| "pass@4": 0.9764339019189767, |
| "pass@8": 0.9840698697287039, |
| "pass@16": 0.9892746985021801, |
| "pass@32": 0.9922997890645133, |
| "pass@64": 0.9946955387643621, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9988383746355685, |
| "pass@2": 0.9995426671793577, |
| "pass@4": 0.9999317049654508, |
| "pass@8": 0.999998701511788, |
| "pass@16": 0.9999999997759129, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9998629385964912, |
| "pass@2": 0.9999964025878345, |
| "pass@4": 0.9999999986295574, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9906020220588235, |
| "pass@2": 0.9926593619731356, |
| "pass@4": 0.9945639799620636, |
| "pass@8": 0.9960446992193006, |
| "pass@16": 0.9968320635603195, |
| "pass@32": 0.997048210623365, |
| "pass@64": 0.9970588207360587, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9999753548895899, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9933494857594937, |
| "pass@2": 0.9949833518140139, |
| "pass@4": 0.9964600671949234, |
| "pass@8": 0.9976420959829561, |
| "pass@16": 0.9985911723443348, |
| "pass@32": 0.9994590693089006, |
| "pass@64": 0.9999562541175095, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9913194444444444, |
| "pass@2": 0.9966260551890473, |
| "pass@4": 0.9991402473902474, |
| "pass@8": 0.9999150306942846, |
| "pass@16": 0.9999991362232366, |
| "pass@32": 0.9999999999722541, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9943478697749196, |
| "pass@2": 0.9973257462592097, |
| "pass@4": 0.9988976594967428, |
| "pass@8": 0.9997409379688414, |
| "pass@16": 0.9999841784174797, |
| "pass@32": 0.999999971428724, |
| "pass@64": 0.9999999999999997, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9710813492063493, |
| "pass@2": 0.9827689507561553, |
| "pass@4": 0.9910190773772323, |
| "pass@8": 0.9964069864008952, |
| "pass@16": 0.9991975166401181, |
| "pass@32": 0.9999605809858803, |
| "pass@64": 0.9999999808615654, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9651662844036697, |
| "pass@2": 0.9830220297864142, |
| "pass@4": 0.9943498286911382, |
| "pass@8": 0.9989944241978972, |
| "pass@16": 0.9999475847370622, |
| "pass@32": 0.9999998481010195, |
| "pass@64": 0.9999999999999678, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9363425925925926, |
| "pass@2": 0.9636441929133857, |
| "pass@4": 0.9812329428150182, |
| "pass@8": 0.9909109265469578, |
| "pass@16": 0.995563713627039, |
| "pass@32": 0.997463540610219, |
| "pass@64": 0.9984545373712423, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9556982566765578, |
| "pass@2": 0.971536645131896, |
| "pass@4": 0.9815543184142042, |
| "pass@8": 0.9879400706835424, |
| "pass@16": 0.9919984009149101, |
| "pass@32": 0.9955077699507003, |
| "pass@64": 0.9984439818222332, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9340036651234568, |
| "pass@2": 0.9567897437299503, |
| "pass@4": 0.9704516715503154, |
| "pass@8": 0.9800978369588287, |
| "pass@16": 0.9872273677362894, |
| "pass@32": 0.9925171450697239, |
| "pass@64": 0.9958359460135142, |
| "pass@128": 0.9969135802469136 |
| }, |
| "16": { |
| "pass@1": 0.8969625737463127, |
| "pass@2": 0.9258290653380717, |
| "pass@4": 0.9432881274243372, |
| "pass@8": 0.9546124219600274, |
| "pass@16": 0.9626515201381516, |
| "pass@32": 0.9696621383207606, |
| "pass@64": 0.9756120011655725, |
| "pass@128": 0.9793510324483776 |
| }, |
| "17": { |
| "pass@1": 0.8387142319277109, |
| "pass@2": 0.8761247005739499, |
| "pass@4": 0.9008967927390097, |
| "pass@8": 0.9209284437120231, |
| "pass@16": 0.937609642297589, |
| "pass@32": 0.9496274880196173, |
| "pass@64": 0.9569150319903035, |
| "pass@128": 0.9608433734939759 |
| }, |
| "18": { |
| "pass@1": 0.819304435483871, |
| "pass@2": 0.8832704946659896, |
| "pass@4": 0.9202918151360103, |
| "pass@8": 0.9459392546840153, |
| "pass@16": 0.9635935725119584, |
| "pass@32": 0.9729778671198072, |
| "pass@64": 0.9781526163741832, |
| "pass@128": 0.9838709677419355 |
| }, |
| "19": { |
| "pass@1": 0.6321988293051359, |
| "pass@2": 0.7248134827651832, |
| "pass@4": 0.7858956545650826, |
| "pass@8": 0.828820803601541, |
| "pass@16": 0.8605542666697465, |
| "pass@32": 0.8848513260783547, |
| "pass@64": 0.905053052393738, |
| "pass@128": 0.9214501510574018 |
| }, |
| "20": { |
| "pass@1": 0.5138419669669669, |
| "pass@2": 0.6109437439407909, |
| "pass@4": 0.6861744289283656, |
| "pass@8": 0.7468378163580444, |
| "pass@16": 0.7959367036137914, |
| "pass@32": 0.8357036484296656, |
| "pass@64": 0.8675307030307697, |
| "pass@128": 0.8888888888888888 |
| } |
| } |
| } |
| } |
| } |
| ] |
