| [ |
| { |
| "checkpoint": "checkpoint-42828", |
| "id_acc": 0.9903732638888889, |
| "ood_acc": 0.47812890625, |
| "total_acc": 0.7207709703947368, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.94452256944444, |
| "ood_avg_resp_len": 338.39947109375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 577160, |
| "answer_accuracy": 0.7127825640214985, |
| "avg_response_len": 258.88227775252926, |
| "resp_tokens_sum": 209624229, |
| "pass_at_k": { |
| "pass@1": 0.7127825640214985, |
| "pass@2": 0.7533919040530093, |
| "pass@4": 0.7882775814598341, |
| "pass@8": 0.8174817268476742, |
| "pass@16": 0.8412814213608717, |
| "pass@32": 0.8608748440965824, |
| "pass@64": 0.8782130919282408, |
| "pass@128": 0.8942459690167562 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9391362463126843, |
| "pass@2": 0.9502618865119741, |
| "pass@4": 0.9565611159777595, |
| "pass@8": 0.960697067871047, |
| "pass@16": 0.9641969779408918, |
| "pass@32": 0.9691040751361097, |
| "pass@64": 0.9768076556801932, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999951171875, |
| "pass@2": 0.9999996155265748, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9950892857142857, |
| "pass@2": 0.9976178758905138, |
| "pass@4": 0.9989978952035756, |
| "pass@8": 0.9997296666578273, |
| "pass@16": 0.9999812306164019, |
| "pass@32": 0.9999999574634271, |
| "pass@64": 0.9999999999999986, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9981989146706587, |
| "pass@2": 0.9989778071809138, |
| "pass@4": 0.9996588796535164, |
| "pass@8": 0.9999646694808768, |
| "pass@16": 0.9999997257540574, |
| "pass@32": 0.9999999999968382, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9969720784023669, |
| "pass@2": 0.9974847703023807, |
| "pass@4": 0.9978717981997812, |
| "pass@8": 0.9984862120344895, |
| "pass@16": 0.9992618879557866, |
| "pass@32": 0.9998528845966035, |
| "pass@64": 0.999998024451384, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9927884615384616, |
| "pass@2": 0.9952408104936402, |
| "pass@4": 0.9968641546008672, |
| "pass@8": 0.9985039928739285, |
| "pass@16": 0.9996691654274897, |
| "pass@32": 0.9999870535678677, |
| "pass@64": 0.9999999969426491, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9916593309859155, |
| "pass@2": 0.9964192636131751, |
| "pass@4": 0.9983151256621091, |
| "pass@8": 0.9993721194559138, |
| "pass@16": 0.9998884598509388, |
| "pass@32": 0.9999974897141034, |
| "pass@64": 0.9999999999243792, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9259693696883853, |
| "pass@2": 0.947184635073944, |
| "pass@4": 0.9595871744785438, |
| "pass@8": 0.9678898043700129, |
| "pass@16": 0.9737432000120178, |
| "pass@32": 0.9782974664456189, |
| "pass@64": 0.9823106914719336, |
| "pass@128": 0.9858356940509915 |
| }, |
| "12": { |
| "pass@1": 0.8313691737288136, |
| "pass@2": 0.8758886766537666, |
| "pass@4": 0.9012220606958026, |
| "pass@8": 0.918590237623188, |
| "pass@16": 0.9326627176304815, |
| "pass@32": 0.9442766994564674, |
| "pass@64": 0.9530374167763507, |
| "pass@128": 0.9576271186440678 |
| }, |
| "13": { |
| "pass@1": 0.7017535266457681, |
| "pass@2": 0.7727542702342459, |
| "pass@4": 0.8188653515685144, |
| "pass@8": 0.8522624263765721, |
| "pass@16": 0.8756735247780262, |
| "pass@32": 0.8917525794424754, |
| "pass@64": 0.9062939546723526, |
| "pass@128": 0.9216300940438872 |
| }, |
| "14": { |
| "pass@1": 0.6107135903426791, |
| "pass@2": 0.6995911981504643, |
| "pass@4": 0.7615712451831373, |
| "pass@8": 0.8077732564202801, |
| "pass@16": 0.8452598509358014, |
| "pass@32": 0.8754393054155718, |
| "pass@64": 0.8993014806085093, |
| "pass@128": 0.9190031152647975 |
| }, |
| "15": { |
| "pass@1": 0.4949421965317919, |
| "pass@2": 0.5938915217787082, |
| "pass@4": 0.6744975657847677, |
| "pass@8": 0.737580929276051, |
| "pass@16": 0.785904780521841, |
| "pass@32": 0.8238610481001554, |
| "pass@64": 0.8551167092765369, |
| "pass@128": 0.8815028901734104 |
| }, |
| "16": { |
| "pass@1": 0.3142641129032258, |
| "pass@2": 0.4167489522479044, |
| "pass@4": 0.5133889400921661, |
| "pass@8": 0.595774204307558, |
| "pass@16": 0.6626249637042165, |
| "pass@32": 0.7184575003185609, |
| "pass@64": 0.7686606169806237, |
| "pass@128": 0.8161290322580645 |
| }, |
| "17": { |
| "pass@1": 0.2604768786127168, |
| "pass@2": 0.3510835305630146, |
| "pass@4": 0.4378864013927445, |
| "pass@8": 0.5130409664619432, |
| "pass@16": 0.5733728302206639, |
| "pass@32": 0.6215663020835889, |
| "pass@64": 0.663231219544036, |
| "pass@128": 0.6965317919075145 |
| }, |
| "18": { |
| "pass@1": 0.20383928571428572, |
| "pass@2": 0.2901342800899887, |
| "pass@4": 0.37793249933044054, |
| "pass@8": 0.4567220739411263, |
| "pass@16": 0.5217816873300685, |
| "pass@32": 0.5729463732998272, |
| "pass@64": 0.6138528084122289, |
| "pass@128": 0.6485714285714286 |
| }, |
| "19": { |
| "pass@1": 0.16096565315315314, |
| "pass@2": 0.23618795961315653, |
| "pass@4": 0.31359704361279167, |
| "pass@8": 0.3814168055094861, |
| "pass@16": 0.4386634635102604, |
| "pass@32": 0.4892526277950988, |
| "pass@64": 0.5395220286288696, |
| "pass@128": 0.5945945945945946 |
| }, |
| "20": { |
| "pass@1": 0.15700075075075076, |
| "pass@2": 0.22422582523468362, |
| "pass@4": 0.2970158927206171, |
| "pass@8": 0.3685546240663337, |
| "pass@16": 0.4339257200950346, |
| "pass@32": 0.4925902703065918, |
| "pass@64": 0.547693493756059, |
| "pass@128": 0.6036036036036037 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 596632, |
| "answer_accuracy": 0.7266075604053001, |
| "avg_response_len": 269.16072559431024, |
| "resp_tokens_sum": 221013255, |
| "pass_at_k": { |
| "pass@1": 0.7266075604053001, |
| "pass@2": 0.7684823801253143, |
| "pass@4": 0.8034604694337151, |
| "pass@8": 0.8317035862100778, |
| "pass@16": 0.8539699234765759, |
| "pass@32": 0.8720368246121353, |
| "pass@64": 0.887513624518416, |
| "pass@128": 0.9011691348402182 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9294478527607362, |
| "pass@2": 0.9466284841312018, |
| "pass@4": 0.9571646012614984, |
| "pass@8": 0.9635862044908389, |
| "pass@16": 0.9679916416901503, |
| "pass@32": 0.9721634565198133, |
| "pass@64": 0.9758088391108484, |
| "pass@128": 0.9785276073619632 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999763257575758, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9968976449275362, |
| "pass@2": 0.9970971699189775, |
| "pass@4": 0.9971014479168364, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9988359598853869, |
| "pass@2": 0.9997881320082124, |
| "pass@4": 0.9999931305148461, |
| "pass@8": 0.999999995394987, |
| "pass@16": 0.9999999999999999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9967711343930635, |
| "pass@2": 0.997127250125165, |
| "pass@4": 0.9971999508009476, |
| "pass@8": 0.997290462425144, |
| "pass@16": 0.997471098265896, |
| "pass@32": 0.9978323699421965, |
| "pass@64": 0.9985549132947977, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9954445422535211, |
| "pass@2": 0.9973179133858266, |
| "pass@4": 0.9986211822113786, |
| "pass@8": 0.9995469673676058, |
| "pass@16": 0.9999426641178323, |
| "pass@32": 0.9999994185630952, |
| "pass@64": 0.9999999999982961, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9878134356287425, |
| "pass@2": 0.9916725358338442, |
| "pass@4": 0.9937421916571807, |
| "pass@8": 0.9957880813290706, |
| "pass@16": 0.9978980094262961, |
| "pass@32": 0.9993986125945581, |
| "pass@64": 0.9999581943983665, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9418768825301205, |
| "pass@2": 0.9599654770657432, |
| "pass@4": 0.9683396273507975, |
| "pass@8": 0.9735100751334598, |
| "pass@16": 0.9769297712624254, |
| "pass@32": 0.9790286004592866, |
| "pass@64": 0.9803791219396885, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8510726880877743, |
| "pass@2": 0.8924041357095258, |
| "pass@4": 0.9151241544454282, |
| "pass@8": 0.9299759245700525, |
| "pass@16": 0.9410987595365371, |
| "pass@32": 0.9503070478915217, |
| "pass@64": 0.9591426372189908, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.7162114845938375, |
| "pass@2": 0.7996261496724675, |
| "pass@4": 0.8501936338839996, |
| "pass@8": 0.8820293064882619, |
| "pass@16": 0.9039283198592027, |
| "pass@32": 0.9203175166364791, |
| "pass@64": 0.9332433569571942, |
| "pass@128": 0.9439775910364145 |
| }, |
| "14": { |
| "pass@1": 0.6420184576023392, |
| "pass@2": 0.728438190703136, |
| "pass@4": 0.7885472805482644, |
| "pass@8": 0.8277271874588323, |
| "pass@16": 0.8511195248343622, |
| "pass@32": 0.8669647904579663, |
| "pass@64": 0.8803333166221328, |
| "pass@128": 0.8918128654970761 |
| }, |
| "15": { |
| "pass@1": 0.5269412878787879, |
| "pass@2": 0.6264215730136006, |
| "pass@4": 0.6986491589119537, |
| "pass@8": 0.7511878067235214, |
| "pass@16": 0.7906412093161832, |
| "pass@32": 0.8247690661576631, |
| "pass@64": 0.856615497975502, |
| "pass@128": 0.8848484848484849 |
| }, |
| "16": { |
| "pass@1": 0.32289440883190884, |
| "pass@2": 0.42797890447988907, |
| "pass@4": 0.5235572764942842, |
| "pass@8": 0.6020844538967673, |
| "pass@16": 0.6653369741453462, |
| "pass@32": 0.7188300176879009, |
| "pass@64": 0.7655181009036083, |
| "pass@128": 0.8034188034188035 |
| }, |
| "17": { |
| "pass@1": 0.3027950310559006, |
| "pass@2": 0.4041748942387634, |
| "pass@4": 0.498130968101037, |
| "pass@8": 0.5755826570354862, |
| "pass@16": 0.6377447403306611, |
| "pass@32": 0.6878838678929854, |
| "pass@64": 0.7274527788066238, |
| "pass@128": 0.7608695652173914 |
| }, |
| "18": { |
| "pass@1": 0.2521599264705882, |
| "pass@2": 0.3417890950671606, |
| "pass@4": 0.4293763864994814, |
| "pass@8": 0.5076022232730653, |
| "pass@16": 0.5732797195419035, |
| "pass@32": 0.6261815943845891, |
| "pass@64": 0.6687111040670097, |
| "pass@128": 0.7058823529411765 |
| }, |
| "19": { |
| "pass@1": 0.17959449404761904, |
| "pass@2": 0.2535843908183353, |
| "pass@4": 0.3336726154208402, |
| "pass@8": 0.4090762902920383, |
| "pass@16": 0.4727559068833661, |
| "pass@32": 0.5258139147075629, |
| "pass@64": 0.574969613587171, |
| "pass@128": 0.625 |
| }, |
| "20": { |
| "pass@1": 0.1503555389221557, |
| "pass@2": 0.22317950221604052, |
| "pass@4": 0.30321984265439883, |
| "pass@8": 0.3782157543112791, |
| "pass@16": 0.4411765096518501, |
| "pass@32": 0.4920488219301042, |
| "pass@64": 0.5357323720323429, |
| "pass@128": 0.5748502994011976 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 579123, |
| "answer_accuracy": 0.7228628275283592, |
| "avg_response_len": 287.76802529357724, |
| "resp_tokens_sum": 230545929, |
| "pass_at_k": { |
| "pass@1": 0.7228628275283592, |
| "pass@2": 0.7634899838720332, |
| "pass@4": 0.7979183262191598, |
| "pass@8": 0.826502668879155, |
| "pass@16": 0.8497180965445399, |
| "pass@32": 0.8686404966119444, |
| "pass@64": 0.8846004907996181, |
| "pass@128": 0.8980667838312829 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9512826492537313, |
| "pass@2": 0.9646837172405689, |
| "pass@4": 0.972477203689091, |
| "pass@8": 0.9777543703411323, |
| "pass@16": 0.9816709250558314, |
| "pass@32": 0.9848746781312615, |
| "pass@64": 0.98779576845511, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.99995444606414, |
| "pass@2": 0.9999996413075917, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9876608455882353, |
| "pass@2": 0.9910252142195461, |
| "pass@4": 0.9940828544409888, |
| "pass@8": 0.9964006499766762, |
| "pass@16": 0.9978178397028429, |
| "pass@32": 0.998761342664818, |
| "pass@64": 0.9996410363492089, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997781940063092, |
| "pass@2": 0.9999980594401252, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9995302610759493, |
| "pass@2": 0.9999898771553873, |
| "pass@4": 0.9999999896175953, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9925628753753754, |
| "pass@2": 0.9967686682745736, |
| "pass@4": 0.9988088071198306, |
| "pass@8": 0.9997720961533224, |
| "pass@16": 0.9999910671569961, |
| "pass@32": 0.9999999924554084, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9900522508038585, |
| "pass@2": 0.995649590475226, |
| "pass@4": 0.9978545178838177, |
| "pass@8": 0.9989875020480404, |
| "pass@16": 0.9997189514965811, |
| "pass@32": 0.9999837050490139, |
| "pass@64": 0.9999999927877713, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9483878968253968, |
| "pass@2": 0.9712664823147109, |
| "pass@4": 0.9824181054749105, |
| "pass@8": 0.9882313135423663, |
| "pass@16": 0.9915123252403345, |
| "pass@32": 0.993682504433677, |
| "pass@64": 0.9951896239672378, |
| "pass@128": 0.9968253968253968 |
| }, |
| "12": { |
| "pass@1": 0.8620269495412844, |
| "pass@2": 0.9103235841219393, |
| "pass@4": 0.9362392352905429, |
| "pass@8": 0.9526972140428558, |
| "pass@16": 0.9638965581267493, |
| "pass@32": 0.9710619309818433, |
| "pass@64": 0.9754546286852732, |
| "pass@128": 0.9785932721712538 |
| }, |
| "13": { |
| "pass@1": 0.6687162422839507, |
| "pass@2": 0.7427760766015359, |
| "pass@4": 0.7947847190397491, |
| "pass@8": 0.8326836269325879, |
| "pass@16": 0.8608774264261173, |
| "pass@32": 0.8812346351825862, |
| "pass@64": 0.897989758263987, |
| "pass@128": 0.9135802469135802 |
| }, |
| "14": { |
| "pass@1": 0.6195752967359051, |
| "pass@2": 0.7071631346526789, |
| "pass@4": 0.7685654877339142, |
| "pass@8": 0.8150053180091682, |
| "pass@16": 0.8516327834780938, |
| "pass@32": 0.8813865048924064, |
| "pass@64": 0.9077576228327011, |
| "pass@128": 0.9287833827893175 |
| }, |
| "15": { |
| "pass@1": 0.5160590277777778, |
| "pass@2": 0.6018286885146302, |
| "pass@4": 0.668300905095196, |
| "pass@8": 0.720608796057457, |
| "pass@16": 0.7627357601757521, |
| "pass@32": 0.7963453889710991, |
| "pass@64": 0.8214735977682994, |
| "pass@128": 0.8395061728395061 |
| }, |
| "16": { |
| "pass@1": 0.32913901179941, |
| "pass@2": 0.43340657155134343, |
| "pass@4": 0.5325866487374916, |
| "pass@8": 0.6193580310655307, |
| "pass@16": 0.6897998814914108, |
| "pass@32": 0.7441026163170258, |
| "pass@64": 0.7872053937870015, |
| "pass@128": 0.8259587020648967 |
| }, |
| "17": { |
| "pass@1": 0.25230609939759036, |
| "pass@2": 0.3407472162271132, |
| "pass@4": 0.4249335446397511, |
| "pass@8": 0.4971805284165239, |
| "pass@16": 0.5591699239740667, |
| "pass@32": 0.6153184013677834, |
| "pass@64": 0.6683035385369585, |
| "pass@128": 0.7078313253012049 |
| }, |
| "18": { |
| "pass@1": 0.25740927419354837, |
| "pass@2": 0.34158663004826023, |
| "pass@4": 0.4214958056654211, |
| "pass@8": 0.4880174444049805, |
| "pass@16": 0.5401664641920964, |
| "pass@32": 0.5837977213272798, |
| "pass@64": 0.622184241601249, |
| "pass@128": 0.6548387096774193 |
| }, |
| "19": { |
| "pass@1": 0.18820808157099697, |
| "pass@2": 0.26322384149677674, |
| "pass@4": 0.3385367960991281, |
| "pass@8": 0.4080986241808218, |
| "pass@16": 0.4682389995974202, |
| "pass@32": 0.5162918392839718, |
| "pass@64": 0.5523291041867587, |
| "pass@128": 0.5800604229607251 |
| }, |
| "20": { |
| "pass@1": 0.1591356981981982, |
| "pass@2": 0.23524508760729218, |
| "pass@4": 0.31971246471443326, |
| "pass@8": 0.3994183122873029, |
| "pass@16": 0.46805212252335276, |
| "pass@32": 0.5279196875495892, |
| "pass@64": 0.582608547499917, |
| "pass@128": 0.6366366366366366 |
| } |
| } |
| } |
| } |
| } |
| ] |
