[ { "checkpoint": "checkpoint-42828", "id_acc": 0.9903732638888889, "ood_acc": 0.47812890625, "total_acc": 0.7207709703947368, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.94452256944444, "ood_avg_resp_len": 338.39947109375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 577160, "answer_accuracy": 0.7127825640214985, "avg_response_len": 258.88227775252926, "resp_tokens_sum": 209624229, "pass_at_k": { "pass@1": 0.7127825640214985, "pass@2": 0.7533919040530093, "pass@4": 0.7882775814598341, "pass@8": 0.8174817268476742, "pass@16": 0.8412814213608717, "pass@32": 0.8608748440965824, "pass@64": 0.8782130919282408, "pass@128": 0.8942459690167562 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9391362463126843, "pass@2": 0.9502618865119741, "pass@4": 0.9565611159777595, "pass@8": 0.960697067871047, "pass@16": 0.9641969779408918, "pass@32": 0.9691040751361097, "pass@64": 0.9768076556801932, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999951171875, "pass@2": 0.9999996155265748, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9950892857142857, "pass@2": 0.9976178758905138, "pass@4": 0.9989978952035756, "pass@8": 0.9997296666578273, "pass@16": 0.9999812306164019, "pass@32": 0.9999999574634271, "pass@64": 0.9999999999999986, "pass@128": 1.0 }, "6": { "pass@1": 0.9981989146706587, "pass@2": 0.9989778071809138, "pass@4": 0.9996588796535164, "pass@8": 0.9999646694808768, "pass@16": 0.9999997257540574, "pass@32": 0.9999999999968382, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9969720784023669, "pass@2": 0.9974847703023807, "pass@4": 0.9978717981997812, "pass@8": 0.9984862120344895, "pass@16": 0.9992618879557866, "pass@32": 0.9998528845966035, "pass@64": 0.999998024451384, "pass@128": 1.0 }, "8": { "pass@1": 0.9927884615384616, "pass@2": 0.9952408104936402, "pass@4": 0.9968641546008672, "pass@8": 0.9985039928739285, "pass@16": 0.9996691654274897, "pass@32": 0.9999870535678677, "pass@64": 0.9999999969426491, "pass@128": 1.0 }, "9": { "pass@1": 0.9916593309859155, "pass@2": 0.9964192636131751, "pass@4": 0.9983151256621091, "pass@8": 0.9993721194559138, "pass@16": 0.9998884598509388, "pass@32": 0.9999974897141034, "pass@64": 0.9999999999243792, "pass@128": 1.0 }, "11": { "pass@1": 0.9259693696883853, "pass@2": 0.947184635073944, "pass@4": 0.9595871744785438, "pass@8": 0.9678898043700129, "pass@16": 0.9737432000120178, "pass@32": 0.9782974664456189, "pass@64": 0.9823106914719336, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.8313691737288136, "pass@2": 0.8758886766537666, "pass@4": 0.9012220606958026, "pass@8": 0.918590237623188, "pass@16": 0.9326627176304815, "pass@32": 0.9442766994564674, "pass@64": 0.9530374167763507, "pass@128": 0.9576271186440678 }, "13": { "pass@1": 0.7017535266457681, "pass@2": 0.7727542702342459, "pass@4": 0.8188653515685144, "pass@8": 0.8522624263765721, "pass@16": 0.8756735247780262, "pass@32": 0.8917525794424754, "pass@64": 0.9062939546723526, "pass@128": 0.9216300940438872 }, "14": { "pass@1": 0.6107135903426791, "pass@2": 0.6995911981504643, "pass@4": 0.7615712451831373, "pass@8": 0.8077732564202801, "pass@16": 0.8452598509358014, "pass@32": 0.8754393054155718, "pass@64": 0.8993014806085093, "pass@128": 0.9190031152647975 }, "15": { "pass@1": 0.4949421965317919, "pass@2": 0.5938915217787082, "pass@4": 0.6744975657847677, "pass@8": 0.737580929276051, "pass@16": 0.785904780521841, "pass@32": 0.8238610481001554, "pass@64": 0.8551167092765369, "pass@128": 0.8815028901734104 }, "16": { "pass@1": 0.3142641129032258, "pass@2": 0.4167489522479044, "pass@4": 0.5133889400921661, "pass@8": 0.595774204307558, "pass@16": 0.6626249637042165, "pass@32": 0.7184575003185609, "pass@64": 0.7686606169806237, "pass@128": 0.8161290322580645 }, "17": { "pass@1": 0.2604768786127168, "pass@2": 0.3510835305630146, "pass@4": 0.4378864013927445, "pass@8": 0.5130409664619432, "pass@16": 0.5733728302206639, "pass@32": 0.6215663020835889, "pass@64": 0.663231219544036, "pass@128": 0.6965317919075145 }, "18": { "pass@1": 0.20383928571428572, "pass@2": 0.2901342800899887, "pass@4": 0.37793249933044054, "pass@8": 0.4567220739411263, "pass@16": 0.5217816873300685, "pass@32": 0.5729463732998272, "pass@64": 0.6138528084122289, "pass@128": 0.6485714285714286 }, "19": { "pass@1": 0.16096565315315314, "pass@2": 0.23618795961315653, "pass@4": 0.31359704361279167, "pass@8": 0.3814168055094861, "pass@16": 0.4386634635102604, "pass@32": 0.4892526277950988, "pass@64": 0.5395220286288696, "pass@128": 0.5945945945945946 }, "20": { "pass@1": 0.15700075075075076, "pass@2": 0.22422582523468362, "pass@4": 0.2970158927206171, "pass@8": 0.3685546240663337, "pass@16": 0.4339257200950346, "pass@32": 0.4925902703065918, "pass@64": 0.547693493756059, "pass@128": 0.6036036036036037 } } }, "teachers_in_school": { "count": 821120, "correct": 596632, "answer_accuracy": 0.7266075604053001, "avg_response_len": 269.16072559431024, "resp_tokens_sum": 221013255, "pass_at_k": { "pass@1": 0.7266075604053001, "pass@2": 0.7684823801253143, "pass@4": 0.8034604694337151, "pass@8": 0.8317035862100778, "pass@16": 0.8539699234765759, "pass@32": 0.8720368246121353, "pass@64": 0.887513624518416, "pass@128": 0.9011691348402182 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9294478527607362, "pass@2": 0.9466284841312018, "pass@4": 0.9571646012614984, "pass@8": 0.9635862044908389, "pass@16": 0.9679916416901503, "pass@32": 0.9721634565198133, "pass@64": 0.9758088391108484, "pass@128": 0.9785276073619632 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999763257575758, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9968976449275362, "pass@2": 0.9970971699189775, "pass@4": 0.9971014479168364, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9988359598853869, "pass@2": 0.9997881320082124, "pass@4": 0.9999931305148461, "pass@8": 0.999999995394987, "pass@16": 0.9999999999999999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9967711343930635, "pass@2": 0.997127250125165, "pass@4": 0.9971999508009476, "pass@8": 0.997290462425144, "pass@16": 0.997471098265896, "pass@32": 0.9978323699421965, "pass@64": 0.9985549132947977, "pass@128": 1.0 }, "8": { "pass@1": 0.9954445422535211, "pass@2": 0.9973179133858266, "pass@4": 0.9986211822113786, "pass@8": 0.9995469673676058, "pass@16": 0.9999426641178323, "pass@32": 0.9999994185630952, "pass@64": 0.9999999999982961, "pass@128": 1.0 }, "9": { "pass@1": 0.9878134356287425, "pass@2": 0.9916725358338442, "pass@4": 0.9937421916571807, "pass@8": 0.9957880813290706, "pass@16": 0.9978980094262961, "pass@32": 0.9993986125945581, "pass@64": 0.9999581943983665, "pass@128": 1.0 }, "11": { "pass@1": 0.9418768825301205, "pass@2": 0.9599654770657432, "pass@4": 0.9683396273507975, "pass@8": 0.9735100751334598, "pass@16": 0.9769297712624254, "pass@32": 0.9790286004592866, "pass@64": 0.9803791219396885, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8510726880877743, "pass@2": 0.8924041357095258, "pass@4": 0.9151241544454282, "pass@8": 0.9299759245700525, "pass@16": 0.9410987595365371, "pass@32": 0.9503070478915217, "pass@64": 0.9591426372189908, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.7162114845938375, "pass@2": 0.7996261496724675, "pass@4": 0.8501936338839996, "pass@8": 0.8820293064882619, "pass@16": 0.9039283198592027, "pass@32": 0.9203175166364791, "pass@64": 0.9332433569571942, "pass@128": 0.9439775910364145 }, "14": { "pass@1": 0.6420184576023392, "pass@2": 0.728438190703136, "pass@4": 0.7885472805482644, "pass@8": 0.8277271874588323, "pass@16": 0.8511195248343622, "pass@32": 0.8669647904579663, "pass@64": 0.8803333166221328, "pass@128": 0.8918128654970761 }, "15": { "pass@1": 0.5269412878787879, "pass@2": 0.6264215730136006, "pass@4": 0.6986491589119537, "pass@8": 0.7511878067235214, "pass@16": 0.7906412093161832, "pass@32": 0.8247690661576631, "pass@64": 0.856615497975502, "pass@128": 0.8848484848484849 }, "16": { "pass@1": 0.32289440883190884, "pass@2": 0.42797890447988907, "pass@4": 0.5235572764942842, "pass@8": 0.6020844538967673, "pass@16": 0.6653369741453462, "pass@32": 0.7188300176879009, "pass@64": 0.7655181009036083, "pass@128": 0.8034188034188035 }, "17": { "pass@1": 0.3027950310559006, "pass@2": 0.4041748942387634, "pass@4": 0.498130968101037, "pass@8": 0.5755826570354862, "pass@16": 0.6377447403306611, "pass@32": 0.6878838678929854, "pass@64": 0.7274527788066238, "pass@128": 0.7608695652173914 }, "18": { "pass@1": 0.2521599264705882, "pass@2": 0.3417890950671606, "pass@4": 0.4293763864994814, "pass@8": 0.5076022232730653, "pass@16": 0.5732797195419035, "pass@32": 0.6261815943845891, "pass@64": 0.6687111040670097, "pass@128": 0.7058823529411765 }, "19": { "pass@1": 0.17959449404761904, "pass@2": 0.2535843908183353, "pass@4": 0.3336726154208402, "pass@8": 0.4090762902920383, "pass@16": 0.4727559068833661, "pass@32": 0.5258139147075629, "pass@64": 0.574969613587171, "pass@128": 0.625 }, "20": { "pass@1": 0.1503555389221557, "pass@2": 0.22317950221604052, "pass@4": 0.30321984265439883, "pass@8": 0.3782157543112791, "pass@16": 0.4411765096518501, "pass@32": 0.4920488219301042, "pass@64": 0.5357323720323429, "pass@128": 0.5748502994011976 } } }, "movie_festival_awards": { "count": 801152, "correct": 579123, "answer_accuracy": 0.7228628275283592, "avg_response_len": 287.76802529357724, "resp_tokens_sum": 230545929, "pass_at_k": { "pass@1": 0.7228628275283592, "pass@2": 0.7634899838720332, "pass@4": 0.7979183262191598, "pass@8": 0.826502668879155, "pass@16": 0.8497180965445399, "pass@32": 0.8686404966119444, "pass@64": 0.8846004907996181, "pass@128": 0.8980667838312829 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9512826492537313, "pass@2": 0.9646837172405689, "pass@4": 0.972477203689091, "pass@8": 0.9777543703411323, "pass@16": 0.9816709250558314, "pass@32": 0.9848746781312615, "pass@64": 0.98779576845511, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.99995444606414, "pass@2": 0.9999996413075917, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9876608455882353, "pass@2": 0.9910252142195461, "pass@4": 0.9940828544409888, "pass@8": 0.9964006499766762, "pass@16": 0.9978178397028429, "pass@32": 0.998761342664818, "pass@64": 0.9996410363492089, "pass@128": 1.0 }, "6": { "pass@1": 0.9997781940063092, "pass@2": 0.9999980594401252, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9995302610759493, "pass@2": 0.9999898771553873, "pass@4": 0.9999999896175953, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9925628753753754, "pass@2": 0.9967686682745736, "pass@4": 0.9988088071198306, "pass@8": 0.9997720961533224, "pass@16": 0.9999910671569961, "pass@32": 0.9999999924554084, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9900522508038585, "pass@2": 0.995649590475226, "pass@4": 0.9978545178838177, "pass@8": 0.9989875020480404, "pass@16": 0.9997189514965811, "pass@32": 0.9999837050490139, "pass@64": 0.9999999927877713, "pass@128": 1.0 }, "11": { "pass@1": 0.9483878968253968, "pass@2": 0.9712664823147109, "pass@4": 0.9824181054749105, "pass@8": 0.9882313135423663, "pass@16": 0.9915123252403345, "pass@32": 0.993682504433677, "pass@64": 0.9951896239672378, "pass@128": 0.9968253968253968 }, "12": { "pass@1": 0.8620269495412844, "pass@2": 0.9103235841219393, "pass@4": 0.9362392352905429, "pass@8": 0.9526972140428558, "pass@16": 0.9638965581267493, "pass@32": 0.9710619309818433, "pass@64": 0.9754546286852732, "pass@128": 0.9785932721712538 }, "13": { "pass@1": 0.6687162422839507, "pass@2": 0.7427760766015359, "pass@4": 0.7947847190397491, "pass@8": 0.8326836269325879, "pass@16": 0.8608774264261173, "pass@32": 0.8812346351825862, "pass@64": 0.897989758263987, "pass@128": 0.9135802469135802 }, "14": { "pass@1": 0.6195752967359051, "pass@2": 0.7071631346526789, "pass@4": 0.7685654877339142, "pass@8": 0.8150053180091682, "pass@16": 0.8516327834780938, "pass@32": 0.8813865048924064, "pass@64": 0.9077576228327011, "pass@128": 0.9287833827893175 }, "15": { "pass@1": 0.5160590277777778, "pass@2": 0.6018286885146302, "pass@4": 0.668300905095196, "pass@8": 0.720608796057457, "pass@16": 0.7627357601757521, "pass@32": 0.7963453889710991, "pass@64": 0.8214735977682994, "pass@128": 0.8395061728395061 }, "16": { "pass@1": 0.32913901179941, "pass@2": 0.43340657155134343, "pass@4": 0.5325866487374916, "pass@8": 0.6193580310655307, "pass@16": 0.6897998814914108, "pass@32": 0.7441026163170258, "pass@64": 0.7872053937870015, "pass@128": 0.8259587020648967 }, "17": { "pass@1": 0.25230609939759036, "pass@2": 0.3407472162271132, "pass@4": 0.4249335446397511, "pass@8": 0.4971805284165239, "pass@16": 0.5591699239740667, "pass@32": 0.6153184013677834, "pass@64": 0.6683035385369585, "pass@128": 0.7078313253012049 }, "18": { "pass@1": 0.25740927419354837, "pass@2": 0.34158663004826023, "pass@4": 0.4214958056654211, "pass@8": 0.4880174444049805, "pass@16": 0.5401664641920964, "pass@32": 0.5837977213272798, "pass@64": 0.622184241601249, "pass@128": 0.6548387096774193 }, "19": { "pass@1": 0.18820808157099697, "pass@2": 0.26322384149677674, "pass@4": 0.3385367960991281, "pass@8": 0.4080986241808218, "pass@16": 0.4682389995974202, "pass@32": 0.5162918392839718, "pass@64": 0.5523291041867587, "pass@128": 0.5800604229607251 }, "20": { "pass@1": 0.1591356981981982, "pass@2": 0.23524508760729218, "pass@4": 0.31971246471443326, "pass@8": 0.3994183122873029, "pass@16": 0.46805212252335276, "pass@32": 0.5279196875495892, "pass@64": 0.582608547499917, "pass@128": 0.6366366366366366 } } } } } ]