[ { "checkpoint": "checkpoint-42637", "id_acc": 0.9941970486111111, "ood_acc": 0.4707515625, "total_acc": 0.7186994243421052, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.88271614583334, "ood_avg_resp_len": 334.27673828125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 581842, "answer_accuracy": 0.7185647526082832, "avg_response_len": 258.66944455422066, "resp_tokens_sum": 209451892, "pass_at_k": { "pass@1": 0.7185647526082832, "pass@2": 0.75602823213036, "pass@4": 0.7878101727759803, "pass@8": 0.8145649755006021, "pass@16": 0.8371640901307201, "pass@32": 0.8566885000456224, "pass@64": 0.8737977972069244, "pass@128": 0.889819791337338 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9687960914454278, "pass@2": 0.9793811552040507, "pass@4": 0.9855387260773818, "pass@8": 0.9895855761211984, "pass@16": 0.993205351931165, "pass@32": 0.9969276403610227, "pass@64": 0.9994266373151081, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999951171875, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.999280753968254, "pass@2": 0.999841426071741, "pass@4": 0.9999929321334833, "pass@8": 0.9999999904694469, "pass@16": 0.9999999999999977, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9999766092814372, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9999306582840237, "pass@2": 0.999999636001491, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9983473557692307, "pass@2": 0.999869081869574, "pass@4": 0.9999989123955657, "pass@8": 0.9999999999563497, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9925836267605633, "pass@2": 0.9957299406676279, "pass@4": 0.996981904286612, "pass@8": 0.9980265940483249, "pass@16": 0.9989089041537829, "pass@32": 0.9996447874301746, "pass@64": 0.9999814875425008, "pass@128": 1.0 }, "11": { "pass@1": 0.9433649079320113, "pass@2": 0.9598854587227589, "pass@4": 0.9709821838842381, "pass@8": 0.979647196562842, "pass@16": 0.986569394954202, "pass@32": 0.9919601338887413, "pass@64": 0.9957846498939595, "pass@128": 0.9971671388101983 }, "12": { "pass@1": 0.8521142302259888, "pass@2": 0.8889202839316692, "pass@4": 0.9088263230867325, "pass@8": 0.9222749079635884, "pass@16": 0.9333221410048815, "pass@32": 0.9428487904373719, "pass@64": 0.9526996632881869, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.7507347178683386, "pass@2": 0.8211750703477896, "pass@4": 0.8696429661613606, "pass@8": 0.9027677226699853, "pass@16": 0.9256297663526131, "pass@32": 0.943869019129676, "pass@64": 0.959372529236264, "pass@128": 0.9686520376175548 }, "14": { "pass@1": 0.6630403037383178, "pass@2": 0.7529734527927001, "pass@4": 0.8128668262261602, "pass@8": 0.8514626225366564, "pass@16": 0.8785846368956592, "pass@32": 0.9000179018307316, "pass@64": 0.9186362233035702, "pass@128": 0.9345794392523364 }, "15": { "pass@1": 0.4444544797687861, "pass@2": 0.5561496546584132, "pass@4": 0.6492220088276531, "pass@8": 0.7242411986701806, "pass@16": 0.7859895019657533, "pass@32": 0.8361443325254894, "pass@64": 0.8720891570704674, "pass@128": 0.8959537572254336 }, "16": { "pass@1": 0.3129788306451613, "pass@2": 0.4100731045212089, "pass@4": 0.49875687041135996, "pass@8": 0.5743338483660304, "pass@16": 0.6368816949652301, "pass@32": 0.69032613396105, "pass@64": 0.7390156845678801, "pass@128": 0.7903225806451613 }, "17": { "pass@1": 0.2636831647398844, "pass@2": 0.343687417504893, "pass@4": 0.41615807877700245, "pass@8": 0.4793526481615962, "pass@16": 0.5369910222767479, "pass@32": 0.5928939465721405, "pass@64": 0.6439995649514992, "pass@128": 0.6907514450867052 }, "18": { "pass@1": 0.19145089285714287, "pass@2": 0.2559550759280089, "pass@4": 0.3198097857410683, "pass@8": 0.37938951796119275, "pass@16": 0.43442354487101104, "pass@32": 0.4876361090120352, "pass@64": 0.5388035092328448, "pass@128": 0.5914285714285714 }, "19": { "pass@1": 0.15915915915915915, "pass@2": 0.22416892778605388, "pass@4": 0.2921275865291613, "pass@8": 0.3568417628674982, "pass@16": 0.41219829331888214, "pass@32": 0.4587800289416799, "pass@64": 0.5010260866513289, "pass@128": 0.5435435435435435 }, "20": { "pass@1": 0.14900056306306306, "pass@2": 0.2112783304958502, "pass@4": 0.2804440731507661, "pass@8": 0.3497426466965647, "pass@16": 0.4118297340417605, "pass@32": 0.4617236790192555, "pass@64": 0.5047195350094015, "pass@128": 0.5525525525525525 } } }, "teachers_in_school": { "count": 821120, "correct": 593599, "answer_accuracy": 0.7229138250194855, "avg_response_len": 266.8017342166797, "resp_tokens_sum": 219076240, "pass_at_k": { "pass@1": 0.7229138250194855, "pass@2": 0.7602572019933501, "pass@4": 0.7919012047889885, "pass@8": 0.8183717978819763, "pass@16": 0.8400214595011362, "pass@32": 0.8579900890881219, "pass@64": 0.8735227023472388, "pass@128": 0.8871395167575994 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9532208588957055, "pass@2": 0.9648209174677551, "pass@4": 0.9726692355556782, "pass@8": 0.9774522644799202, "pass@16": 0.9813146027266942, "pass@32": 0.9854651025600398, "pass@64": 0.9906248710861815, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9984601449275362, "pass@2": 0.9995299840237362, "pass@4": 0.9999458151970135, "pass@8": 0.9999992347147649, "pass@16": 0.9999999999299809, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9998871026011561, "pass@2": 0.9999978665058487, "pass@4": 0.9999999997290802, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9943441901408451, "pass@2": 0.9970559082843518, "pass@4": 0.9987552754849306, "pass@8": 0.9997291043540086, "pass@16": 0.9999872482571568, "pass@32": 0.9999999833775981, "pass@64": 0.9999999999999999, "pass@128": 1.0 }, "9": { "pass@1": 0.9922342814371258, "pass@2": 0.9959167540902447, "pass@4": 0.9968730825750074, "pass@8": 0.9970028499720828, "pass@16": 0.9970059866860305, "pass@32": 0.9970059880239521, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9493128765060241, "pass@2": 0.9636015765819186, "pass@4": 0.9711504778958353, "pass@8": 0.9755672360765648, "pass@16": 0.9787001525555642, "pass@32": 0.9820897087991751, "pass@64": 0.9854231767677829, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.8868289576802508, "pass@2": 0.9180556241206526, "pass@4": 0.9330513665219744, "pass@8": 0.9419079793353073, "pass@16": 0.9482846002018529, "pass@32": 0.953817114198506, "pass@64": 0.9594737760960846, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.7516850490196079, "pass@2": 0.8223514799620636, "pass@4": 0.8654987858660521, "pass@8": 0.8950942529230584, "pass@16": 0.9166506427974176, "pass@32": 0.932769084530391, "pass@64": 0.9444615276990928, "pass@128": 0.9523809523809523 }, "14": { "pass@1": 0.6499223318713451, "pass@2": 0.7420087086153707, "pass@4": 0.808748102759084, "pass@8": 0.8564296041354489, "pass@16": 0.8884699326583958, "pass@32": 0.9099686539488756, "pass@64": 0.9248935366253965, "pass@128": 0.935672514619883 }, "15": { "pass@1": 0.42935606060606063, "pass@2": 0.5310013272488666, "pass@4": 0.6162868610173727, "pass@8": 0.6842442735294081, "pass@16": 0.7353767237967629, "pass@32": 0.7743462434214112, "pass@64": 0.8088428752559313, "pass@128": 0.8454545454545455 }, "16": { "pass@1": 0.3317752849002849, "pass@2": 0.42550846008031074, "pass@4": 0.5119554006069751, "pass@8": 0.5865682817959353, "pass@16": 0.6453208885221663, "pass@32": 0.6902534034493225, "pass@64": 0.7262957676617002, "pass@128": 0.7578347578347578 }, "17": { "pass@1": 0.26188858695652173, "pass@2": 0.3495890289529023, "pass@4": 0.43264134044579816, "pass@8": 0.5048982539994722, "pass@16": 0.5673585332660139, "pass@32": 0.6226776023203451, "pass@64": 0.6708663372871893, "pass@128": 0.7111801242236024 }, "18": { "pass@1": 0.21394761029411766, "pass@2": 0.2852800052107456, "pass@4": 0.35701416413021886, "pass@8": 0.4270564041129192, "pass@16": 0.49366217922714584, "pass@32": 0.5542598676013846, "pass@64": 0.6051306373410962, "pass@128": 0.6411764705882353 }, "19": { "pass@1": 0.1627139136904762, "pass@2": 0.22867563917791525, "pass@4": 0.2975243234104665, "pass@8": 0.360661471867346, "pass@16": 0.4130810347582172, "pass@32": 0.45745585728347204, "pass@64": 0.5002541905062068, "pass@128": 0.5446428571428571 }, "20": { "pass@1": 0.1417243637724551, "pass@2": 0.20355229501626662, "pass@4": 0.26672529126224503, "pass@8": 0.32590969364532546, "pass@16": 0.3796096644851099, "pass@32": 0.4273850373338695, "pass@64": 0.47092423596150823, "pass@128": 0.5089820359281437 } } }, "movie_festival_awards": { "count": 801152, "correct": 572436, "answer_accuracy": 0.7145160968205784, "avg_response_len": 286.60102202827926, "resp_tokens_sum": 229610982, "pass_at_k": { "pass@1": 0.7145160968205784, "pass@2": 0.7527273128270038, "pass@4": 0.7860822440582388, "pass@8": 0.8140762477528575, "pass@16": 0.8369282999686033, "pass@32": 0.8557953360867006, "pass@64": 0.8722638147896078, "pass@128": 0.8880012781594504 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9624067164179104, "pass@2": 0.975417205312023, "pass@4": 0.9847762363659763, "pass@8": 0.9908718089738304, "pass@16": 0.9939022031152825, "pass@32": 0.9952772076566918, "pass@64": 0.996274435078237, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9963465073529412, "pass@2": 0.9974311747336728, "pass@4": 0.9981440230816737, "pass@8": 0.9988677828521257, "pass@16": 0.9995936566871415, "pass@32": 0.9999595287156425, "pass@64": 0.9999999190634002, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.999728045886076, "pass@2": 0.9999957172580484, "pass@4": 0.9999999985167992, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9925159534534534, "pass@2": 0.9958682846232058, "pass@4": 0.9980679568770121, "pass@8": 0.9992814494115804, "pass@16": 0.9998603327603918, "pass@32": 0.9999961820571546, "pass@64": 0.9999999997993955, "pass@128": 1.0 }, "9": { "pass@1": 0.995528536977492, "pass@2": 0.9985944343874218, "pass@4": 0.9995877176325628, "pass@8": 0.9999526745076545, "pass@16": 0.9999995290774132, "pass@32": 0.9999999999899641, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9457589285714286, "pass@2": 0.96404707224097, "pass@4": 0.9766151028740455, "pass@8": 0.9856252554867775, "pass@16": 0.9908219708926169, "pass@32": 0.9937564389934301, "pass@64": 0.9964369554609406, "pass@128": 1.0 }, "12": { "pass@1": 0.8864678899082569, "pass@2": 0.9235993221604183, "pass@4": 0.9459036496584721, "pass@8": 0.9595074062188171, "pass@16": 0.9688241392990102, "pass@32": 0.9766277650741706, "pass@64": 0.9835963805827815, "pass@128": 0.9877675840978594 }, "13": { "pass@1": 0.7122878086419753, "pass@2": 0.7814277121609797, "pass@4": 0.8304292944284741, "pass@8": 0.866997528599185, "pass@16": 0.8936047507231509, "pass@32": 0.914350728516724, "pass@64": 0.9333733264169507, "pass@128": 0.9537037037037037 }, "14": { "pass@1": 0.5824137611275965, "pass@2": 0.6770105609944155, "pass@4": 0.7459523428451261, "pass@8": 0.7958813888644206, "pass@16": 0.8346850477744818, "pass@32": 0.8658138954912998, "pass@64": 0.8890842154249262, "pass@128": 0.9050445103857567 }, "15": { "pass@1": 0.4339795524691358, "pass@2": 0.5428802736463503, "pass@4": 0.636253939611715, "pass@8": 0.7117330997977531, "pass@16": 0.7692532223852747, "pass@32": 0.8116883270736213, "pass@64": 0.8456626827157813, "pass@128": 0.8796296296296297 }, "16": { "pass@1": 0.28627396755162243, "pass@2": 0.379523494297726, "pass@4": 0.46878642686257155, "pass@8": 0.5470726852882885, "pass@16": 0.6136327559761472, "pass@32": 0.6708614151643482, "pass@64": 0.7234624171514149, "pass@128": 0.775811209439528 }, "17": { "pass@1": 0.2334807981927711, "pass@2": 0.3097336442225595, "pass@4": 0.3842031367878711, "pass@8": 0.4514941996906894, "pass@16": 0.5112657263527319, "pass@32": 0.565402163812652, "pass@64": 0.6137205286978972, "pass@128": 0.6566265060240963 }, "18": { "pass@1": 0.2010836693548387, "pass@2": 0.2725036512573024, "pass@4": 0.34566656960218684, "pass@8": 0.4135777640011059, "pass@16": 0.47034295492910155, "pass@32": 0.517802576979538, "pass@64": 0.5624985965312049, "pass@128": 0.6096774193548387 }, "19": { "pass@1": 0.1719458081570997, "pass@2": 0.2432756411018865, "pass@4": 0.3147084124869588, "pass@8": 0.37850946870900554, "pass@16": 0.4304803723786819, "pass@32": 0.47062480819864155, "pass@64": 0.5040946423043504, "pass@128": 0.5377643504531722 }, "20": { "pass@1": 0.16124718468468469, "pass@2": 0.22772575725331637, "pass@4": 0.29506495415550527, "pass@8": 0.3579550856767762, "pass@16": 0.41602252358689235, "pass@32": 0.46901661519061, "pass@64": 0.5164136250893406, "pass@128": 0.5615615615615616 } } } } } ]