| [ |
| { |
| "checkpoint": "checkpoint-42637", |
| "id_acc": 0.9941970486111111, |
| "ood_acc": 0.4707515625, |
| "total_acc": 0.7186994243421052, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.88271614583334, |
| "ood_avg_resp_len": 334.27673828125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 581842, |
| "answer_accuracy": 0.7185647526082832, |
| "avg_response_len": 258.66944455422066, |
| "resp_tokens_sum": 209451892, |
| "pass_at_k": { |
| "pass@1": 0.7185647526082832, |
| "pass@2": 0.75602823213036, |
| "pass@4": 0.7878101727759803, |
| "pass@8": 0.8145649755006021, |
| "pass@16": 0.8371640901307201, |
| "pass@32": 0.8566885000456224, |
| "pass@64": 0.8737977972069244, |
| "pass@128": 0.889819791337338 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9687960914454278, |
| "pass@2": 0.9793811552040507, |
| "pass@4": 0.9855387260773818, |
| "pass@8": 0.9895855761211984, |
| "pass@16": 0.993205351931165, |
| "pass@32": 0.9969276403610227, |
| "pass@64": 0.9994266373151081, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999951171875, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.999280753968254, |
| "pass@2": 0.999841426071741, |
| "pass@4": 0.9999929321334833, |
| "pass@8": 0.9999999904694469, |
| "pass@16": 0.9999999999999977, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999766092814372, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9999306582840237, |
| "pass@2": 0.999999636001491, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9983473557692307, |
| "pass@2": 0.999869081869574, |
| "pass@4": 0.9999989123955657, |
| "pass@8": 0.9999999999563497, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9925836267605633, |
| "pass@2": 0.9957299406676279, |
| "pass@4": 0.996981904286612, |
| "pass@8": 0.9980265940483249, |
| "pass@16": 0.9989089041537829, |
| "pass@32": 0.9996447874301746, |
| "pass@64": 0.9999814875425008, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9433649079320113, |
| "pass@2": 0.9598854587227589, |
| "pass@4": 0.9709821838842381, |
| "pass@8": 0.979647196562842, |
| "pass@16": 0.986569394954202, |
| "pass@32": 0.9919601338887413, |
| "pass@64": 0.9957846498939595, |
| "pass@128": 0.9971671388101983 |
| }, |
| "12": { |
| "pass@1": 0.8521142302259888, |
| "pass@2": 0.8889202839316692, |
| "pass@4": 0.9088263230867325, |
| "pass@8": 0.9222749079635884, |
| "pass@16": 0.9333221410048815, |
| "pass@32": 0.9428487904373719, |
| "pass@64": 0.9526996632881869, |
| "pass@128": 0.963276836158192 |
| }, |
| "13": { |
| "pass@1": 0.7507347178683386, |
| "pass@2": 0.8211750703477896, |
| "pass@4": 0.8696429661613606, |
| "pass@8": 0.9027677226699853, |
| "pass@16": 0.9256297663526131, |
| "pass@32": 0.943869019129676, |
| "pass@64": 0.959372529236264, |
| "pass@128": 0.9686520376175548 |
| }, |
| "14": { |
| "pass@1": 0.6630403037383178, |
| "pass@2": 0.7529734527927001, |
| "pass@4": 0.8128668262261602, |
| "pass@8": 0.8514626225366564, |
| "pass@16": 0.8785846368956592, |
| "pass@32": 0.9000179018307316, |
| "pass@64": 0.9186362233035702, |
| "pass@128": 0.9345794392523364 |
| }, |
| "15": { |
| "pass@1": 0.4444544797687861, |
| "pass@2": 0.5561496546584132, |
| "pass@4": 0.6492220088276531, |
| "pass@8": 0.7242411986701806, |
| "pass@16": 0.7859895019657533, |
| "pass@32": 0.8361443325254894, |
| "pass@64": 0.8720891570704674, |
| "pass@128": 0.8959537572254336 |
| }, |
| "16": { |
| "pass@1": 0.3129788306451613, |
| "pass@2": 0.4100731045212089, |
| "pass@4": 0.49875687041135996, |
| "pass@8": 0.5743338483660304, |
| "pass@16": 0.6368816949652301, |
| "pass@32": 0.69032613396105, |
| "pass@64": 0.7390156845678801, |
| "pass@128": 0.7903225806451613 |
| }, |
| "17": { |
| "pass@1": 0.2636831647398844, |
| "pass@2": 0.343687417504893, |
| "pass@4": 0.41615807877700245, |
| "pass@8": 0.4793526481615962, |
| "pass@16": 0.5369910222767479, |
| "pass@32": 0.5928939465721405, |
| "pass@64": 0.6439995649514992, |
| "pass@128": 0.6907514450867052 |
| }, |
| "18": { |
| "pass@1": 0.19145089285714287, |
| "pass@2": 0.2559550759280089, |
| "pass@4": 0.3198097857410683, |
| "pass@8": 0.37938951796119275, |
| "pass@16": 0.43442354487101104, |
| "pass@32": 0.4876361090120352, |
| "pass@64": 0.5388035092328448, |
| "pass@128": 0.5914285714285714 |
| }, |
| "19": { |
| "pass@1": 0.15915915915915915, |
| "pass@2": 0.22416892778605388, |
| "pass@4": 0.2921275865291613, |
| "pass@8": 0.3568417628674982, |
| "pass@16": 0.41219829331888214, |
| "pass@32": 0.4587800289416799, |
| "pass@64": 0.5010260866513289, |
| "pass@128": 0.5435435435435435 |
| }, |
| "20": { |
| "pass@1": 0.14900056306306306, |
| "pass@2": 0.2112783304958502, |
| "pass@4": 0.2804440731507661, |
| "pass@8": 0.3497426466965647, |
| "pass@16": 0.4118297340417605, |
| "pass@32": 0.4617236790192555, |
| "pass@64": 0.5047195350094015, |
| "pass@128": 0.5525525525525525 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 593599, |
| "answer_accuracy": 0.7229138250194855, |
| "avg_response_len": 266.8017342166797, |
| "resp_tokens_sum": 219076240, |
| "pass_at_k": { |
| "pass@1": 0.7229138250194855, |
| "pass@2": 0.7602572019933501, |
| "pass@4": 0.7919012047889885, |
| "pass@8": 0.8183717978819763, |
| "pass@16": 0.8400214595011362, |
| "pass@32": 0.8579900890881219, |
| "pass@64": 0.8735227023472388, |
| "pass@128": 0.8871395167575994 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9532208588957055, |
| "pass@2": 0.9648209174677551, |
| "pass@4": 0.9726692355556782, |
| "pass@8": 0.9774522644799202, |
| "pass@16": 0.9813146027266942, |
| "pass@32": 0.9854651025600398, |
| "pass@64": 0.9906248710861815, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9984601449275362, |
| "pass@2": 0.9995299840237362, |
| "pass@4": 0.9999458151970135, |
| "pass@8": 0.9999992347147649, |
| "pass@16": 0.9999999999299809, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9998871026011561, |
| "pass@2": 0.9999978665058487, |
| "pass@4": 0.9999999997290802, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9943441901408451, |
| "pass@2": 0.9970559082843518, |
| "pass@4": 0.9987552754849306, |
| "pass@8": 0.9997291043540086, |
| "pass@16": 0.9999872482571568, |
| "pass@32": 0.9999999833775981, |
| "pass@64": 0.9999999999999999, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9922342814371258, |
| "pass@2": 0.9959167540902447, |
| "pass@4": 0.9968730825750074, |
| "pass@8": 0.9970028499720828, |
| "pass@16": 0.9970059866860305, |
| "pass@32": 0.9970059880239521, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9493128765060241, |
| "pass@2": 0.9636015765819186, |
| "pass@4": 0.9711504778958353, |
| "pass@8": 0.9755672360765648, |
| "pass@16": 0.9787001525555642, |
| "pass@32": 0.9820897087991751, |
| "pass@64": 0.9854231767677829, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.8868289576802508, |
| "pass@2": 0.9180556241206526, |
| "pass@4": 0.9330513665219744, |
| "pass@8": 0.9419079793353073, |
| "pass@16": 0.9482846002018529, |
| "pass@32": 0.953817114198506, |
| "pass@64": 0.9594737760960846, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.7516850490196079, |
| "pass@2": 0.8223514799620636, |
| "pass@4": 0.8654987858660521, |
| "pass@8": 0.8950942529230584, |
| "pass@16": 0.9166506427974176, |
| "pass@32": 0.932769084530391, |
| "pass@64": 0.9444615276990928, |
| "pass@128": 0.9523809523809523 |
| }, |
| "14": { |
| "pass@1": 0.6499223318713451, |
| "pass@2": 0.7420087086153707, |
| "pass@4": 0.808748102759084, |
| "pass@8": 0.8564296041354489, |
| "pass@16": 0.8884699326583958, |
| "pass@32": 0.9099686539488756, |
| "pass@64": 0.9248935366253965, |
| "pass@128": 0.935672514619883 |
| }, |
| "15": { |
| "pass@1": 0.42935606060606063, |
| "pass@2": 0.5310013272488666, |
| "pass@4": 0.6162868610173727, |
| "pass@8": 0.6842442735294081, |
| "pass@16": 0.7353767237967629, |
| "pass@32": 0.7743462434214112, |
| "pass@64": 0.8088428752559313, |
| "pass@128": 0.8454545454545455 |
| }, |
| "16": { |
| "pass@1": 0.3317752849002849, |
| "pass@2": 0.42550846008031074, |
| "pass@4": 0.5119554006069751, |
| "pass@8": 0.5865682817959353, |
| "pass@16": 0.6453208885221663, |
| "pass@32": 0.6902534034493225, |
| "pass@64": 0.7262957676617002, |
| "pass@128": 0.7578347578347578 |
| }, |
| "17": { |
| "pass@1": 0.26188858695652173, |
| "pass@2": 0.3495890289529023, |
| "pass@4": 0.43264134044579816, |
| "pass@8": 0.5048982539994722, |
| "pass@16": 0.5673585332660139, |
| "pass@32": 0.6226776023203451, |
| "pass@64": 0.6708663372871893, |
| "pass@128": 0.7111801242236024 |
| }, |
| "18": { |
| "pass@1": 0.21394761029411766, |
| "pass@2": 0.2852800052107456, |
| "pass@4": 0.35701416413021886, |
| "pass@8": 0.4270564041129192, |
| "pass@16": 0.49366217922714584, |
| "pass@32": 0.5542598676013846, |
| "pass@64": 0.6051306373410962, |
| "pass@128": 0.6411764705882353 |
| }, |
| "19": { |
| "pass@1": 0.1627139136904762, |
| "pass@2": 0.22867563917791525, |
| "pass@4": 0.2975243234104665, |
| "pass@8": 0.360661471867346, |
| "pass@16": 0.4130810347582172, |
| "pass@32": 0.45745585728347204, |
| "pass@64": 0.5002541905062068, |
| "pass@128": 0.5446428571428571 |
| }, |
| "20": { |
| "pass@1": 0.1417243637724551, |
| "pass@2": 0.20355229501626662, |
| "pass@4": 0.26672529126224503, |
| "pass@8": 0.32590969364532546, |
| "pass@16": 0.3796096644851099, |
| "pass@32": 0.4273850373338695, |
| "pass@64": 0.47092423596150823, |
| "pass@128": 0.5089820359281437 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 572436, |
| "answer_accuracy": 0.7145160968205784, |
| "avg_response_len": 286.60102202827926, |
| "resp_tokens_sum": 229610982, |
| "pass_at_k": { |
| "pass@1": 0.7145160968205784, |
| "pass@2": 0.7527273128270038, |
| "pass@4": 0.7860822440582388, |
| "pass@8": 0.8140762477528575, |
| "pass@16": 0.8369282999686033, |
| "pass@32": 0.8557953360867006, |
| "pass@64": 0.8722638147896078, |
| "pass@128": 0.8880012781594504 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9624067164179104, |
| "pass@2": 0.975417205312023, |
| "pass@4": 0.9847762363659763, |
| "pass@8": 0.9908718089738304, |
| "pass@16": 0.9939022031152825, |
| "pass@32": 0.9952772076566918, |
| "pass@64": 0.996274435078237, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9963465073529412, |
| "pass@2": 0.9974311747336728, |
| "pass@4": 0.9981440230816737, |
| "pass@8": 0.9988677828521257, |
| "pass@16": 0.9995936566871415, |
| "pass@32": 0.9999595287156425, |
| "pass@64": 0.9999999190634002, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.999728045886076, |
| "pass@2": 0.9999957172580484, |
| "pass@4": 0.9999999985167992, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9925159534534534, |
| "pass@2": 0.9958682846232058, |
| "pass@4": 0.9980679568770121, |
| "pass@8": 0.9992814494115804, |
| "pass@16": 0.9998603327603918, |
| "pass@32": 0.9999961820571546, |
| "pass@64": 0.9999999997993955, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.995528536977492, |
| "pass@2": 0.9985944343874218, |
| "pass@4": 0.9995877176325628, |
| "pass@8": 0.9999526745076545, |
| "pass@16": 0.9999995290774132, |
| "pass@32": 0.9999999999899641, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9457589285714286, |
| "pass@2": 0.96404707224097, |
| "pass@4": 0.9766151028740455, |
| "pass@8": 0.9856252554867775, |
| "pass@16": 0.9908219708926169, |
| "pass@32": 0.9937564389934301, |
| "pass@64": 0.9964369554609406, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.8864678899082569, |
| "pass@2": 0.9235993221604183, |
| "pass@4": 0.9459036496584721, |
| "pass@8": 0.9595074062188171, |
| "pass@16": 0.9688241392990102, |
| "pass@32": 0.9766277650741706, |
| "pass@64": 0.9835963805827815, |
| "pass@128": 0.9877675840978594 |
| }, |
| "13": { |
| "pass@1": 0.7122878086419753, |
| "pass@2": 0.7814277121609797, |
| "pass@4": 0.8304292944284741, |
| "pass@8": 0.866997528599185, |
| "pass@16": 0.8936047507231509, |
| "pass@32": 0.914350728516724, |
| "pass@64": 0.9333733264169507, |
| "pass@128": 0.9537037037037037 |
| }, |
| "14": { |
| "pass@1": 0.5824137611275965, |
| "pass@2": 0.6770105609944155, |
| "pass@4": 0.7459523428451261, |
| "pass@8": 0.7958813888644206, |
| "pass@16": 0.8346850477744818, |
| "pass@32": 0.8658138954912998, |
| "pass@64": 0.8890842154249262, |
| "pass@128": 0.9050445103857567 |
| }, |
| "15": { |
| "pass@1": 0.4339795524691358, |
| "pass@2": 0.5428802736463503, |
| "pass@4": 0.636253939611715, |
| "pass@8": 0.7117330997977531, |
| "pass@16": 0.7692532223852747, |
| "pass@32": 0.8116883270736213, |
| "pass@64": 0.8456626827157813, |
| "pass@128": 0.8796296296296297 |
| }, |
| "16": { |
| "pass@1": 0.28627396755162243, |
| "pass@2": 0.379523494297726, |
| "pass@4": 0.46878642686257155, |
| "pass@8": 0.5470726852882885, |
| "pass@16": 0.6136327559761472, |
| "pass@32": 0.6708614151643482, |
| "pass@64": 0.7234624171514149, |
| "pass@128": 0.775811209439528 |
| }, |
| "17": { |
| "pass@1": 0.2334807981927711, |
| "pass@2": 0.3097336442225595, |
| "pass@4": 0.3842031367878711, |
| "pass@8": 0.4514941996906894, |
| "pass@16": 0.5112657263527319, |
| "pass@32": 0.565402163812652, |
| "pass@64": 0.6137205286978972, |
| "pass@128": 0.6566265060240963 |
| }, |
| "18": { |
| "pass@1": 0.2010836693548387, |
| "pass@2": 0.2725036512573024, |
| "pass@4": 0.34566656960218684, |
| "pass@8": 0.4135777640011059, |
| "pass@16": 0.47034295492910155, |
| "pass@32": 0.517802576979538, |
| "pass@64": 0.5624985965312049, |
| "pass@128": 0.6096774193548387 |
| }, |
| "19": { |
| "pass@1": 0.1719458081570997, |
| "pass@2": 0.2432756411018865, |
| "pass@4": 0.3147084124869588, |
| "pass@8": 0.37850946870900554, |
| "pass@16": 0.4304803723786819, |
| "pass@32": 0.47062480819864155, |
| "pass@64": 0.5040946423043504, |
| "pass@128": 0.5377643504531722 |
| }, |
| "20": { |
| "pass@1": 0.16124718468468469, |
| "pass@2": 0.22772575725331637, |
| "pass@4": 0.29506495415550527, |
| "pass@8": 0.3579550856767762, |
| "pass@16": 0.41602252358689235, |
| "pass@32": 0.46901661519061, |
| "pass@64": 0.5164136250893406, |
| "pass@128": 0.5615615615615616 |
| } |
| } |
| } |
| } |
| } |
| ] |
