| [ |
| { |
| "checkpoint": "checkpoint-85656", |
| "id_acc": 0.9933975694444445, |
| "ood_acc": 0.48817109375, |
| "total_acc": 0.7274888980263158, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.00196440972223, |
| "ood_avg_resp_len": 334.2508890625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 586420, |
| "answer_accuracy": 0.7242185030034777, |
| "avg_response_len": 256.7089454730477, |
| "resp_tokens_sum": 207864421, |
| "pass_at_k": { |
| "pass@1": 0.7242185030034777, |
| "pass@2": 0.7585805035959495, |
| "pass@4": 0.7882152405970935, |
| "pass@8": 0.8131007504036403, |
| "pass@16": 0.8335299105303158, |
| "pass@32": 0.8507363433743848, |
| "pass@64": 0.8662116073517198, |
| "pass@128": 0.8808093582042364 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9612140486725663, |
| "pass@2": 0.9701686729147794, |
| "pass@4": 0.97611373799514, |
| "pass@8": 0.9795318507801245, |
| "pass@16": 0.9815704330658933, |
| "pass@32": 0.9828295616605529, |
| "pass@64": 0.9837715482760134, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9966269841269841, |
| "pass@2": 0.9969132764654418, |
| "pass@4": 0.9970214452360122, |
| "pass@8": 0.9972112860892366, |
| "pass@16": 0.9975721784776902, |
| "pass@32": 0.9982189726284215, |
| "pass@64": 0.9992125984251968, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999766092814372, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9993296967455622, |
| "pass@2": 0.9998813364860458, |
| "pass@4": 0.9999958538836491, |
| "pass@8": 0.9999999967670793, |
| "pass@16": 0.9999999999999998, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9997495993589743, |
| "pass@2": 0.9999948736876642, |
| "pass@4": 0.999999998497784, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9941901408450704, |
| "pass@2": 0.9947054868581569, |
| "pass@4": 0.9950140287921757, |
| "pass@8": 0.9955275550723595, |
| "pass@16": 0.996246252427952, |
| "pass@32": 0.9969218386791731, |
| "pass@64": 0.9971743778471076, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.9172716005665722, |
| "pass@2": 0.9294245332470837, |
| "pass@4": 0.9375553278927952, |
| "pass@8": 0.9436644091134152, |
| "pass@16": 0.94896922068123, |
| "pass@32": 0.9540347827409994, |
| "pass@64": 0.9584288926634769, |
| "pass@128": 0.9603399433427762 |
| }, |
| "12": { |
| "pass@1": 0.8369085451977402, |
| "pass@2": 0.8658925413719473, |
| "pass@4": 0.8836768536877804, |
| "pass@8": 0.8958159717383948, |
| "pass@16": 0.9059099640119125, |
| "pass@32": 0.9152839590889877, |
| "pass@64": 0.9230974982514136, |
| "pass@128": 0.9293785310734464 |
| }, |
| "13": { |
| "pass@1": 0.7135579937304075, |
| "pass@2": 0.7642971083355962, |
| "pass@4": 0.7975250877731188, |
| "pass@8": 0.8216605066521206, |
| "pass@16": 0.8408669871052371, |
| "pass@32": 0.8585289603051615, |
| "pass@64": 0.8786083453443386, |
| "pass@128": 0.9028213166144201 |
| }, |
| "14": { |
| "pass@1": 0.6607038551401869, |
| "pass@2": 0.7342098081781829, |
| "pass@4": 0.7866001411038571, |
| "pass@8": 0.8223529852870952, |
| "pass@16": 0.8463869020391322, |
| "pass@32": 0.8634275011957604, |
| "pass@64": 0.8774430319994344, |
| "pass@128": 0.8909657320872274 |
| }, |
| "15": { |
| "pass@1": 0.5573744580924855, |
| "pass@2": 0.6510904999772427, |
| "pass@4": 0.723555908654481, |
| "pass@8": 0.779664294770354, |
| "pass@16": 0.8218733604726072, |
| "pass@32": 0.8537991239873666, |
| "pass@64": 0.8786629030021994, |
| "pass@128": 0.8988439306358381 |
| }, |
| "16": { |
| "pass@1": 0.32026209677419354, |
| "pass@2": 0.4102227266954534, |
| "pass@4": 0.4922931598267957, |
| "pass@8": 0.5608738002824728, |
| "pass@16": 0.6161741026713436, |
| "pass@32": 0.6634223589324809, |
| "pass@64": 0.7052034534598515, |
| "pass@128": 0.7419354838709677 |
| }, |
| "17": { |
| "pass@1": 0.26948609104046245, |
| "pass@2": 0.3551862398161212, |
| "pass@4": 0.43630361992323174, |
| "pass@8": 0.505574472344914, |
| "pass@16": 0.5626248447366223, |
| "pass@32": 0.612875860834388, |
| "pass@64": 0.6613171736457973, |
| "pass@128": 0.7052023121387283 |
| }, |
| "18": { |
| "pass@1": 0.22223214285714285, |
| "pass@2": 0.29839848143981995, |
| "pass@4": 0.37236364186619514, |
| "pass@8": 0.43890352397685345, |
| "pass@16": 0.49460411827912604, |
| "pass@32": 0.5408992361679525, |
| "pass@64": 0.5819298205174266, |
| "pass@128": 0.6228571428571429 |
| }, |
| "19": { |
| "pass@1": 0.1733765015015015, |
| "pass@2": 0.24151267409141433, |
| "pass@4": 0.3106473104150269, |
| "pass@8": 0.3760434973464308, |
| "pass@16": 0.4339640400717074, |
| "pass@32": 0.48326526375921236, |
| "pass@64": 0.5279349721453175, |
| "pass@128": 0.5735735735735735 |
| }, |
| "20": { |
| "pass@1": 0.16915352852852852, |
| "pass@2": 0.22966285675439213, |
| "pass@4": 0.29331480636992446, |
| "pass@8": 0.3554372484491605, |
| "pass@16": 0.4113463579552269, |
| "pass@32": 0.4595926430033481, |
| "pass@64": 0.5028409960228297, |
| "pass@128": 0.5435435435435435 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 601723, |
| "answer_accuracy": 0.7328076286048324, |
| "avg_response_len": 266.3788885911925, |
| "resp_tokens_sum": 218729033, |
| "pass_at_k": { |
| "pass@1": 0.7328076286048324, |
| "pass@2": 0.7669891057192438, |
| "pass@4": 0.7962575649205126, |
| "pass@8": 0.8205455374066479, |
| "pass@16": 0.8399386311082003, |
| "pass@32": 0.8559486775608024, |
| "pass@64": 0.8701074575696454, |
| "pass@128": 0.883398285268901 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9406873082822086, |
| "pass@2": 0.9511727241437608, |
| "pass@4": 0.9582467286489494, |
| "pass@8": 0.962884319554284, |
| "pass@16": 0.9658664939297285, |
| "pass@32": 0.9683840162934146, |
| "pass@64": 0.9708284840200408, |
| "pass@128": 0.9723926380368099 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9996866045845272, |
| "pass@2": 0.9999725029894185, |
| "pass@4": 0.9999998079573864, |
| "pass@8": 0.9999999999974207, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9996387283236994, |
| "pass@2": 0.9999765315643349, |
| "pass@4": 0.9999998658946533, |
| "pass@8": 0.9999999999989994, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9976012323943662, |
| "pass@2": 0.9993276588665854, |
| "pass@4": 0.9999400980687273, |
| "pass@8": 0.9999995286192314, |
| "pass@16": 0.9999999999862863, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9954622005988024, |
| "pass@2": 0.996530438139469, |
| "pass@4": 0.9974659918071619, |
| "pass@8": 0.9982248854431038, |
| "pass@16": 0.9990042031981354, |
| "pass@32": 0.9997223135661357, |
| "pass@64": 0.999990730945279, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9490540286144579, |
| "pass@2": 0.9586703111659238, |
| "pass@4": 0.963866821334833, |
| "pass@8": 0.9669982809187331, |
| "pass@16": 0.969628195342412, |
| "pass@32": 0.9726172988672541, |
| "pass@64": 0.9751752015368585, |
| "pass@128": 0.9759036144578314 |
| }, |
| "12": { |
| "pass@1": 0.8521012931034483, |
| "pass@2": 0.8769631044356135, |
| "pass@4": 0.893521940635163, |
| "pass@8": 0.9075129211754909, |
| "pass@16": 0.919181931341527, |
| "pass@32": 0.9290936707076372, |
| "pass@64": 0.9385951394414912, |
| "pass@128": 0.9498432601880877 |
| }, |
| "13": { |
| "pass@1": 0.7197785364145658, |
| "pass@2": 0.7765865204349457, |
| "pass@4": 0.8138981214533052, |
| "pass@8": 0.840127920923921, |
| "pass@16": 0.8606898149555469, |
| "pass@32": 0.8784119819890157, |
| "pass@64": 0.8928222471865029, |
| "pass@128": 0.9047619047619048 |
| }, |
| "14": { |
| "pass@1": 0.6990131578947368, |
| "pass@2": 0.7748933727034123, |
| "pass@4": 0.8276604991262051, |
| "pass@8": 0.8608615541084115, |
| "pass@16": 0.8813606703486647, |
| "pass@32": 0.8960207950792911, |
| "pass@64": 0.9067741271436445, |
| "pass@128": 0.9152046783625731 |
| }, |
| "15": { |
| "pass@1": 0.5314867424242424, |
| "pass@2": 0.6243128877356237, |
| "pass@4": 0.6995647515651452, |
| "pass@8": 0.75847718932693, |
| "pass@16": 0.8041517108171061, |
| "pass@32": 0.8403438310227098, |
| "pass@64": 0.8678819774825477, |
| "pass@128": 0.8878787878787879 |
| }, |
| "16": { |
| "pass@1": 0.3511173433048433, |
| "pass@2": 0.4367120375978642, |
| "pass@4": 0.509192721636291, |
| "pass@8": 0.567463683171276, |
| "pass@16": 0.6138049647381483, |
| "pass@32": 0.6543388540897418, |
| "pass@64": 0.6920704030785279, |
| "pass@128": 0.7236467236467237 |
| }, |
| "17": { |
| "pass@1": 0.2860539596273292, |
| "pass@2": 0.3759124199149017, |
| "pass@4": 0.4613117808061256, |
| "pass@8": 0.5321531449577279, |
| "pass@16": 0.5842617561652651, |
| "pass@32": 0.6244032611091793, |
| "pass@64": 0.6603615858501944, |
| "pass@128": 0.6987577639751553 |
| }, |
| "18": { |
| "pass@1": 0.2511948529411765, |
| "pass@2": 0.3300544233441406, |
| "pass@4": 0.40649941000022055, |
| "pass@8": 0.4779186811576755, |
| "pass@16": 0.5402423482438927, |
| "pass@32": 0.5939876301314176, |
| "pass@64": 0.6458831650585033, |
| "pass@128": 0.6970588235294117 |
| }, |
| "19": { |
| "pass@1": 0.18038504464285715, |
| "pass@2": 0.24445003456599168, |
| "pass@4": 0.31143215635991933, |
| "pass@8": 0.3728059388521139, |
| "pass@16": 0.42353768253638485, |
| "pass@32": 0.4649215111100438, |
| "pass@64": 0.500723873663761, |
| "pass@128": 0.5386904761904762 |
| }, |
| "20": { |
| "pass@1": 0.1533261601796407, |
| "pass@2": 0.21130733120373435, |
| "pass@4": 0.2719651745440502, |
| "pass@8": 0.332414175156139, |
| "pass@16": 0.3859289673999605, |
| "pass@32": 0.4304629843267275, |
| "pass@64": 0.4713483469991754, |
| "pass@128": 0.5119760479041916 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 581110, |
| "answer_accuracy": 0.7253430060712573, |
| "avg_response_len": 284.832275273606, |
| "resp_tokens_sum": 228193947, |
| "pass_at_k": { |
| "pass@1": 0.7253430060712573, |
| "pass@2": 0.75873765399871, |
| "pass@4": 0.7882019855861954, |
| "pass@8": 0.8138588630545672, |
| "pass@16": 0.8355427556244789, |
| "pass@32": 0.8539066962141767, |
| "pass@64": 0.8698343314586575, |
| "pass@128": 0.8838472599456783 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9534048507462687, |
| "pass@2": 0.9626968503937007, |
| "pass@4": 0.9692538331962235, |
| "pass@8": 0.9737725528077535, |
| "pass@16": 0.9771025187908413, |
| "pass@32": 0.9801357911203106, |
| "pass@64": 0.9834547978205753, |
| "pass@128": 0.9880597014925373 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9966452205882353, |
| "pass@2": 0.9975343040759611, |
| "pass@4": 0.9980915448068992, |
| "pass@8": 0.9987831969997992, |
| "pass@16": 0.9995279850408865, |
| "pass@32": 0.9999439253289022, |
| "pass@64": 0.9999998174959024, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999507097791798, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9998022151898734, |
| "pass@2": 0.999996106598226, |
| "pass@4": 0.9999999985167994, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9918355855855856, |
| "pass@2": 0.9945075488874701, |
| "pass@4": 0.9965787761890124, |
| "pass@8": 0.998290842921839, |
| "pass@16": 0.9994100203212533, |
| "pass@32": 0.9999208020070317, |
| "pass@64": 0.99999958431727, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9956541398713826, |
| "pass@2": 0.9983808105678912, |
| "pass@4": 0.999648498318739, |
| "pass@8": 0.9999806352115692, |
| "pass@16": 0.9999999483522362, |
| "pass@32": 0.9999999999999448, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9447420634920635, |
| "pass@2": 0.9581868672665916, |
| "pass@4": 0.967509306872355, |
| "pass@8": 0.9740486174543461, |
| "pass@16": 0.9786847931164305, |
| "pass@32": 0.9824150336632566, |
| "pass@64": 0.9857183739073222, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.8612146406727829, |
| "pass@2": 0.8891486220472444, |
| "pass@4": 0.9075370982870257, |
| "pass@8": 0.921338707735371, |
| "pass@16": 0.9329246372544424, |
| "pass@32": 0.9447859108257943, |
| "pass@64": 0.9570330678851205, |
| "pass@128": 0.9694189602446484 |
| }, |
| "13": { |
| "pass@1": 0.6789400077160493, |
| "pass@2": 0.73740597963449, |
| "pass@4": 0.7799996001657193, |
| "pass@8": 0.8137514624128047, |
| "pass@16": 0.8405120187983943, |
| "pass@32": 0.8611789198572768, |
| "pass@64": 0.8775786513793102, |
| "pass@128": 0.8919753086419753 |
| }, |
| "14": { |
| "pass@1": 0.6469538204747775, |
| "pass@2": 0.7170783049837612, |
| "pass@4": 0.7698086949072013, |
| "pass@8": 0.8095645184136512, |
| "pass@16": 0.8393733146970238, |
| "pass@32": 0.8621325130569488, |
| "pass@64": 0.8802217668630712, |
| "pass@128": 0.8931750741839762 |
| }, |
| "15": { |
| "pass@1": 0.5352527006172839, |
| "pass@2": 0.6160802165354328, |
| "pass@4": 0.6814122117837121, |
| "pass@8": 0.734581191554782, |
| "pass@16": 0.7757085390245353, |
| "pass@32": 0.8070609426882411, |
| "pass@64": 0.8332631723379745, |
| "pass@128": 0.8549382716049383 |
| }, |
| "16": { |
| "pass@1": 0.3416758849557522, |
| "pass@2": 0.42799572619794185, |
| "pass@4": 0.5040647457784591, |
| "pass@8": 0.5700512110067496, |
| "pass@16": 0.6254832399008291, |
| "pass@32": 0.6707192692761402, |
| "pass@64": 0.7088023579005431, |
| "pass@128": 0.7433628318584071 |
| }, |
| "17": { |
| "pass@1": 0.2613422439759036, |
| "pass@2": 0.3423844245090598, |
| "pass@4": 0.41293712645858993, |
| "pass@8": 0.46923112448581195, |
| "pass@16": 0.5187456803719004, |
| "pass@32": 0.5701175454546831, |
| "pass@64": 0.6236549755551729, |
| "pass@128": 0.6716867469879518 |
| }, |
| "18": { |
| "pass@1": 0.2354334677419355, |
| "pass@2": 0.3043799212598424, |
| "pass@4": 0.37419252270885495, |
| "pass@8": 0.4401254353269728, |
| "pass@16": 0.4954914482495745, |
| "pass@32": 0.5391664434428325, |
| "pass@64": 0.5774079786759414, |
| "pass@128": 0.6193548387096774 |
| }, |
| "19": { |
| "pass@1": 0.17546261329305135, |
| "pass@2": 0.24499771035040552, |
| "pass@4": 0.31780488079095853, |
| "pass@8": 0.38831339902772993, |
| "pass@16": 0.45257478979141497, |
| "pass@32": 0.5076430412025037, |
| "pass@64": 0.5521427562189815, |
| "pass@128": 0.5891238670694864 |
| }, |
| "20": { |
| "pass@1": 0.1487424924924925, |
| "pass@2": 0.2120966931498427, |
| "pass@4": 0.28508847148610916, |
| "pass@8": 0.3607474325394686, |
| "pass@16": 0.42986908543734814, |
| "pass@32": 0.48956023306096935, |
| "pass@64": 0.5385710498391255, |
| "pass@128": 0.5765765765765766 |
| } |
| } |
| } |
| } |
| } |
| ] |
