| [ |
| { |
| "checkpoint": "checkpoint-21546", |
| "id_acc": 0.9743967013888889, |
| "ood_acc": 0.39801640625, |
| "total_acc": 0.6710386513157894, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.49886805555556, |
| "ood_avg_resp_len": 321.86122421875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 540343, |
| "answer_accuracy": 0.6673142092159342, |
| "avg_response_len": 252.07113746838445, |
| "resp_tokens_sum": 204109058, |
| "pass_at_k": { |
| "pass@1": 0.6673142092159342, |
| "pass@2": 0.7184972467083713, |
| "pass@4": 0.7582710489269354, |
| "pass@8": 0.7898036372954599, |
| "pass@16": 0.8149666354226288, |
| "pass@32": 0.8356683187114132, |
| "pass@64": 0.8536326257414687, |
| "pass@128": 0.8695858362314258 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9078401548672567, |
| "pass@2": 0.9451660598564567, |
| "pass@4": 0.9627415558121603, |
| "pass@8": 0.9724228068912785, |
| "pass@16": 0.9786434853777529, |
| "pass@32": 0.9833303999293327, |
| "pass@64": 0.9874800169721869, |
| "pass@128": 0.9911504424778761 |
| }, |
| "2": { |
| "pass@1": 0.9968847352024922, |
| "pass@2": 0.9968847352024922, |
| "pass@4": 0.9968847352024922, |
| "pass@8": 0.9968847352024922, |
| "pass@16": 0.9968847352024922, |
| "pass@32": 0.9968847352024922, |
| "pass@64": 0.9968847352024922, |
| "pass@128": 0.9968847352024922 |
| }, |
| "3": { |
| "pass@1": 0.9974913990825688, |
| "pass@2": 0.998993925931277, |
| "pass@4": 0.999781667199857, |
| "pass@8": 0.9999876849218283, |
| "pass@16": 0.9999999719746717, |
| "pass@32": 0.9999999999999853, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9863525390625, |
| "pass@2": 0.9885619156003939, |
| "pass@4": 0.9898306295697413, |
| "pass@8": 0.9915581459135234, |
| "pass@16": 0.9938875771099299, |
| "pass@32": 0.996529537062657, |
| "pass@64": 0.9989487041172099, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9679315476190476, |
| "pass@2": 0.9789893450818652, |
| "pass@4": 0.9852945253867076, |
| "pass@8": 0.9903654763094896, |
| "pass@16": 0.9944361907939756, |
| "pass@32": 0.9969451642580184, |
| "pass@64": 0.9983683333808681, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9894975673652695, |
| "pass@2": 0.996165027229007, |
| "pass@4": 0.998533353147473, |
| "pass@8": 0.999620189417031, |
| "pass@16": 0.9999636861019171, |
| "pass@32": 0.9999998029761207, |
| "pass@64": 0.999999999999911, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9860392011834319, |
| "pass@2": 0.9959920124167174, |
| "pass@4": 0.9977027054310518, |
| "pass@8": 0.9983763394978522, |
| "pass@16": 0.9991472297394309, |
| "pass@32": 0.9997987731838602, |
| "pass@64": 0.999995725631176, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9701522435897436, |
| "pass@2": 0.9889413612961848, |
| "pass@4": 0.9943698822502955, |
| "pass@8": 0.9962621361835249, |
| "pass@16": 0.9970686948727937, |
| "pass@32": 0.9975932975941004, |
| "pass@64": 0.998397435811393, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9600132042253521, |
| "pass@2": 0.9878528058112463, |
| "pass@4": 0.9946557779749362, |
| "pass@8": 0.9974029094078771, |
| "pass@16": 0.9989479245309896, |
| "pass@32": 0.9997375771890578, |
| "pass@64": 0.9999912792521506, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8549043909348442, |
| "pass@2": 0.9105383272735389, |
| "pass@4": 0.9394676640101288, |
| "pass@8": 0.9567641672474922, |
| "pass@16": 0.9689281622131964, |
| "pass@32": 0.9792995108322007, |
| "pass@64": 0.989377221593619, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.7604387358757062, |
| "pass@2": 0.8512497080608568, |
| "pass@4": 0.9022677093859018, |
| "pass@8": 0.9306185243521176, |
| "pass@16": 0.947629076857396, |
| "pass@32": 0.9613728628093047, |
| "pass@64": 0.9722807480889516, |
| "pass@128": 0.9774011299435028 |
| }, |
| "13": { |
| "pass@1": 0.6151792711598746, |
| "pass@2": 0.7272553717325303, |
| "pass@4": 0.8012909736204601, |
| "pass@8": 0.8513849982189186, |
| "pass@16": 0.8858777402360082, |
| "pass@32": 0.9113844472956131, |
| "pass@64": 0.9327076612903673, |
| "pass@128": 0.9529780564263323 |
| }, |
| "14": { |
| "pass@1": 0.4572381230529595, |
| "pass@2": 0.5804936437559792, |
| "pass@4": 0.6836558835519382, |
| "pass@8": 0.7584661690159947, |
| "pass@16": 0.8090695107690499, |
| "pass@32": 0.8452244630136615, |
| "pass@64": 0.8744698913107548, |
| "pass@128": 0.9034267912772586 |
| }, |
| "15": { |
| "pass@1": 0.3591943641618497, |
| "pass@2": 0.46582035694779467, |
| "pass@4": 0.557234378759013, |
| "pass@8": 0.6333810374573632, |
| "pass@16": 0.6991402754904769, |
| "pass@32": 0.7537907497738942, |
| "pass@64": 0.7989936415700499, |
| "pass@128": 0.838150289017341 |
| }, |
| "16": { |
| "pass@1": 0.2480594758064516, |
| "pass@2": 0.3369277051054099, |
| "pass@4": 0.41882028587152403, |
| "pass@8": 0.48785088132261173, |
| "pass@16": 0.5464977216622237, |
| "pass@32": 0.5997006252636178, |
| "pass@64": 0.6464878664953922, |
| "pass@128": 0.6838709677419355 |
| }, |
| "17": { |
| "pass@1": 0.19413836705202311, |
| "pass@2": 0.2639743866915478, |
| "pass@4": 0.3301886702384742, |
| "pass@8": 0.3913913054415934, |
| "pass@16": 0.44674849852516724, |
| "pass@32": 0.49537678199969953, |
| "pass@64": 0.5397606358748235, |
| "pass@128": 0.5809248554913294 |
| }, |
| "18": { |
| "pass@1": 0.16417410714285716, |
| "pass@2": 0.23900625703037112, |
| "pass@4": 0.31737553055868006, |
| "pass@8": 0.3877756897146819, |
| "pass@16": 0.44484807064643955, |
| "pass@32": 0.4911672177360723, |
| "pass@64": 0.5336450417048393, |
| "pass@128": 0.5771428571428572 |
| }, |
| "19": { |
| "pass@1": 0.1477805930930931, |
| "pass@2": 0.21373120167411505, |
| "pass@4": 0.2819873895267596, |
| "pass@8": 0.3444790109299255, |
| "pass@16": 0.398624559518731, |
| "pass@32": 0.4448169543649647, |
| "pass@64": 0.48421600118684854, |
| "pass@128": 0.5165165165165165 |
| }, |
| "20": { |
| "pass@1": 0.15219125375375375, |
| "pass@2": 0.21601005533092138, |
| "pass@4": 0.2833787189889551, |
| "pass@8": 0.34737669283247447, |
| "pass@16": 0.40206167741449067, |
| "pass@32": 0.4470107741738278, |
| "pass@64": 0.4875770578788106, |
| "pass@128": 0.5225225225225225 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 550391, |
| "answer_accuracy": 0.6702930144193296, |
| "avg_response_len": 260.58617132696804, |
| "resp_tokens_sum": 213972517, |
| "pass_at_k": { |
| "pass@1": 0.6702930144193296, |
| "pass@2": 0.7203984686174745, |
| "pass@4": 0.7605172725681272, |
| "pass@8": 0.7936184035372785, |
| "pass@16": 0.8205157430956861, |
| "pass@32": 0.8418489124402874, |
| "pass@64": 0.8581997269526279, |
| "pass@128": 0.8693686671862821 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8947948619631901, |
| "pass@2": 0.9342487228877832, |
| "pass@4": 0.9553914430174755, |
| "pass@8": 0.9685340029247087, |
| "pass@16": 0.9778976377826292, |
| "pass@32": 0.9854515349169342, |
| "pass@64": 0.991566746760942, |
| "pass@128": 0.9938650306748467 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9943181818181818, |
| "pass@2": 0.9965987383679314, |
| "pass@4": 0.9986907074115734, |
| "pass@8": 0.9997900403945096, |
| "pass@16": 0.999993778423042, |
| "pass@32": 0.9999999968162022, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986825073964497, |
| "pass@2": 0.9998864324651727, |
| "pass@4": 0.9999964917477031, |
| "pass@8": 0.999999997761824, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9817028985507247, |
| "pass@2": 0.9890858581536004, |
| "pass@4": 0.9919462368834332, |
| "pass@8": 0.9941619355764031, |
| "pass@16": 0.9961781821357537, |
| "pass@32": 0.9975886476682482, |
| "pass@64": 0.9985464572309225, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9912249283667621, |
| "pass@2": 0.9971071734765248, |
| "pass@4": 0.998236967692792, |
| "pass@8": 0.9989742809016088, |
| "pass@16": 0.9996596953889924, |
| "pass@32": 0.9999716397876668, |
| "pass@64": 0.9999999654169256, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9866781069364162, |
| "pass@2": 0.9940098595876383, |
| "pass@4": 0.9954577830950322, |
| "pass@8": 0.9965385997356417, |
| "pass@16": 0.997911202425725, |
| "pass@32": 0.9992275285274425, |
| "pass@64": 0.9999158252269373, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9713248239436619, |
| "pass@2": 0.9873073084174341, |
| "pass@4": 0.9928848371066289, |
| "pass@8": 0.9962645987397817, |
| "pass@16": 0.9977159520362265, |
| "pass@32": 0.9984178461141198, |
| "pass@64": 0.9993013196994364, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9549728667664671, |
| "pass@2": 0.9786595607996605, |
| "pass@4": 0.9846184301812573, |
| "pass@8": 0.9881064318849376, |
| "pass@16": 0.9915007486161633, |
| "pass@32": 0.9943169158716935, |
| "pass@64": 0.996171853688097, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.8858951430722891, |
| "pass@2": 0.9403234246039275, |
| "pass@4": 0.9654911386264966, |
| "pass@8": 0.9802103258206285, |
| "pass@16": 0.9892633104282124, |
| "pass@32": 0.9943230187012654, |
| "pass@64": 0.9966323831791278, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.770865987460815, |
| "pass@2": 0.8529438081603437, |
| "pass@4": 0.8969163801625106, |
| "pass@8": 0.9239001333151642, |
| "pass@16": 0.944273209688222, |
| "pass@32": 0.9607568063656658, |
| "pass@64": 0.9735690051847953, |
| "pass@128": 0.9843260188087775 |
| }, |
| "13": { |
| "pass@1": 0.55249912464986, |
| "pass@2": 0.670897295926244, |
| "pass@4": 0.7573695231373386, |
| "pass@8": 0.8195883022487538, |
| "pass@16": 0.8639400641786898, |
| "pass@32": 0.8959443632193429, |
| "pass@64": 0.9194230288568553, |
| "pass@128": 0.938375350140056 |
| }, |
| "14": { |
| "pass@1": 0.4458607456140351, |
| "pass@2": 0.5551821441727679, |
| "pass@4": 0.6490961617736377, |
| "pass@8": 0.7264317974040324, |
| "pass@16": 0.7878208627637026, |
| "pass@32": 0.8337530255493398, |
| "pass@64": 0.8650871772849, |
| "pass@128": 0.8830409356725146 |
| }, |
| "15": { |
| "pass@1": 0.3692708333333333, |
| "pass@2": 0.4844406167979, |
| "pass@4": 0.5807708542114053, |
| "pass@8": 0.6566659618047733, |
| "pass@16": 0.71822453037185, |
| "pass@32": 0.7690809020128775, |
| "pass@64": 0.8074737398378564, |
| "pass@128": 0.8272727272727273 |
| }, |
| "16": { |
| "pass@1": 0.2302795584045584, |
| "pass@2": 0.3132508075913587, |
| "pass@4": 0.3953974561940442, |
| "pass@8": 0.4699005180824398, |
| "pass@16": 0.5316027123774437, |
| "pass@32": 0.5810105949043947, |
| "pass@64": 0.6208832644017411, |
| "pass@128": 0.6495726495726496 |
| }, |
| "17": { |
| "pass@1": 0.21753687888198758, |
| "pass@2": 0.29977732063383367, |
| "pass@4": 0.3802775075281738, |
| "pass@8": 0.4530973687186417, |
| "pass@16": 0.5146178409330402, |
| "pass@32": 0.5636207169721692, |
| "pass@64": 0.6026909514942224, |
| "pass@128": 0.6335403726708074 |
| }, |
| "18": { |
| "pass@1": 0.19386488970588237, |
| "pass@2": 0.27067797591477505, |
| "pass@4": 0.34721876282008873, |
| "pass@8": 0.4179357468470566, |
| "pass@16": 0.4785538356301472, |
| "pass@32": 0.5267746944091646, |
| "pass@64": 0.5637993560978994, |
| "pass@128": 0.5882352941176471 |
| }, |
| "19": { |
| "pass@1": 0.14381045386904762, |
| "pass@2": 0.20448420509936244, |
| "pass@4": 0.26923672822147243, |
| "pass@8": 0.3304539572282448, |
| "pass@16": 0.3857649133616538, |
| "pass@32": 0.4348844940956561, |
| "pass@64": 0.4749150124076846, |
| "pass@128": 0.5029761904761905 |
| }, |
| "20": { |
| "pass@1": 0.140812125748503, |
| "pass@2": 0.2086588388184262, |
| "pass@4": 0.2810939201462094, |
| "pass@8": 0.3485268808493757, |
| "pass@16": 0.4055889275550898, |
| "pass@32": 0.45131218202379575, |
| "pass@64": 0.4876867028458811, |
| "pass@128": 0.5149700598802395 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 541232, |
| "answer_accuracy": 0.6755671832561112, |
| "avg_response_len": 280.69016616072855, |
| "resp_tokens_sum": 224875488, |
| "pass_at_k": { |
| "pass@1": 0.6755671832561112, |
| "pass@2": 0.7267714805640489, |
| "pass@4": 0.7671935018298011, |
| "pass@8": 0.7993583792277895, |
| "pass@16": 0.8249747156351722, |
| "pass@32": 0.8456421305459204, |
| "pass@64": 0.8626208528041777, |
| "pass@128": 0.8763380731746285 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9066464552238805, |
| "pass@2": 0.9443434892466809, |
| "pass@4": 0.9631634325560046, |
| "pass@8": 0.9746091114318679, |
| "pass@16": 0.981989067498895, |
| "pass@32": 0.9872894382051791, |
| "pass@64": 0.9904128899753277, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 0.9999553571428571, |
| "pass@2": 0.9999996484814399, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9983600583090378, |
| "pass@2": 0.9995706451872088, |
| "pass@4": 0.9999545667214339, |
| "pass@8": 0.9999994678161623, |
| "pass@16": 0.9999999999690319, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9955226608187134, |
| "pass@2": 0.996956229566699, |
| "pass@4": 0.9970747003664014, |
| "pass@8": 0.9970760232358671, |
| "pass@16": 0.9970760233918129, |
| "pass@32": 0.9970760233918129, |
| "pass@64": 0.9970760233918129, |
| "pass@128": 0.9970760233918129 |
| }, |
| "5": { |
| "pass@1": 0.9707720588235295, |
| "pass@2": 0.9819418712366834, |
| "pass@4": 0.9875488412294049, |
| "pass@8": 0.9906721170939422, |
| "pass@16": 0.992369679677989, |
| "pass@32": 0.9934273310473238, |
| "pass@64": 0.9940328828017424, |
| "pass@128": 0.9941176470588236 |
| }, |
| "6": { |
| "pass@1": 0.994011238170347, |
| "pass@2": 0.9966948384212229, |
| "pass@4": 0.9969382767414325, |
| "pass@8": 0.9970425818238966, |
| "pass@16": 0.9972397476340691, |
| "pass@32": 0.9976340694006309, |
| "pass@64": 0.998422712933754, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9809632120253164, |
| "pass@2": 0.9907531707864047, |
| "pass@4": 0.9939196128964891, |
| "pass@8": 0.9955120902657174, |
| "pass@16": 0.9963967136338048, |
| "pass@32": 0.996791897857649, |
| "pass@64": 0.9968353559542914, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9713072447447447, |
| "pass@2": 0.9862585272280152, |
| "pass@4": 0.9918887061977614, |
| "pass@8": 0.9948537929562266, |
| "pass@16": 0.9965289626590625, |
| "pass@32": 0.9975971169999002, |
| "pass@64": 0.9984964932745174, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9532003617363344, |
| "pass@2": 0.9822585417879847, |
| "pass@4": 0.9900546741343824, |
| "pass@8": 0.9928854174919578, |
| "pass@16": 0.9951733768710284, |
| "pass@32": 0.9976992193537383, |
| "pass@64": 0.9996070290208584, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8584325396825396, |
| "pass@2": 0.9158269278840144, |
| "pass@4": 0.947487876217854, |
| "pass@8": 0.9662182976284491, |
| "pass@16": 0.976791807385288, |
| "pass@32": 0.9819247624165014, |
| "pass@64": 0.9848252233387427, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.7979023318042814, |
| "pass@2": 0.8880074767030265, |
| "pass@4": 0.934417512604502, |
| "pass@8": 0.9589038010248868, |
| "pass@16": 0.9715989629294857, |
| "pass@32": 0.9774837125997929, |
| "pass@64": 0.9811677016027062, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.5472608024691358, |
| "pass@2": 0.6648135996889273, |
| "pass@4": 0.7540624178343447, |
| "pass@8": 0.817288586664587, |
| "pass@16": 0.8629907822666152, |
| "pass@32": 0.8987447149981824, |
| "pass@64": 0.927510348163161, |
| "pass@128": 0.9475308641975309 |
| }, |
| "14": { |
| "pass@1": 0.4938102744807122, |
| "pass@2": 0.6095301584149166, |
| "pass@4": 0.7005231909067747, |
| "pass@8": 0.7657016935167862, |
| "pass@16": 0.8124875164470288, |
| "pass@32": 0.8503301329249001, |
| "pass@64": 0.8858854705277092, |
| "pass@128": 0.9169139465875371 |
| }, |
| "15": { |
| "pass@1": 0.37755594135802467, |
| "pass@2": 0.48756584463400393, |
| "pass@4": 0.5814024534548923, |
| "pass@8": 0.6590459728887901, |
| "pass@16": 0.7236160111797545, |
| "pass@32": 0.7766907707481191, |
| "pass@64": 0.8177650626424972, |
| "pass@128": 0.845679012345679 |
| }, |
| "16": { |
| "pass@1": 0.24944690265486727, |
| "pass@2": 0.33750624230599463, |
| "pass@4": 0.4218391044592877, |
| "pass@8": 0.49854943027379045, |
| "pass@16": 0.5657431421036233, |
| "pass@32": 0.6217821165667055, |
| "pass@64": 0.6690913993944967, |
| "pass@128": 0.7109144542772862 |
| }, |
| "17": { |
| "pass@1": 0.20524284638554216, |
| "pass@2": 0.2843695154634285, |
| "pass@4": 0.3594979064175711, |
| "pass@8": 0.4248822789520208, |
| "pass@16": 0.48281238948550975, |
| "pass@32": 0.5343758170889205, |
| "pass@64": 0.5769024260817495, |
| "pass@128": 0.608433734939759 |
| }, |
| "18": { |
| "pass@1": 0.20216733870967743, |
| "pass@2": 0.28294585026670055, |
| "pass@4": 0.36513060706121425, |
| "pass@8": 0.4360889347472986, |
| "pass@16": 0.49089282710175347, |
| "pass@32": 0.5346986024780715, |
| "pass@64": 0.5735686830652535, |
| "pass@128": 0.6096774193548387 |
| }, |
| "19": { |
| "pass@1": 0.16880664652567975, |
| "pass@2": 0.23587070616361772, |
| "pass@4": 0.30202498139394224, |
| "pass@8": 0.3625748851412769, |
| "pass@16": 0.416689758692117, |
| "pass@32": 0.4624262685514275, |
| "pass@64": 0.49878692433702804, |
| "pass@128": 0.5287009063444109 |
| }, |
| "20": { |
| "pass@1": 0.14442567567567569, |
| "pass@2": 0.20913137546995805, |
| "pass@4": 0.27880532810650926, |
| "pass@8": 0.34708860136362146, |
| "pass@16": 0.40649366622612243, |
| "pass@32": 0.4543060270928111, |
| "pass@64": 0.493016680586197, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| } |
| } |
| } |
| ] |
