AR-Diff / SDAR-100M-midtrain-diff_diff /composition /low_confidence_static /block8 /checkpoint-38796_metrics.json
| { | |
| "checkpoint": "checkpoint-38796", | |
| "id": { | |
| "answer_accuracy": 0.7004678819444444, | |
| "avg_response_len": 198.95842534722223, | |
| "avg_loss": 0.0, | |
| "count": 1152000, | |
| "validation_metrics": {}, | |
| "pass_at_k": { | |
| "pass@1": 0.7004678819444444, | |
| "pass@2": 0.803145190835523, | |
| "pass@4": 0.8810327584676866, | |
| "pass@8": 0.930000105282735, | |
| "pass@16": 0.9581204489043869, | |
| "pass@32": 0.9742042919530437, | |
| "pass@64": 0.9838618914836799, | |
| "pass@128": 0.9898888888888889 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5105703125, | |
| "pass@2": 0.6699938484251962, | |
| "pass@4": 0.7991659679415073, | |
| "pass@8": 0.8847154633623784, | |
| "pass@16": 0.9347854223296508, | |
| "pass@32": 0.9632970265066139, | |
| "pass@64": 0.980442548083157, | |
| "pass@128": 0.992 | |
| }, | |
| "2": { | |
| "pass@1": 0.935890625, | |
| "pass@2": 0.9592861712598416, | |
| "pass@4": 0.9768458330521185, | |
| "pass@8": 0.9867971535992609, | |
| "pass@16": 0.9919445464167224, | |
| "pass@32": 0.9942186611862515, | |
| "pass@64": 0.9948690786426989, | |
| "pass@128": 0.995 | |
| }, | |
| "3": { | |
| "pass@1": 0.93975, | |
| "pass@2": 0.9609361466535429, | |
| "pass@4": 0.9775336966629172, | |
| "pass@8": 0.9882813969399472, | |
| "pass@16": 0.9934026899376711, | |
| "pass@32": 0.9952107555410625, | |
| "pass@64": 0.9959019362061534, | |
| "pass@128": 0.996 | |
| }, | |
| "4": { | |
| "pass@1": 0.806453125, | |
| "pass@2": 0.8676486220472444, | |
| "pass@4": 0.911353344019498, | |
| "pass@8": 0.9400612877585252, | |
| "pass@16": 0.9589647416451935, | |
| "pass@32": 0.9720786045524892, | |
| "pass@64": 0.9811436774483744, | |
| "pass@128": 0.987 | |
| }, | |
| "5": { | |
| "pass@1": 0.719234375, | |
| "pass@2": 0.8196967273622047, | |
| "pass@4": 0.8851043299587543, | |
| "pass@8": 0.9222199389254376, | |
| "pass@16": 0.9448174036914444, | |
| "pass@32": 0.9598280784653056, | |
| "pass@64": 0.9709283962665971, | |
| "pass@128": 0.98 | |
| }, | |
| "6": { | |
| "pass@1": 0.698921875, | |
| "pass@2": 0.809659571850394, | |
| "pass@4": 0.8861456253280843, | |
| "pass@8": 0.9296744960557235, | |
| "pass@16": 0.9532426321493441, | |
| "pass@32": 0.9681859214131414, | |
| "pass@64": 0.9788188969365212, | |
| "pass@128": 0.986 | |
| }, | |
| "7": { | |
| "pass@1": 0.618640625, | |
| "pass@2": 0.7534297490157486, | |
| "pass@4": 0.8522904651293589, | |
| "pass@8": 0.9134351163840331, | |
| "pass@16": 0.9500415651070467, | |
| "pass@32": 0.9713268310837263, | |
| "pass@64": 0.9833246380222623, | |
| "pass@128": 0.99 | |
| }, | |
| "8": { | |
| "pass@1": 0.57221875, | |
| "pass@2": 0.7231927903543292, | |
| "pass@4": 0.8385967574990625, | |
| "pass@8": 0.909620771341596, | |
| "pass@16": 0.9483642685489777, | |
| "pass@32": 0.9693944224018802, | |
| "pass@64": 0.9815823234229324, | |
| "pass@128": 0.989 | |
| }, | |
| "9": { | |
| "pass@1": 0.50253125, | |
| "pass@2": 0.6644630905511801, | |
| "pass@4": 0.8022588066179234, | |
| "pass@8": 0.8951953231778146, | |
| "pass@16": 0.9475207703133638, | |
| "pass@32": 0.9742983264269067, | |
| "pass@64": 0.9877455283243902, | |
| "pass@128": 0.994 | |
| } | |
| }, | |
| "per_op_accuracy": { | |
| "10": 0.5105703125, | |
| "2": 0.935890625, | |
| "3": 0.93975, | |
| "4": 0.806453125, | |
| "5": 0.719234375, | |
| "6": 0.698921875, | |
| "7": 0.618640625, | |
| "8": 0.57221875, | |
| "9": 0.50253125 | |
| }, | |
| "per_op_avg_response_len": { | |
| "10": 280.0350625, | |
| "2": 116.76371875, | |
| "3": 149.778203125, | |
| "4": 158.538859375, | |
| "5": 173.8068046875, | |
| "6": 193.58321875, | |
| "7": 222.0638046875, | |
| "8": 240.2696328125, | |
| "9": 255.7865234375 | |
| }, | |
| "per_op_avg_loss": {}, | |
| "per_op_length": { | |
| "10": 280.0350625, | |
| "2": 116.76371875, | |
| "3": 149.778203125, | |
| "4": 158.538859375, | |
| "5": 173.8068046875, | |
| "6": 193.58321875, | |
| "7": 222.0638046875, | |
| "8": 240.2696328125, | |
| "9": 255.7865234375 | |
| }, | |
| "per_op_loss": {}, | |
| "per_template": { | |
| "crazy_zootopia": { | |
| "count": 379008, | |
| "correct": 260419, | |
| "answer_accuracy": 0.6871068684566025, | |
| "avg_response_len": 189.2360583417764, | |
| "resp_tokens_sum": 71721980, | |
| "pass_at_k": { | |
| "pass@1": 0.6871068684566025, | |
| "pass@2": 0.792426960592694, | |
| "pass@4": 0.8731736406872669, | |
| "pass@8": 0.9240051211601905, | |
| "pass@16": 0.9530947039397669, | |
| "pass@32": 0.9700868842236148, | |
| "pass@64": 0.9808123981985275, | |
| "pass@128": 0.9878419452887538 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5095409292035398, | |
| "pass@2": 0.6676066418135783, | |
| "pass@4": 0.792585876710101, | |
| "pass@8": 0.8760817636863799, | |
| "pass@16": 0.9283330831148673, | |
| "pass@32": 0.9602328298448352, | |
| "pass@64": 0.9791876634794598, | |
| "pass@128": 0.9911504424778761 | |
| }, | |
| "2": { | |
| "pass@1": 0.9343360591900312, | |
| "pass@2": 0.9592853901439887, | |
| "pass@4": 0.9779000282674944, | |
| "pass@8": 0.9878251114374832, | |
| "pass@16": 0.9922969056587208, | |
| "pass@32": 0.9936688924295337, | |
| "pass@64": 0.9937693490770984, | |
| "pass@128": 0.9937694704049844 | |
| }, | |
| "3": { | |
| "pass@1": 0.9195575305810397, | |
| "pass@2": 0.9484343771822101, | |
| "pass@4": 0.9708153361563746, | |
| "pass@8": 0.984238780440124, | |
| "pass@16": 0.9897953555580769, | |
| "pass@32": 0.990792748570468, | |
| "pass@64": 0.9908256717392108, | |
| "pass@128": 0.9908256880733946 | |
| }, | |
| "4": { | |
| "pass@1": 0.7866455078125, | |
| "pass@2": 0.8459907111220479, | |
| "pass@4": 0.8906744697928376, | |
| "pass@8": 0.9215507509441558, | |
| "pass@16": 0.9437763487973065, | |
| "pass@32": 0.9611323349556897, | |
| "pass@64": 0.9735555459387797, | |
| "pass@128": 0.98125 | |
| }, | |
| "5": { | |
| "pass@1": 0.6709573412698413, | |
| "pass@2": 0.7741130014998127, | |
| "pass@4": 0.8445359746698321, | |
| "pass@8": 0.8894758925880082, | |
| "pass@16": 0.9211739208974751, | |
| "pass@32": 0.9452852732182677, | |
| "pass@64": 0.964295583575878, | |
| "pass@128": 0.9777777777777777 | |
| }, | |
| "6": { | |
| "pass@1": 0.7365503368263473, | |
| "pass@2": 0.8420378730727522, | |
| "pass@4": 0.9124382115970529, | |
| "pass@8": 0.9505378525143745, | |
| "pass@16": 0.9685954065179547, | |
| "pass@32": 0.97972634565819, | |
| "pass@64": 0.9880378752295537, | |
| "pass@128": 0.9940119760479041 | |
| }, | |
| "7": { | |
| "pass@1": 0.6175110946745562, | |
| "pass@2": 0.7572351983646276, | |
| "pass@4": 0.8602960354807709, | |
| "pass@8": 0.9199085087926454, | |
| "pass@16": 0.9515635930824008, | |
| "pass@32": 0.9699748728805815, | |
| "pass@64": 0.9825938231042854, | |
| "pass@128": 0.9911242603550295 | |
| }, | |
| "8": { | |
| "pass@1": 0.5553635817307693, | |
| "pass@2": 0.705602191853422, | |
| "pass@4": 0.821673709956928, | |
| "pass@8": 0.8943235864636814, | |
| "pass@16": 0.9343372725706356, | |
| "pass@32": 0.9560323516975169, | |
| "pass@64": 0.9697434965886538, | |
| "pass@128": 0.9807692307692307 | |
| }, | |
| "9": { | |
| "pass@1": 0.479137323943662, | |
| "pass@2": 0.6481472496395699, | |
| "pass@4": 0.7957091023833289, | |
| "pass@8": 0.8944526189705332, | |
| "pass@16": 0.9475698851465509, | |
| "pass@32": 0.9725689666299432, | |
| "pass@64": 0.9838555505224045, | |
| "pass@128": 0.9887323943661972 | |
| } | |
| } | |
| }, | |
| "teachers_in_school": { | |
| "count": 390656, | |
| "correct": 276029, | |
| "answer_accuracy": 0.7065781659567497, | |
| "avg_response_len": 197.2905318233945, | |
| "resp_tokens_sum": 77072730, | |
| "pass_at_k": { | |
| "pass@1": 0.7065781659567497, | |
| "pass@2": 0.80870363986182, | |
| "pass@4": 0.8869682795425502, | |
| "pass@8": 0.9368394808127022, | |
| "pass@16": 0.9653128692761188, | |
| "pass@32": 0.9804199727921247, | |
| "pass@64": 0.9885924403583403, | |
| "pass@128": 0.9934469200524246 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.4727521088957055, | |
| "pass@2": 0.6345907957345057, | |
| "pass@4": 0.77542852455943, | |
| "pass@8": 0.8752167459860357, | |
| "pass@16": 0.9336091179643451, | |
| "pass@32": 0.9627348671897347, | |
| "pass@64": 0.9780142800175672, | |
| "pass@128": 0.9877300613496932 | |
| }, | |
| "2": { | |
| "pass@1": 0.9381648936170213, | |
| "pass@2": 0.9569980614125363, | |
| "pass@4": 0.971669399322805, | |
| "pass@8": 0.9823670230919718, | |
| "pass@16": 0.9902618175090221, | |
| "pass@32": 0.9948116541729152, | |
| "pass@64": 0.9965635996372417, | |
| "pass@128": 0.9969604863221885 | |
| }, | |
| "3": { | |
| "pass@1": 0.950686553030303, | |
| "pass@2": 0.9672959914101645, | |
| "pass@4": 0.9803795730079196, | |
| "pass@8": 0.9893317365210512, | |
| "pass@16": 0.9939045470160155, | |
| "pass@32": 0.9959745105782435, | |
| "pass@64": 0.9968729838687728, | |
| "pass@128": 0.996969696969697 | |
| }, | |
| "4": { | |
| "pass@1": 0.8468703772189349, | |
| "pass@2": 0.9065033429623076, | |
| "pass@4": 0.9489256611044922, | |
| "pass@8": 0.9743967832425964, | |
| "pass@16": 0.9872230174014897, | |
| "pass@32": 0.9930567852310904, | |
| "pass@64": 0.9965815370797454, | |
| "pass@128": 1.0 | |
| }, | |
| "5": { | |
| "pass@1": 0.7758605072463768, | |
| "pass@2": 0.8736209774049982, | |
| "pass@4": 0.9324389600756424, | |
| "pass@8": 0.9618868319100985, | |
| "pass@16": 0.9760629639219559, | |
| "pass@32": 0.9825935008857577, | |
| "pass@64": 0.9869077034509552, | |
| "pass@128": 0.991304347826087 | |
| }, | |
| "6": { | |
| "pass@1": 0.6769340974212035, | |
| "pass@2": 0.7943660740473344, | |
| "pass@4": 0.8786454614018506, | |
| "pass@8": 0.9272978475082277, | |
| "pass@16": 0.9543528148099992, | |
| "pass@32": 0.9711407927719327, | |
| "pass@64": 0.9821259539329217, | |
| "pass@128": 0.9885386819484241 | |
| }, | |
| "7": { | |
| "pass@1": 0.6066428829479769, | |
| "pass@2": 0.7398836818988667, | |
| "pass@4": 0.8397892167380809, | |
| "pass@8": 0.9042434643738783, | |
| "pass@16": 0.9441212894901903, | |
| "pass@32": 0.96651931919803, | |
| "pass@64": 0.9778479793557311, | |
| "pass@128": 0.9826589595375722 | |
| }, | |
| "8": { | |
| "pass@1": 0.5825044014084507, | |
| "pass@2": 0.7344803565487412, | |
| "pass@4": 0.8506169114424071, | |
| "pass@8": 0.9214648743029239, | |
| "pass@16": 0.9596192806093881, | |
| "pass@32": 0.9801233610769453, | |
| "pass@64": 0.9908749402461413, | |
| "pass@128": 0.9971830985915493 | |
| }, | |
| "9": { | |
| "pass@1": 0.5183383233532934, | |
| "pass@2": 0.6750163550851055, | |
| "pass@4": 0.8066507348071007, | |
| "pass@8": 0.8964758234235036, | |
| "pass@16": 0.9496126362729699, | |
| "pass@32": 0.9775149761319307, | |
| "pass@64": 0.9920008501197732, | |
| "pass@128": 1.0 | |
| } | |
| } | |
| }, | |
| "movie_festival_awards": { | |
| "count": 382336, | |
| "correct": 270491, | |
| "answer_accuracy": 0.7074693463341145, | |
| "avg_response_len": 210.3003536156679, | |
| "resp_tokens_sum": 80405396, | |
| "pass_at_k": { | |
| "pass@1": 0.7074693463341145, | |
| "pass@2": 0.8080907192321637, | |
| "pass@4": 0.8827587837195806, | |
| "pass@8": 0.928954699815554, | |
| "pass@16": 0.9557535134727481, | |
| "pass@32": 0.9719349201304631, | |
| "pass@64": 0.9820513506238909, | |
| "pass@128": 0.9882825577502511 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5484141791044777, | |
| "pass@2": 0.7068614848983433, | |
| "pass@4": 0.8289243484489802, | |
| "pass@8": 0.9026957799440277, | |
| "pass@16": 0.9424595065592135, | |
| "pass@32": 0.9669448671204803, | |
| "pass@64": 0.9840754473966935, | |
| "pass@128": 0.9970149253731343 | |
| }, | |
| "2": { | |
| "pass@1": 0.9351785714285714, | |
| "pass@2": 0.9614377109111365, | |
| "pass@4": 0.9807448331458566, | |
| "pass@8": 0.9900186920873438, | |
| "pass@16": 0.9932031478280128, | |
| "pass@32": 0.994165464267091, | |
| "pass@64": 0.994284838023708, | |
| "pass@128": 0.9942857142857143 | |
| }, | |
| "3": { | |
| "pass@1": 0.9484784985422741, | |
| "pass@2": 0.9667359421271322, | |
| "pass@4": 0.9812006491171111, | |
| "pass@8": 0.9911249057844892, | |
| "pass@16": 0.9963589159034837, | |
| "pass@32": 0.9986878666696775, | |
| "pass@64": 0.9998071628884445, | |
| "pass@128": 1.0 | |
| }, | |
| "4": { | |
| "pass@1": 0.7850420321637427, | |
| "pass@2": 0.8495130542892672, | |
| "pass@4": 0.8935691234867573, | |
| "pass@8": 0.9234471775450245, | |
| "pass@16": 0.9452483337671119, | |
| "pass@32": 0.961587906311577, | |
| "pass@64": 0.9729863836696224, | |
| "pass@128": 0.97953216374269 | |
| }, | |
| "5": { | |
| "pass@1": 0.7065027573529412, | |
| "pass@2": 0.8072114549559978, | |
| "pass@4": 0.8746589903284145, | |
| "pass@8": 0.9123061051506803, | |
| "pass@16": 0.9350173413401953, | |
| "pass@32": 0.9502013517528385, | |
| "pass@64": 0.9608592051459295, | |
| "pass@128": 0.9705882352941176 | |
| }, | |
| "6": { | |
| "pass@1": 0.6834828470031545, | |
| "pass@2": 0.7923822157778388, | |
| "pass@4": 0.8667003048120562, | |
| "pass@8": 0.9103088471153025, | |
| "pass@16": 0.9358442713049175, | |
| "pass@32": 0.9527734551605721, | |
| "pass@64": 0.9654646330828386, | |
| "pass@128": 0.9747634069400631 | |
| }, | |
| "7": { | |
| "pass@1": 0.6329855617088608, | |
| "pass@2": 0.7641914494667598, | |
| "pass@4": 0.8574156207135493, | |
| "pass@8": 0.9165753219580909, | |
| "pass@16": 0.9548959002581976, | |
| "pass@32": 0.9780368342011752, | |
| "pass@64": 0.990102927075729, | |
| "pass@128": 0.9968354430379747 | |
| }, | |
| "8": { | |
| "pass@1": 0.5770457957957958, | |
| "pass@2": 0.7276407805443237, | |
| "pass@4": 0.8416383075989373, | |
| "pass@8": 0.911326672634802, | |
| "pass@16": 0.949508092764589, | |
| "pass@32": 0.9704760825523712, | |
| "pass@64": 0.98276801411379, | |
| "pass@128": 0.987987987987988 | |
| }, | |
| "9": { | |
| "pass@1": 0.5122588424437299, | |
| "pass@2": 0.6717535509026002, | |
| "pass@4": 0.8050184239429874, | |
| "pass@8": 0.8946679048869003, | |
| "pass@16": 0.9452181368847813, | |
| "pass@32": 0.9728178175087213, | |
| "pass@64": 0.9876158326332224, | |
| "pass@128": 0.9935691318327974 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "ood": { | |
| "answer_accuracy": 0.331778125, | |
| "avg_response_len": 371.70684140625, | |
| "avg_loss": 0.0, | |
| "count": 1280000, | |
| "validation_metrics": {}, | |
| "pass_at_k": { | |
| "pass@1": 0.331778125, | |
| "pass@2": 0.4780030634842523, | |
| "pass@4": 0.6261648663948085, | |
| "pass@8": 0.7526011526766907, | |
| "pass@16": 0.8449462991420205, | |
| "pass@32": 0.9051796623764043, | |
| "pass@64": 0.9428432153839094, | |
| "pass@128": 0.9652 | |
| }, | |
| "per_op_pass_at_k": { | |
| "11": { | |
| "pass@1": 0.4547109375, | |
| "pass@2": 0.6138021653543304, | |
| "pass@4": 0.7549666171728527, | |
| "pass@8": 0.8583315153211275, | |
| "pass@16": 0.9221412824985413, | |
| "pass@32": 0.9570664977233017, | |
| "pass@64": 0.974931983836227, | |
| "pass@128": 0.983 | |
| }, | |
| "12": { | |
| "pass@1": 0.4180625, | |
| "pass@2": 0.5797405265748027, | |
| "pass@4": 0.7296782709973749, | |
| "pass@8": 0.8443357826368147, | |
| "pass@16": 0.916390867717859, | |
| "pass@32": 0.9551260363971661, | |
| "pass@64": 0.975269380378476, | |
| "pass@128": 0.986 | |
| }, | |
| "13": { | |
| "pass@1": 0.396984375, | |
| "pass@2": 0.5582675935039371, | |
| "pass@4": 0.7075908911698547, | |
| "pass@8": 0.8218632521719159, | |
| "pass@16": 0.8957610496843468, | |
| "pass@32": 0.9375014501187352, | |
| "pass@64": 0.9603788119235547, | |
| "pass@128": 0.975 | |
| }, | |
| "14": { | |
| "pass@1": 0.3836953125, | |
| "pass@2": 0.5430013533464566, | |
| "pass@4": 0.6984549793775782, | |
| "pass@8": 0.8231021884341475, | |
| "pass@16": 0.9053910203817964, | |
| "pass@32": 0.9510216257343269, | |
| "pass@64": 0.9741567124841395, | |
| "pass@128": 0.985 | |
| }, | |
| "15": { | |
| "pass@1": 0.33775, | |
| "pass@2": 0.4967496309055121, | |
| "pass@4": 0.6602598811398573, | |
| "pass@8": 0.7980327584444624, | |
| "pass@16": 0.8912756658908414, | |
| "pass@32": 0.9429084818635887, | |
| "pass@64": 0.9696608164732217, | |
| "pass@128": 0.983 | |
| }, | |
| "16": { | |
| "pass@1": 0.3182265625, | |
| "pass@2": 0.4695752952755903, | |
| "pass@4": 0.6255005896137982, | |
| "pass@8": 0.7595363695753027, | |
| "pass@16": 0.8578304309643211, | |
| "pass@32": 0.9197597659036819, | |
| "pass@64": 0.9536475067446567, | |
| "pass@128": 0.972 | |
| }, | |
| "17": { | |
| "pass@1": 0.299875, | |
| "pass@2": 0.4421136811023624, | |
| "pass@4": 0.5927719518185219, | |
| "pass@8": 0.7277104389245667, | |
| "pass@16": 0.8309745449337134, | |
| "pass@32": 0.9007491803788847, | |
| "pass@64": 0.9454100163755238, | |
| "pass@128": 0.972 | |
| }, | |
| "18": { | |
| "pass@1": 0.272328125, | |
| "pass@2": 0.40941916830708663, | |
| "pass@4": 0.5598332344394451, | |
| "pass@8": 0.6993310814147358, | |
| "pass@16": 0.808673856643445, | |
| "pass@32": 0.8838542581208818, | |
| "pass@64": 0.9322295342493997, | |
| "pass@128": 0.957 | |
| }, | |
| "19": { | |
| "pass@1": 0.2403671875, | |
| "pass@2": 0.3612985974409448, | |
| "pass@4": 0.49474866122984607, | |
| "pass@8": 0.6234513899075376, | |
| "pass@16": 0.7351839126274962, | |
| "pass@32": 0.8242222274517007, | |
| "pass@64": 0.8913528812708557, | |
| "pass@128": 0.937 | |
| }, | |
| "20": { | |
| "pass@1": 0.19578125, | |
| "pass@2": 0.3060626230314962, | |
| "pass@4": 0.43784358698912595, | |
| "pass@8": 0.5703167499362239, | |
| "pass@16": 0.6858403600775729, | |
| "pass@32": 0.7795871000716412, | |
| "pass@64": 0.8513945101029146, | |
| "pass@128": 0.902 | |
| } | |
| }, | |
| "per_op_accuracy": { | |
| "11": 0.4547109375, | |
| "12": 0.4180625, | |
| "13": 0.396984375, | |
| "14": 0.3836953125, | |
| "15": 0.33775, | |
| "16": 0.3182265625, | |
| "17": 0.299875, | |
| "18": 0.272328125, | |
| "19": 0.2403671875, | |
| "20": 0.19578125 | |
| }, | |
| "per_op_avg_response_len": { | |
| "11": 296.70078125, | |
| "12": 310.4506328125, | |
| "13": 328.7139609375, | |
| "14": 352.013390625, | |
| "15": 373.0260234375, | |
| "16": 381.777359375, | |
| "17": 401.5364765625, | |
| "18": 417.4558203125, | |
| "19": 421.0910859375, | |
| "20": 434.3028828125 | |
| }, | |
| "per_op_avg_loss": {}, | |
| "per_op_length": { | |
| "11": 296.70078125, | |
| "12": 310.4506328125, | |
| "13": 328.7139609375, | |
| "14": 352.013390625, | |
| "15": 373.0260234375, | |
| "16": 381.777359375, | |
| "17": 401.5364765625, | |
| "18": 417.4558203125, | |
| "19": 421.0910859375, | |
| "20": 434.3028828125 | |
| }, | |
| "per_op_loss": {}, | |
| "per_template": { | |
| "crazy_zootopia": { | |
| "count": 430720, | |
| "correct": 141383, | |
| "answer_accuracy": 0.3282480497771174, | |
| "avg_response_len": 353.0143457466568, | |
| "resp_tokens_sum": 152050339, | |
| "pass_at_k": { | |
| "pass@1": 0.3282480497771174, | |
| "pass@2": 0.4742019588515418, | |
| "pass@4": 0.6229426643456395, | |
| "pass@8": 0.7505481103707128, | |
| "pass@16": 0.8438042986957948, | |
| "pass@32": 0.9042307574553091, | |
| "pass@64": 0.9417688787017761, | |
| "pass@128": 0.9643387815750372 | |
| }, | |
| "per_op_pass_at_k": { | |
| "11": { | |
| "pass@1": 0.4631506728045326, | |
| "pass@2": 0.6252648836742432, | |
| "pass@4": 0.767624272798053, | |
| "pass@8": 0.871077584229325, | |
| "pass@16": 0.9329364153366072, | |
| "pass@32": 0.9639094963103593, | |
| "pass@64": 0.9780315644820233, | |
| "pass@128": 0.9858356940509915 | |
| }, | |
| "12": { | |
| "pass@1": 0.4075962217514124, | |
| "pass@2": 0.5693932392455183, | |
| "pass@4": 0.7213146841814255, | |
| "pass@8": 0.8371093473559977, | |
| "pass@16": 0.9086267449422752, | |
| "pass@32": 0.9469433243538209, | |
| "pass@64": 0.9686512377816593, | |
| "pass@128": 0.9830508474576272 | |
| }, | |
| "13": { | |
| "pass@1": 0.39882934952978055, | |
| "pass@2": 0.5654986516673662, | |
| "pass@4": 0.7186619650600091, | |
| "pass@8": 0.8329572629488781, | |
| "pass@16": 0.9032880963070734, | |
| "pass@32": 0.9407221789825072, | |
| "pass@64": 0.9613466943723601, | |
| "pass@128": 0.9749216300940439 | |
| }, | |
| "14": { | |
| "pass@1": 0.35604069314641745, | |
| "pass@2": 0.5193354919420117, | |
| "pass@4": 0.684745257018106, | |
| "pass@8": 0.8188914601096546, | |
| "pass@16": 0.9064022768205324, | |
| "pass@32": 0.9520687996104888, | |
| "pass@64": 0.9726630069903832, | |
| "pass@128": 0.9813084112149533 | |
| }, | |
| "15": { | |
| "pass@1": 0.33921152456647397, | |
| "pass@2": 0.4931290820854757, | |
| "pass@4": 0.650143435556827, | |
| "pass@8": 0.7837146243177703, | |
| "pass@16": 0.8784636507395034, | |
| "pass@32": 0.9341449136359978, | |
| "pass@64": 0.9638817286290657, | |
| "pass@128": 0.9797687861271677 | |
| }, | |
| "16": { | |
| "pass@1": 0.3211945564516129, | |
| "pass@2": 0.47427451104902224, | |
| "pass@4": 0.6338670295245349, | |
| "pass@8": 0.7704789743204787, | |
| "pass@16": 0.8674678402262296, | |
| "pass@32": 0.9269334345131763, | |
| "pass@64": 0.95867323469044, | |
| "pass@128": 0.9774193548387097 | |
| }, | |
| "17": { | |
| "pass@1": 0.2943686777456647, | |
| "pass@2": 0.43410561080515186, | |
| "pass@4": 0.5843995006404546, | |
| "pass@8": 0.7225977142959461, | |
| "pass@16": 0.8319339455322081, | |
| "pass@32": 0.9085076669393349, | |
| "pass@64": 0.9564128585594077, | |
| "pass@128": 0.976878612716763 | |
| }, | |
| "18": { | |
| "pass@1": 0.2702678571428571, | |
| "pass@2": 0.40575260123734536, | |
| "pass@4": 0.5552940701162348, | |
| "pass@8": 0.6981102462106233, | |
| "pass@16": 0.8141778003682741, | |
| "pass@32": 0.894818854884761, | |
| "pass@64": 0.9449149724164841, | |
| "pass@128": 0.9685714285714285 | |
| }, | |
| "19": { | |
| "pass@1": 0.23252158408408408, | |
| "pass@2": 0.34998618204819004, | |
| "pass@4": 0.48075028543729326, | |
| "pass@8": 0.6082667201968596, | |
| "pass@16": 0.7195758859617793, | |
| "pass@32": 0.8085267342570868, | |
| "pass@64": 0.8769663601857598, | |
| "pass@128": 0.9279279279279279 | |
| }, | |
| "20": { | |
| "pass@1": 0.19352946696696696, | |
| "pass@2": 0.29999401468397513, | |
| "pass@4": 0.42863513513513496, | |
| "pass@8": 0.5593568000768633, | |
| "pass@16": 0.6727429737905991, | |
| "pass@32": 0.763426098355866, | |
| "pass@64": 0.8337823046158129, | |
| "pass@128": 0.8858858858858859 | |
| } | |
| } | |
| }, | |
| "teachers_in_school": { | |
| "count": 430464, | |
| "correct": 144421, | |
| "answer_accuracy": 0.3355007619684805, | |
| "avg_response_len": 368.40574124665477, | |
| "resp_tokens_sum": 158585409, | |
| "pass_at_k": { | |
| "pass@1": 0.3355007619684805, | |
| "pass@2": 0.4832102067192541, | |
| "pass@4": 0.6323251844300062, | |
| "pass@8": 0.7584816977922026, | |
| "pass@16": 0.8501143874517795, | |
| "pass@32": 0.9098559724700692, | |
| "pass@64": 0.9465408118527427, | |
| "pass@128": 0.967885816235504 | |
| }, | |
| "per_op_pass_at_k": { | |
| "11": { | |
| "pass@1": 0.4303934487951807, | |
| "pass@2": 0.5927442545773641, | |
| "pass@4": 0.7420111311387282, | |
| "pass@8": 0.851233626656646, | |
| "pass@16": 0.9172232851276387, | |
| "pass@32": 0.9531123039455416, | |
| "pass@64": 0.9715619962195704, | |
| "pass@128": 0.9789156626506024 | |
| }, | |
| "12": { | |
| "pass@1": 0.41724627742946707, | |
| "pass@2": 0.577324331078913, | |
| "pass@4": 0.7282632558861173, | |
| "pass@8": 0.8475615268998387, | |
| "pass@16": 0.9247642940425248, | |
| "pass@32": 0.9665189848115742, | |
| "pass@64": 0.9858146148658699, | |
| "pass@128": 0.9937304075235109 | |
| }, | |
| "13": { | |
| "pass@1": 0.3948266806722689, | |
| "pass@2": 0.5571427882176495, | |
| "pass@4": 0.7088043327567247, | |
| "pass@8": 0.8258372669093875, | |
| "pass@16": 0.9026431000892041, | |
| "pass@32": 0.9463133172020857, | |
| "pass@64": 0.9691138082317794, | |
| "pass@128": 0.9831932773109243 | |
| }, | |
| "14": { | |
| "pass@1": 0.41015625, | |
| "pass@2": 0.5688616636736198, | |
| "pass@4": 0.7169494355968666, | |
| "pass@8": 0.831644961695096, | |
| "pass@16": 0.9084661438307753, | |
| "pass@32": 0.9549772171257439, | |
| "pass@64": 0.9812690835166658, | |
| "pass@128": 0.9941520467836257 | |
| }, | |
| "15": { | |
| "pass@1": 0.33780776515151517, | |
| "pass@2": 0.4990828561202573, | |
| "pass@4": 0.6665392868505067, | |
| "pass@8": 0.807065398160388, | |
| "pass@16": 0.8994904127312967, | |
| "pass@32": 0.9489088718389327, | |
| "pass@64": 0.9717879055264091, | |
| "pass@128": 0.9787878787878788 | |
| }, | |
| "16": { | |
| "pass@1": 0.32298344017094016, | |
| "pass@2": 0.4738822711263657, | |
| "pass@4": 0.6295656763524213, | |
| "pass@8": 0.7637132042842444, | |
| "pass@16": 0.8619682864031663, | |
| "pass@32": 0.9226160339647616, | |
| "pass@64": 0.9550762847040853, | |
| "pass@128": 0.9743589743589743 | |
| }, | |
| "17": { | |
| "pass@1": 0.32038528726708076, | |
| "pass@2": 0.4652963301462315, | |
| "pass@4": 0.6150197741198496, | |
| "pass@8": 0.7468171297086604, | |
| "pass@16": 0.8433732629472633, | |
| "pass@32": 0.9032106328795099, | |
| "pass@64": 0.9403173018208404, | |
| "pass@128": 0.968944099378882 | |
| }, | |
| "18": { | |
| "pass@1": 0.2850873161764706, | |
| "pass@2": 0.4292677454840204, | |
| "pass@4": 0.5829545498908223, | |
| "pass@8": 0.7183212578697612, | |
| "pass@16": 0.8203104066913215, | |
| "pass@32": 0.8889514779071954, | |
| "pass@64": 0.932731144763619, | |
| "pass@128": 0.9558823529411765 | |
| }, | |
| "19": { | |
| "pass@1": 0.24262927827380953, | |
| "pass@2": 0.3626660925196849, | |
| "pass@4": 0.4944864068330747, | |
| "pass@8": 0.621680543117198, | |
| "pass@16": 0.7348198506994026, | |
| "pass@32": 0.8285702067226429, | |
| "pass@64": 0.8994453819696483, | |
| "pass@128": 0.9434523809523809 | |
| }, | |
| "20": { | |
| "pass@1": 0.19344124251497005, | |
| "pass@2": 0.3052857130227736, | |
| "pass@4": 0.43797066861777034, | |
| "pass@8": 0.5705444770411404, | |
| "pass@16": 0.6877959075104455, | |
| "pass@32": 0.7850349116495576, | |
| "pass@64": 0.8579923825438611, | |
| "pass@128": 0.907185628742515 | |
| } | |
| } | |
| }, | |
| "movie_festival_awards": { | |
| "count": 418816, | |
| "correct": 138872, | |
| "answer_accuracy": 0.3315823655256724, | |
| "avg_response_len": 394.32354303560516, | |
| "resp_tokens_sum": 165149009, | |
| "pass_at_k": { | |
| "pass@1": 0.3315823655256724, | |
| "pass@2": 0.4765602439212221, | |
| "pass@4": 0.6231470058640338, | |
| "pass@8": 0.7486684552855316, | |
| "pass@16": 0.8408089368301784, | |
| "pass@32": 0.9013491716106101, | |
| "pass@64": 0.940147654873669, | |
| "pass@128": 0.9633251833740831 | |
| }, | |
| "per_op_pass_at_k": { | |
| "11": { | |
| "pass@1": 0.4708829365079365, | |
| "pass@2": 0.6231510123734529, | |
| "pass@4": 0.7544366772605811, | |
| "pass@8": 0.851528774724346, | |
| "pass@16": 0.9152272927693456, | |
| "pass@32": 0.9535655892565871, | |
| "pass@64": 0.9750103454894454, | |
| "pass@128": 0.9841269841269841 | |
| }, | |
| "12": { | |
| "pass@1": 0.43018922018348627, | |
| "pass@2": 0.593299260757543, | |
| "pass@4": 0.7401128262063567, | |
| "pass@8": 0.849012069087901, | |
| "pass@16": 0.916627492993055, | |
| "pass@32": 0.9528701633670399, | |
| "pass@64": 0.9721467280169921, | |
| "pass@128": 0.981651376146789 | |
| }, | |
| "13": { | |
| "pass@1": 0.39754533179012347, | |
| "pass@2": 0.5523874945319335, | |
| "pass@4": 0.6953536343258018, | |
| "pass@8": 0.8065616697672022, | |
| "pass@16": 0.8807671612053861, | |
| "pass@32": 0.9246210517968243, | |
| "pass@64": 0.9498011940741536, | |
| "pass@128": 0.9660493827160493 | |
| }, | |
| "14": { | |
| "pass@1": 0.38318341988130566, | |
| "pass@2": 0.5392996185658544, | |
| "pass@4": 0.6927449403579738, | |
| "pass@8": 0.8184434802350886, | |
| "pass@16": 0.9013070276922327, | |
| "pass@32": 0.9460098896212308, | |
| "pass@64": 0.9683616043843487, | |
| "pass@128": 0.9792284866468842 | |
| }, | |
| "15": { | |
| "pass@1": 0.3361304012345679, | |
| "pass@2": 0.4982395863711481, | |
| "pass@4": 0.6646675548658267, | |
| "pass@8": 0.8041232007332922, | |
| "pass@16": 0.8965907609063177, | |
| "pass@32": 0.9461555988848293, | |
| "pass@64": 0.9736658319254623, | |
| "pass@128": 0.9907407407407407 | |
| }, | |
| "16": { | |
| "pass@1": 0.3105872050147493, | |
| "pass@2": 0.4608186421387594, | |
| "pass@4": 0.6136408792374404, | |
| "pass@8": 0.7452051705964161, | |
| "pass@16": 0.844733132645075, | |
| "pass@32": 0.9102423990647945, | |
| "pass@64": 0.9475723541577755, | |
| "pass@128": 0.9646017699115044 | |
| }, | |
| "17": { | |
| "pass@1": 0.28572100903614456, | |
| "pass@2": 0.4279750646285927, | |
| "pass@4": 0.5799197509949807, | |
| "pass@8": 0.7145075723252433, | |
| "pass@16": 0.8179494250317864, | |
| "pass@32": 0.8902762163574444, | |
| "pass@64": 0.9388825184567935, | |
| "pass@128": 0.9698795180722891 | |
| }, | |
| "18": { | |
| "pass@1": 0.26066028225806454, | |
| "pass@2": 0.39178943357886703, | |
| "pass@4": 0.5395992352770422, | |
| "pass@8": 0.6798815082751558, | |
| "pass@16": 0.7896970588371004, | |
| "pass@32": 0.8658843755573179, | |
| "pass@64": 0.9173571118838663, | |
| "pass@128": 0.9451612903225807 | |
| }, | |
| "19": { | |
| "pass@1": 0.24596393504531722, | |
| "pass@2": 0.37129121369269935, | |
| "pass@4": 0.5090978352970231, | |
| "pass@8": 0.6405254066302252, | |
| "pass@16": 0.7512558089644261, | |
| "pass@32": 0.8355988987470783, | |
| "pass@64": 0.8976115860640348, | |
| "pass@128": 0.9395770392749244 | |
| }, | |
| "20": { | |
| "pass@1": 0.20038006756756757, | |
| "pass@2": 0.3129104744508288, | |
| "pass@4": 0.44692457558796145, | |
| "pass@8": 0.5810482888254881, | |
| "pass@16": 0.6969763264168647, | |
| "pass@32": 0.7902839304149739, | |
| "pass@64": 0.8623890297183104, | |
| "pass@128": 0.9129129129129129 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "total": { | |
| "answer_accuracy": 0.5064206414473684, | |
| "avg_loss": 0.0, | |
| "count": 2432000, | |
| "validation_metrics": { | |
| "id": {}, | |
| "ood": {} | |
| }, | |
| "pass_at_k": { | |
| "pass@1": 0.5064206414473684, | |
| "pass@2": 0.6320177553874659, | |
| "pass@4": 0.7468917626398424, | |
| "pass@8": 0.8366322354900857, | |
| "pass@16": 0.8985551069241822, | |
| "pass@32": 0.9378755395443024, | |
| "pass@64": 0.9622731145890642, | |
| "pass@128": 0.9768947368421053 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5105703125, | |
| "pass@2": 0.6699938484251962, | |
| "pass@4": 0.7991659679415073, | |
| "pass@8": 0.8847154633623784, | |
| "pass@16": 0.9347854223296508, | |
| "pass@32": 0.9632970265066139, | |
| "pass@64": 0.980442548083157, | |
| "pass@128": 0.992 | |
| }, | |
| "2": { | |
| "pass@1": 0.935890625, | |
| "pass@2": 0.9592861712598416, | |
| "pass@4": 0.9768458330521185, | |
| "pass@8": 0.9867971535992609, | |
| "pass@16": 0.9919445464167224, | |
| "pass@32": 0.9942186611862515, | |
| "pass@64": 0.9948690786426989, | |
| "pass@128": 0.995 | |
| }, | |
| "3": { | |
| "pass@1": 0.93975, | |
| "pass@2": 0.9609361466535429, | |
| "pass@4": 0.9775336966629172, | |
| "pass@8": 0.9882813969399472, | |
| "pass@16": 0.9934026899376711, | |
| "pass@32": 0.9952107555410625, | |
| "pass@64": 0.9959019362061534, | |
| "pass@128": 0.996 | |
| }, | |
| "4": { | |
| "pass@1": 0.806453125, | |
| "pass@2": 0.8676486220472444, | |
| "pass@4": 0.911353344019498, | |
| "pass@8": 0.9400612877585252, | |
| "pass@16": 0.9589647416451935, | |
| "pass@32": 0.9720786045524892, | |
| "pass@64": 0.9811436774483744, | |
| "pass@128": 0.987 | |
| }, | |
| "5": { | |
| "pass@1": 0.719234375, | |
| "pass@2": 0.8196967273622047, | |
| "pass@4": 0.8851043299587543, | |
| "pass@8": 0.9222199389254376, | |
| "pass@16": 0.9448174036914444, | |
| "pass@32": 0.9598280784653056, | |
| "pass@64": 0.9709283962665971, | |
| "pass@128": 0.98 | |
| }, | |
| "6": { | |
| "pass@1": 0.698921875, | |
| "pass@2": 0.809659571850394, | |
| "pass@4": 0.8861456253280843, | |
| "pass@8": 0.9296744960557235, | |
| "pass@16": 0.9532426321493441, | |
| "pass@32": 0.9681859214131414, | |
| "pass@64": 0.9788188969365212, | |
| "pass@128": 0.986 | |
| }, | |
| "7": { | |
| "pass@1": 0.618640625, | |
| "pass@2": 0.7534297490157486, | |
| "pass@4": 0.8522904651293589, | |
| "pass@8": 0.9134351163840331, | |
| "pass@16": 0.9500415651070467, | |
| "pass@32": 0.9713268310837263, | |
| "pass@64": 0.9833246380222623, | |
| "pass@128": 0.99 | |
| }, | |
| "8": { | |
| "pass@1": 0.57221875, | |
| "pass@2": 0.7231927903543292, | |
| "pass@4": 0.8385967574990625, | |
| "pass@8": 0.909620771341596, | |
| "pass@16": 0.9483642685489777, | |
| "pass@32": 0.9693944224018802, | |
| "pass@64": 0.9815823234229324, | |
| "pass@128": 0.989 | |
| }, | |
| "9": { | |
| "pass@1": 0.50253125, | |
| "pass@2": 0.6644630905511801, | |
| "pass@4": 0.8022588066179234, | |
| "pass@8": 0.8951953231778146, | |
| "pass@16": 0.9475207703133638, | |
| "pass@32": 0.9742983264269067, | |
| "pass@64": 0.9877455283243902, | |
| "pass@128": 0.994 | |
| }, | |
| "11": { | |
| "pass@1": 0.4547109375, | |
| "pass@2": 0.6138021653543304, | |
| "pass@4": 0.7549666171728527, | |
| "pass@8": 0.8583315153211275, | |
| "pass@16": 0.9221412824985413, | |
| "pass@32": 0.9570664977233017, | |
| "pass@64": 0.974931983836227, | |
| "pass@128": 0.983 | |
| }, | |
| "12": { | |
| "pass@1": 0.4180625, | |
| "pass@2": 0.5797405265748027, | |
| "pass@4": 0.7296782709973749, | |
| "pass@8": 0.8443357826368147, | |
| "pass@16": 0.916390867717859, | |
| "pass@32": 0.9551260363971661, | |
| "pass@64": 0.975269380378476, | |
| "pass@128": 0.986 | |
| }, | |
| "13": { | |
| "pass@1": 0.396984375, | |
| "pass@2": 0.5582675935039371, | |
| "pass@4": 0.7075908911698547, | |
| "pass@8": 0.8218632521719159, | |
| "pass@16": 0.8957610496843468, | |
| "pass@32": 0.9375014501187352, | |
| "pass@64": 0.9603788119235547, | |
| "pass@128": 0.975 | |
| }, | |
| "14": { | |
| "pass@1": 0.3836953125, | |
| "pass@2": 0.5430013533464566, | |
| "pass@4": 0.6984549793775782, | |
| "pass@8": 0.8231021884341475, | |
| "pass@16": 0.9053910203817964, | |
| "pass@32": 0.9510216257343269, | |
| "pass@64": 0.9741567124841395, | |
| "pass@128": 0.985 | |
| }, | |
| "15": { | |
| "pass@1": 0.33775, | |
| "pass@2": 0.4967496309055121, | |
| "pass@4": 0.6602598811398573, | |
| "pass@8": 0.7980327584444624, | |
| "pass@16": 0.8912756658908414, | |
| "pass@32": 0.9429084818635887, | |
| "pass@64": 0.9696608164732217, | |
| "pass@128": 0.983 | |
| }, | |
| "16": { | |
| "pass@1": 0.3182265625, | |
| "pass@2": 0.4695752952755903, | |
| "pass@4": 0.6255005896137982, | |
| "pass@8": 0.7595363695753027, | |
| "pass@16": 0.8578304309643211, | |
| "pass@32": 0.9197597659036819, | |
| "pass@64": 0.9536475067446567, | |
| "pass@128": 0.972 | |
| }, | |
| "17": { | |
| "pass@1": 0.299875, | |
| "pass@2": 0.4421136811023624, | |
| "pass@4": 0.5927719518185219, | |
| "pass@8": 0.7277104389245667, | |
| "pass@16": 0.8309745449337134, | |
| "pass@32": 0.9007491803788847, | |
| "pass@64": 0.9454100163755238, | |
| "pass@128": 0.972 | |
| }, | |
| "18": { | |
| "pass@1": 0.272328125, | |
| "pass@2": 0.40941916830708663, | |
| "pass@4": 0.5598332344394451, | |
| "pass@8": 0.6993310814147358, | |
| "pass@16": 0.808673856643445, | |
| "pass@32": 0.8838542581208818, | |
| "pass@64": 0.9322295342493997, | |
| "pass@128": 0.957 | |
| }, | |
| "19": { | |
| "pass@1": 0.2403671875, | |
| "pass@2": 0.3612985974409448, | |
| "pass@4": 0.49474866122984607, | |
| "pass@8": 0.6234513899075376, | |
| "pass@16": 0.7351839126274962, | |
| "pass@32": 0.8242222274517007, | |
| "pass@64": 0.8913528812708557, | |
| "pass@128": 0.937 | |
| }, | |
| "20": { | |
| "pass@1": 0.19578125, | |
| "pass@2": 0.3060626230314962, | |
| "pass@4": 0.43784358698912595, | |
| "pass@8": 0.5703167499362239, | |
| "pass@16": 0.6858403600775729, | |
| "pass@32": 0.7795871000716412, | |
| "pass@64": 0.8513945101029146, | |
| "pass@128": 0.902 | |
| } | |
| }, | |
| "per_op_accuracy": { | |
| "10": 0.5105703125, | |
| "2": 0.935890625, | |
| "3": 0.93975, | |
| "4": 0.806453125, | |
| "5": 0.719234375, | |
| "6": 0.698921875, | |
| "7": 0.618640625, | |
| "8": 0.57221875, | |
| "9": 0.50253125, | |
| "11": 0.4547109375, | |
| "12": 0.4180625, | |
| "13": 0.396984375, | |
| "14": 0.3836953125, | |
| "15": 0.33775, | |
| "16": 0.3182265625, | |
| "17": 0.299875, | |
| "18": 0.272328125, | |
| "19": 0.2403671875, | |
| "20": 0.19578125 | |
| }, | |
| "per_op_avg_response_len": { | |
| "10": 280.0350625, | |
| "2": 116.76371875, | |
| "3": 149.778203125, | |
| "4": 158.538859375, | |
| "5": 173.8068046875, | |
| "6": 193.58321875, | |
| "7": 222.0638046875, | |
| "8": 240.2696328125, | |
| "9": 255.7865234375, | |
| "11": 296.70078125, | |
| "12": 310.4506328125, | |
| "13": 328.7139609375, | |
| "14": 352.013390625, | |
| "15": 373.0260234375, | |
| "16": 381.777359375, | |
| "17": 401.5364765625, | |
| "18": 417.4558203125, | |
| "19": 421.0910859375, | |
| "20": 434.3028828125 | |
| }, | |
| "per_op_avg_loss": {}, | |
| "per_op_length": { | |
| "10": 280.0350625, | |
| "2": 116.76371875, | |
| "3": 149.778203125, | |
| "4": 158.538859375, | |
| "5": 173.8068046875, | |
| "6": 193.58321875, | |
| "7": 222.0638046875, | |
| "8": 240.2696328125, | |
| "9": 255.7865234375, | |
| "11": 296.70078125, | |
| "12": 310.4506328125, | |
| "13": 328.7139609375, | |
| "14": 352.013390625, | |
| "15": 373.0260234375, | |
| "16": 381.777359375, | |
| "17": 401.5364765625, | |
| "18": 417.4558203125, | |
| "19": 421.0910859375, | |
| "20": 434.3028828125 | |
| }, | |
| "per_op_loss": {}, | |
| "per_template": { | |
| "crazy_zootopia": { | |
| "count": 809728, | |
| "correct": 401802, | |
| "answer_accuracy": 0.49621848324375595, | |
| "avg_response_len": 276.35492288768575, | |
| "resp_tokens_sum": 223772319, | |
| "pass_at_k": { | |
| "pass@1": 0.49621848324375595, | |
| "pass@2": 0.6231529911240028, | |
| "pass@4": 0.740067849446409, | |
| "pass@8": 0.8317378367298127, | |
| "pass@16": 0.8949596717478773, | |
| "pass@32": 0.9350559220713367, | |
| "pass@64": 0.9600439120925355, | |
| "pass@128": 0.9753398672146696 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5095409292035398, | |
| "pass@2": 0.6676066418135783, | |
| "pass@4": 0.792585876710101, | |
| "pass@8": 0.8760817636863799, | |
| "pass@16": 0.9283330831148673, | |
| "pass@32": 0.9602328298448352, | |
| "pass@64": 0.9791876634794598, | |
| "pass@128": 0.9911504424778761 | |
| }, | |
| "2": { | |
| "pass@1": 0.9343360591900312, | |
| "pass@2": 0.9592853901439887, | |
| "pass@4": 0.9779000282674944, | |
| "pass@8": 0.9878251114374832, | |
| "pass@16": 0.9922969056587208, | |
| "pass@32": 0.9936688924295337, | |
| "pass@64": 0.9937693490770984, | |
| "pass@128": 0.9937694704049844 | |
| }, | |
| "3": { | |
| "pass@1": 0.9195575305810397, | |
| "pass@2": 0.9484343771822101, | |
| "pass@4": 0.9708153361563746, | |
| "pass@8": 0.984238780440124, | |
| "pass@16": 0.9897953555580769, | |
| "pass@32": 0.990792748570468, | |
| "pass@64": 0.9908256717392108, | |
| "pass@128": 0.9908256880733946 | |
| }, | |
| "4": { | |
| "pass@1": 0.7866455078125, | |
| "pass@2": 0.8459907111220479, | |
| "pass@4": 0.8906744697928376, | |
| "pass@8": 0.9215507509441558, | |
| "pass@16": 0.9437763487973065, | |
| "pass@32": 0.9611323349556897, | |
| "pass@64": 0.9735555459387797, | |
| "pass@128": 0.98125 | |
| }, | |
| "5": { | |
| "pass@1": 0.6709573412698413, | |
| "pass@2": 0.7741130014998127, | |
| "pass@4": 0.8445359746698321, | |
| "pass@8": 0.8894758925880082, | |
| "pass@16": 0.9211739208974751, | |
| "pass@32": 0.9452852732182677, | |
| "pass@64": 0.964295583575878, | |
| "pass@128": 0.9777777777777777 | |
| }, | |
| "6": { | |
| "pass@1": 0.7365503368263473, | |
| "pass@2": 0.8420378730727522, | |
| "pass@4": 0.9124382115970529, | |
| "pass@8": 0.9505378525143745, | |
| "pass@16": 0.9685954065179547, | |
| "pass@32": 0.97972634565819, | |
| "pass@64": 0.9880378752295537, | |
| "pass@128": 0.9940119760479041 | |
| }, | |
| "7": { | |
| "pass@1": 0.6175110946745562, | |
| "pass@2": 0.7572351983646276, | |
| "pass@4": 0.8602960354807709, | |
| "pass@8": 0.9199085087926454, | |
| "pass@16": 0.9515635930824008, | |
| "pass@32": 0.9699748728805815, | |
| "pass@64": 0.9825938231042854, | |
| "pass@128": 0.9911242603550295 | |
| }, | |
| "8": { | |
| "pass@1": 0.5553635817307693, | |
| "pass@2": 0.705602191853422, | |
| "pass@4": 0.821673709956928, | |
| "pass@8": 0.8943235864636814, | |
| "pass@16": 0.9343372725706356, | |
| "pass@32": 0.9560323516975169, | |
| "pass@64": 0.9697434965886538, | |
| "pass@128": 0.9807692307692307 | |
| }, | |
| "9": { | |
| "pass@1": 0.479137323943662, | |
| "pass@2": 0.6481472496395699, | |
| "pass@4": 0.7957091023833289, | |
| "pass@8": 0.8944526189705332, | |
| "pass@16": 0.9475698851465509, | |
| "pass@32": 0.9725689666299432, | |
| "pass@64": 0.9838555505224045, | |
| "pass@128": 0.9887323943661972 | |
| }, | |
| "11": { | |
| "pass@1": 0.4631506728045326, | |
| "pass@2": 0.6252648836742432, | |
| "pass@4": 0.767624272798053, | |
| "pass@8": 0.871077584229325, | |
| "pass@16": 0.9329364153366072, | |
| "pass@32": 0.9639094963103593, | |
| "pass@64": 0.9780315644820233, | |
| "pass@128": 0.9858356940509915 | |
| }, | |
| "12": { | |
| "pass@1": 0.4075962217514124, | |
| "pass@2": 0.5693932392455183, | |
| "pass@4": 0.7213146841814255, | |
| "pass@8": 0.8371093473559977, | |
| "pass@16": 0.9086267449422752, | |
| "pass@32": 0.9469433243538209, | |
| "pass@64": 0.9686512377816593, | |
| "pass@128": 0.9830508474576272 | |
| }, | |
| "13": { | |
| "pass@1": 0.39882934952978055, | |
| "pass@2": 0.5654986516673662, | |
| "pass@4": 0.7186619650600091, | |
| "pass@8": 0.8329572629488781, | |
| "pass@16": 0.9032880963070734, | |
| "pass@32": 0.9407221789825072, | |
| "pass@64": 0.9613466943723601, | |
| "pass@128": 0.9749216300940439 | |
| }, | |
| "14": { | |
| "pass@1": 0.35604069314641745, | |
| "pass@2": 0.5193354919420117, | |
| "pass@4": 0.684745257018106, | |
| "pass@8": 0.8188914601096546, | |
| "pass@16": 0.9064022768205324, | |
| "pass@32": 0.9520687996104888, | |
| "pass@64": 0.9726630069903832, | |
| "pass@128": 0.9813084112149533 | |
| }, | |
| "15": { | |
| "pass@1": 0.33921152456647397, | |
| "pass@2": 0.4931290820854757, | |
| "pass@4": 0.650143435556827, | |
| "pass@8": 0.7837146243177703, | |
| "pass@16": 0.8784636507395034, | |
| "pass@32": 0.9341449136359978, | |
| "pass@64": 0.9638817286290657, | |
| "pass@128": 0.9797687861271677 | |
| }, | |
| "16": { | |
| "pass@1": 0.3211945564516129, | |
| "pass@2": 0.47427451104902224, | |
| "pass@4": 0.6338670295245349, | |
| "pass@8": 0.7704789743204787, | |
| "pass@16": 0.8674678402262296, | |
| "pass@32": 0.9269334345131763, | |
| "pass@64": 0.95867323469044, | |
| "pass@128": 0.9774193548387097 | |
| }, | |
| "17": { | |
| "pass@1": 0.2943686777456647, | |
| "pass@2": 0.43410561080515186, | |
| "pass@4": 0.5843995006404546, | |
| "pass@8": 0.7225977142959461, | |
| "pass@16": 0.8319339455322081, | |
| "pass@32": 0.9085076669393349, | |
| "pass@64": 0.9564128585594077, | |
| "pass@128": 0.976878612716763 | |
| }, | |
| "18": { | |
| "pass@1": 0.2702678571428571, | |
| "pass@2": 0.40575260123734536, | |
| "pass@4": 0.5552940701162348, | |
| "pass@8": 0.6981102462106233, | |
| "pass@16": 0.8141778003682741, | |
| "pass@32": 0.894818854884761, | |
| "pass@64": 0.9449149724164841, | |
| "pass@128": 0.9685714285714285 | |
| }, | |
| "19": { | |
| "pass@1": 0.23252158408408408, | |
| "pass@2": 0.34998618204819004, | |
| "pass@4": 0.48075028543729326, | |
| "pass@8": 0.6082667201968596, | |
| "pass@16": 0.7195758859617793, | |
| "pass@32": 0.8085267342570868, | |
| "pass@64": 0.8769663601857598, | |
| "pass@128": 0.9279279279279279 | |
| }, | |
| "20": { | |
| "pass@1": 0.19352946696696696, | |
| "pass@2": 0.29999401468397513, | |
| "pass@4": 0.42863513513513496, | |
| "pass@8": 0.5593568000768633, | |
| "pass@16": 0.6727429737905991, | |
| "pass@32": 0.763426098355866, | |
| "pass@64": 0.8337823046158129, | |
| "pass@128": 0.8858858858858859 | |
| } | |
| } | |
| }, | |
| "teachers_in_school": { | |
| "count": 821120, | |
| "correct": 420450, | |
| "answer_accuracy": 0.5120445245518317, | |
| "avg_response_len": 286.99597988113794, | |
| "resp_tokens_sum": 235658139, | |
| "pass_at_k": { | |
| "pass@1": 0.5120445245518317, | |
| "pass@2": 0.6380669421753976, | |
| "pass@4": 0.7534741674827613, | |
| "pass@8": 0.8433371855207411, | |
| "pass@16": 0.904921209981468, | |
| "pass@32": 0.9434274968633595, | |
| "pass@64": 0.9665472920084971, | |
| "pass@128": 0.9800467653936087 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.4727521088957055, | |
| "pass@2": 0.6345907957345057, | |
| "pass@4": 0.77542852455943, | |
| "pass@8": 0.8752167459860357, | |
| "pass@16": 0.9336091179643451, | |
| "pass@32": 0.9627348671897347, | |
| "pass@64": 0.9780142800175672, | |
| "pass@128": 0.9877300613496932 | |
| }, | |
| "2": { | |
| "pass@1": 0.9381648936170213, | |
| "pass@2": 0.9569980614125363, | |
| "pass@4": 0.971669399322805, | |
| "pass@8": 0.9823670230919718, | |
| "pass@16": 0.9902618175090221, | |
| "pass@32": 0.9948116541729152, | |
| "pass@64": 0.9965635996372417, | |
| "pass@128": 0.9969604863221885 | |
| }, | |
| "3": { | |
| "pass@1": 0.950686553030303, | |
| "pass@2": 0.9672959914101645, | |
| "pass@4": 0.9803795730079196, | |
| "pass@8": 0.9893317365210512, | |
| "pass@16": 0.9939045470160155, | |
| "pass@32": 0.9959745105782435, | |
| "pass@64": 0.9968729838687728, | |
| "pass@128": 0.996969696969697 | |
| }, | |
| "4": { | |
| "pass@1": 0.8468703772189349, | |
| "pass@2": 0.9065033429623076, | |
| "pass@4": 0.9489256611044922, | |
| "pass@8": 0.9743967832425964, | |
| "pass@16": 0.9872230174014897, | |
| "pass@32": 0.9930567852310904, | |
| "pass@64": 0.9965815370797454, | |
| "pass@128": 1.0 | |
| }, | |
| "5": { | |
| "pass@1": 0.7758605072463768, | |
| "pass@2": 0.8736209774049982, | |
| "pass@4": 0.9324389600756424, | |
| "pass@8": 0.9618868319100985, | |
| "pass@16": 0.9760629639219559, | |
| "pass@32": 0.9825935008857577, | |
| "pass@64": 0.9869077034509552, | |
| "pass@128": 0.991304347826087 | |
| }, | |
| "6": { | |
| "pass@1": 0.6769340974212035, | |
| "pass@2": 0.7943660740473344, | |
| "pass@4": 0.8786454614018506, | |
| "pass@8": 0.9272978475082277, | |
| "pass@16": 0.9543528148099992, | |
| "pass@32": 0.9711407927719327, | |
| "pass@64": 0.9821259539329217, | |
| "pass@128": 0.9885386819484241 | |
| }, | |
| "7": { | |
| "pass@1": 0.6066428829479769, | |
| "pass@2": 0.7398836818988667, | |
| "pass@4": 0.8397892167380809, | |
| "pass@8": 0.9042434643738783, | |
| "pass@16": 0.9441212894901903, | |
| "pass@32": 0.96651931919803, | |
| "pass@64": 0.9778479793557311, | |
| "pass@128": 0.9826589595375722 | |
| }, | |
| "8": { | |
| "pass@1": 0.5825044014084507, | |
| "pass@2": 0.7344803565487412, | |
| "pass@4": 0.8506169114424071, | |
| "pass@8": 0.9214648743029239, | |
| "pass@16": 0.9596192806093881, | |
| "pass@32": 0.9801233610769453, | |
| "pass@64": 0.9908749402461413, | |
| "pass@128": 0.9971830985915493 | |
| }, | |
| "9": { | |
| "pass@1": 0.5183383233532934, | |
| "pass@2": 0.6750163550851055, | |
| "pass@4": 0.8066507348071007, | |
| "pass@8": 0.8964758234235036, | |
| "pass@16": 0.9496126362729699, | |
| "pass@32": 0.9775149761319307, | |
| "pass@64": 0.9920008501197732, | |
| "pass@128": 1.0 | |
| }, | |
| "11": { | |
| "pass@1": 0.4303934487951807, | |
| "pass@2": 0.5927442545773641, | |
| "pass@4": 0.7420111311387282, | |
| "pass@8": 0.851233626656646, | |
| "pass@16": 0.9172232851276387, | |
| "pass@32": 0.9531123039455416, | |
| "pass@64": 0.9715619962195704, | |
| "pass@128": 0.9789156626506024 | |
| }, | |
| "12": { | |
| "pass@1": 0.41724627742946707, | |
| "pass@2": 0.577324331078913, | |
| "pass@4": 0.7282632558861173, | |
| "pass@8": 0.8475615268998387, | |
| "pass@16": 0.9247642940425248, | |
| "pass@32": 0.9665189848115742, | |
| "pass@64": 0.9858146148658699, | |
| "pass@128": 0.9937304075235109 | |
| }, | |
| "13": { | |
| "pass@1": 0.3948266806722689, | |
| "pass@2": 0.5571427882176495, | |
| "pass@4": 0.7088043327567247, | |
| "pass@8": 0.8258372669093875, | |
| "pass@16": 0.9026431000892041, | |
| "pass@32": 0.9463133172020857, | |
| "pass@64": 0.9691138082317794, | |
| "pass@128": 0.9831932773109243 | |
| }, | |
| "14": { | |
| "pass@1": 0.41015625, | |
| "pass@2": 0.5688616636736198, | |
| "pass@4": 0.7169494355968666, | |
| "pass@8": 0.831644961695096, | |
| "pass@16": 0.9084661438307753, | |
| "pass@32": 0.9549772171257439, | |
| "pass@64": 0.9812690835166658, | |
| "pass@128": 0.9941520467836257 | |
| }, | |
| "15": { | |
| "pass@1": 0.33780776515151517, | |
| "pass@2": 0.4990828561202573, | |
| "pass@4": 0.6665392868505067, | |
| "pass@8": 0.807065398160388, | |
| "pass@16": 0.8994904127312967, | |
| "pass@32": 0.9489088718389327, | |
| "pass@64": 0.9717879055264091, | |
| "pass@128": 0.9787878787878788 | |
| }, | |
| "16": { | |
| "pass@1": 0.32298344017094016, | |
| "pass@2": 0.4738822711263657, | |
| "pass@4": 0.6295656763524213, | |
| "pass@8": 0.7637132042842444, | |
| "pass@16": 0.8619682864031663, | |
| "pass@32": 0.9226160339647616, | |
| "pass@64": 0.9550762847040853, | |
| "pass@128": 0.9743589743589743 | |
| }, | |
| "17": { | |
| "pass@1": 0.32038528726708076, | |
| "pass@2": 0.4652963301462315, | |
| "pass@4": 0.6150197741198496, | |
| "pass@8": 0.7468171297086604, | |
| "pass@16": 0.8433732629472633, | |
| "pass@32": 0.9032106328795099, | |
| "pass@64": 0.9403173018208404, | |
| "pass@128": 0.968944099378882 | |
| }, | |
| "18": { | |
| "pass@1": 0.2850873161764706, | |
| "pass@2": 0.4292677454840204, | |
| "pass@4": 0.5829545498908223, | |
| "pass@8": 0.7183212578697612, | |
| "pass@16": 0.8203104066913215, | |
| "pass@32": 0.8889514779071954, | |
| "pass@64": 0.932731144763619, | |
| "pass@128": 0.9558823529411765 | |
| }, | |
| "19": { | |
| "pass@1": 0.24262927827380953, | |
| "pass@2": 0.3626660925196849, | |
| "pass@4": 0.4944864068330747, | |
| "pass@8": 0.621680543117198, | |
| "pass@16": 0.7348198506994026, | |
| "pass@32": 0.8285702067226429, | |
| "pass@64": 0.8994453819696483, | |
| "pass@128": 0.9434523809523809 | |
| }, | |
| "20": { | |
| "pass@1": 0.19344124251497005, | |
| "pass@2": 0.3052857130227736, | |
| "pass@4": 0.43797066861777034, | |
| "pass@8": 0.5705444770411404, | |
| "pass@16": 0.6877959075104455, | |
| "pass@32": 0.7850349116495576, | |
| "pass@64": 0.8579923825438611, | |
| "pass@128": 0.907185628742515 | |
| } | |
| } | |
| }, | |
| "movie_festival_awards": { | |
| "count": 801152, | |
| "correct": 409363, | |
| "answer_accuracy": 0.5109679561431538, | |
| "avg_response_len": 306.50164388280876, | |
| "resp_tokens_sum": 245554405, | |
| "pass_at_k": { | |
| "pass@1": 0.5109679561431538, | |
| "pass@2": 0.6347774558965905, | |
| "pass@4": 0.7470422575742857, | |
| "pass@8": 0.8347069618219127, | |
| "pass@16": 0.8956642572378208, | |
| "pass@32": 0.9350350049432259, | |
| "pass@64": 0.9601454722895479, | |
| "pass@128": 0.975235660648666 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.5484141791044777, | |
| "pass@2": 0.7068614848983433, | |
| "pass@4": 0.8289243484489802, | |
| "pass@8": 0.9026957799440277, | |
| "pass@16": 0.9424595065592135, | |
| "pass@32": 0.9669448671204803, | |
| "pass@64": 0.9840754473966935, | |
| "pass@128": 0.9970149253731343 | |
| }, | |
| "2": { | |
| "pass@1": 0.9351785714285714, | |
| "pass@2": 0.9614377109111365, | |
| "pass@4": 0.9807448331458566, | |
| "pass@8": 0.9900186920873438, | |
| "pass@16": 0.9932031478280128, | |
| "pass@32": 0.994165464267091, | |
| "pass@64": 0.994284838023708, | |
| "pass@128": 0.9942857142857143 | |
| }, | |
| "3": { | |
| "pass@1": 0.9484784985422741, | |
| "pass@2": 0.9667359421271322, | |
| "pass@4": 0.9812006491171111, | |
| "pass@8": 0.9911249057844892, | |
| "pass@16": 0.9963589159034837, | |
| "pass@32": 0.9986878666696775, | |
| "pass@64": 0.9998071628884445, | |
| "pass@128": 1.0 | |
| }, | |
| "4": { | |
| "pass@1": 0.7850420321637427, | |
| "pass@2": 0.8495130542892672, | |
| "pass@4": 0.8935691234867573, | |
| "pass@8": 0.9234471775450245, | |
| "pass@16": 0.9452483337671119, | |
| "pass@32": 0.961587906311577, | |
| "pass@64": 0.9729863836696224, | |
| "pass@128": 0.97953216374269 | |
| }, | |
| "5": { | |
| "pass@1": 0.7065027573529412, | |
| "pass@2": 0.8072114549559978, | |
| "pass@4": 0.8746589903284145, | |
| "pass@8": 0.9123061051506803, | |
| "pass@16": 0.9350173413401953, | |
| "pass@32": 0.9502013517528385, | |
| "pass@64": 0.9608592051459295, | |
| "pass@128": 0.9705882352941176 | |
| }, | |
| "6": { | |
| "pass@1": 0.6834828470031545, | |
| "pass@2": 0.7923822157778388, | |
| "pass@4": 0.8667003048120562, | |
| "pass@8": 0.9103088471153025, | |
| "pass@16": 0.9358442713049175, | |
| "pass@32": 0.9527734551605721, | |
| "pass@64": 0.9654646330828386, | |
| "pass@128": 0.9747634069400631 | |
| }, | |
| "7": { | |
| "pass@1": 0.6329855617088608, | |
| "pass@2": 0.7641914494667598, | |
| "pass@4": 0.8574156207135493, | |
| "pass@8": 0.9165753219580909, | |
| "pass@16": 0.9548959002581976, | |
| "pass@32": 0.9780368342011752, | |
| "pass@64": 0.990102927075729, | |
| "pass@128": 0.9968354430379747 | |
| }, | |
| "8": { | |
| "pass@1": 0.5770457957957958, | |
| "pass@2": 0.7276407805443237, | |
| "pass@4": 0.8416383075989373, | |
| "pass@8": 0.911326672634802, | |
| "pass@16": 0.949508092764589, | |
| "pass@32": 0.9704760825523712, | |
| "pass@64": 0.98276801411379, | |
| "pass@128": 0.987987987987988 | |
| }, | |
| "9": { | |
| "pass@1": 0.5122588424437299, | |
| "pass@2": 0.6717535509026002, | |
| "pass@4": 0.8050184239429874, | |
| "pass@8": 0.8946679048869003, | |
| "pass@16": 0.9452181368847813, | |
| "pass@32": 0.9728178175087213, | |
| "pass@64": 0.9876158326332224, | |
| "pass@128": 0.9935691318327974 | |
| }, | |
| "11": { | |
| "pass@1": 0.4708829365079365, | |
| "pass@2": 0.6231510123734529, | |
| "pass@4": 0.7544366772605811, | |
| "pass@8": 0.851528774724346, | |
| "pass@16": 0.9152272927693456, | |
| "pass@32": 0.9535655892565871, | |
| "pass@64": 0.9750103454894454, | |
| "pass@128": 0.9841269841269841 | |
| }, | |
| "12": { | |
| "pass@1": 0.43018922018348627, | |
| "pass@2": 0.593299260757543, | |
| "pass@4": 0.7401128262063567, | |
| "pass@8": 0.849012069087901, | |
| "pass@16": 0.916627492993055, | |
| "pass@32": 0.9528701633670399, | |
| "pass@64": 0.9721467280169921, | |
| "pass@128": 0.981651376146789 | |
| }, | |
| "13": { | |
| "pass@1": 0.39754533179012347, | |
| "pass@2": 0.5523874945319335, | |
| "pass@4": 0.6953536343258018, | |
| "pass@8": 0.8065616697672022, | |
| "pass@16": 0.8807671612053861, | |
| "pass@32": 0.9246210517968243, | |
| "pass@64": 0.9498011940741536, | |
| "pass@128": 0.9660493827160493 | |
| }, | |
| "14": { | |
| "pass@1": 0.38318341988130566, | |
| "pass@2": 0.5392996185658544, | |
| "pass@4": 0.6927449403579738, | |
| "pass@8": 0.8184434802350886, | |
| "pass@16": 0.9013070276922327, | |
| "pass@32": 0.9460098896212308, | |
| "pass@64": 0.9683616043843487, | |
| "pass@128": 0.9792284866468842 | |
| }, | |
| "15": { | |
| "pass@1": 0.3361304012345679, | |
| "pass@2": 0.4982395863711481, | |
| "pass@4": 0.6646675548658267, | |
| "pass@8": 0.8041232007332922, | |
| "pass@16": 0.8965907609063177, | |
| "pass@32": 0.9461555988848293, | |
| "pass@64": 0.9736658319254623, | |
| "pass@128": 0.9907407407407407 | |
| }, | |
| "16": { | |
| "pass@1": 0.3105872050147493, | |
| "pass@2": 0.4608186421387594, | |
| "pass@4": 0.6136408792374404, | |
| "pass@8": 0.7452051705964161, | |
| "pass@16": 0.844733132645075, | |
| "pass@32": 0.9102423990647945, | |
| "pass@64": 0.9475723541577755, | |
| "pass@128": 0.9646017699115044 | |
| }, | |
| "17": { | |
| "pass@1": 0.28572100903614456, | |
| "pass@2": 0.4279750646285927, | |
| "pass@4": 0.5799197509949807, | |
| "pass@8": 0.7145075723252433, | |
| "pass@16": 0.8179494250317864, | |
| "pass@32": 0.8902762163574444, | |
| "pass@64": 0.9388825184567935, | |
| "pass@128": 0.9698795180722891 | |
| }, | |
| "18": { | |
| "pass@1": 0.26066028225806454, | |
| "pass@2": 0.39178943357886703, | |
| "pass@4": 0.5395992352770422, | |
| "pass@8": 0.6798815082751558, | |
| "pass@16": 0.7896970588371004, | |
| "pass@32": 0.8658843755573179, | |
| "pass@64": 0.9173571118838663, | |
| "pass@128": 0.9451612903225807 | |
| }, | |
| "19": { | |
| "pass@1": 0.24596393504531722, | |
| "pass@2": 0.37129121369269935, | |
| "pass@4": 0.5090978352970231, | |
| "pass@8": 0.6405254066302252, | |
| "pass@16": 0.7512558089644261, | |
| "pass@32": 0.8355988987470783, | |
| "pass@64": 0.8976115860640348, | |
| "pass@128": 0.9395770392749244 | |
| }, | |
| "20": { | |
| "pass@1": 0.20038006756756757, | |
| "pass@2": 0.3129104744508288, | |
| "pass@4": 0.44692457558796145, | |
| "pass@8": 0.5810482888254881, | |
| "pass@16": 0.6969763264168647, | |
| "pass@32": 0.7902839304149739, | |
| "pass@64": 0.8623890297183104, | |
| "pass@128": 0.9129129129129129 | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |