| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.9849149305555556, |
| "ood_acc": 0.43210078125, |
| "total_acc": 0.693960115131579, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.25730208333334, |
| "ood_avg_resp_len": 323.8433125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 557621, |
| "answer_accuracy": 0.6886522387764781, |
| "avg_response_len": 251.28136361840026, |
| "resp_tokens_sum": 203469556, |
| "pass_at_k": { |
| "pass@1": 0.6886522387764781, |
| "pass@2": 0.7237809107395733, |
| "pass@4": 0.7521360810474842, |
| "pass@8": 0.7753498817050547, |
| "pass@16": 0.7949650075503566, |
| "pass@32": 0.8127696664662291, |
| "pass@64": 0.8294527524537995, |
| "pass@128": 0.8450837812203604 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9307015117994101, |
| "pass@2": 0.9492464230134947, |
| "pass@4": 0.9603176585780757, |
| "pass@8": 0.9672270820964995, |
| "pass@16": 0.9719429842512252, |
| "pass@32": 0.9760390588911826, |
| "pass@64": 0.9796358752047998, |
| "pass@128": 0.9823008849557522 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999761085626911, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9964111328125, |
| "pass@2": 0.9968580831692913, |
| "pass@4": 0.9969715208450506, |
| "pass@8": 0.9970703123347953, |
| "pass@16": 0.997265625, |
| "pass@32": 0.99765625, |
| "pass@64": 0.9984375, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9848958333333333, |
| "pass@2": 0.9892681383577052, |
| "pass@4": 0.9919947530368227, |
| "pass@8": 0.9940923063815486, |
| "pass@16": 0.9957625810438835, |
| "pass@32": 0.9966789269921299, |
| "pass@64": 0.9968243884338579, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9963276571856288, |
| "pass@2": 0.9970273527747655, |
| "pass@4": 0.9970994958901095, |
| "pass@8": 0.9971931137724341, |
| "pass@16": 0.9973802395209581, |
| "pass@32": 0.9977544910179641, |
| "pass@64": 0.9985029940119761, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9932969674556213, |
| "pass@2": 0.9977625011647955, |
| "pass@4": 0.9993333954557454, |
| "pass@8": 0.9998893882949348, |
| "pass@16": 0.9999969360251098, |
| "pass@32": 0.999999999201496, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9801933092948718, |
| "pass@2": 0.987775322405613, |
| "pass@4": 0.9924649139410457, |
| "pass@8": 0.9956971600694451, |
| "pass@16": 0.9977654601169499, |
| "pass@32": 0.9989849709320879, |
| "pass@64": 0.999809098401515, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.977068661971831, |
| "pass@2": 0.9877346262615062, |
| "pass@4": 0.99279322892737, |
| "pass@8": 0.9960030095287996, |
| "pass@16": 0.9979625327781841, |
| "pass@32": 0.9991010683792878, |
| "pass@64": 0.9998322132410451, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8986366855524079, |
| "pass@2": 0.9266491796970843, |
| "pass@4": 0.9446660811343338, |
| "pass@8": 0.9557997755282336, |
| "pass@16": 0.9619171555267607, |
| "pass@32": 0.9669011364421911, |
| "pass@64": 0.9723664309673853, |
| "pass@128": 0.9773371104815864 |
| }, |
| "12": { |
| "pass@1": 0.7938073799435028, |
| "pass@2": 0.8475351300102323, |
| "pass@4": 0.8779464293658206, |
| "pass@8": 0.898811393022897, |
| "pass@16": 0.9162384786765374, |
| "pass@32": 0.9324306603335383, |
| "pass@64": 0.9477865031402798, |
| "pass@128": 0.96045197740113 |
| }, |
| "13": { |
| "pass@1": 0.6753771551724138, |
| "pass@2": 0.7510833713869619, |
| "pass@4": 0.8050979867720295, |
| "pass@8": 0.8438278075204682, |
| "pass@16": 0.8700994175560868, |
| "pass@32": 0.8896377354395928, |
| "pass@64": 0.9044934104614416, |
| "pass@128": 0.9184952978056427 |
| }, |
| "14": { |
| "pass@1": 0.5608450155763239, |
| "pass@2": 0.6548173154757525, |
| "pass@4": 0.72600939485368, |
| "pass@8": 0.7811643657505926, |
| "pass@16": 0.8254414231401552, |
| "pass@32": 0.8612872741153211, |
| "pass@64": 0.8888848758542358, |
| "pass@128": 0.9096573208722741 |
| }, |
| "15": { |
| "pass@1": 0.36824873554913296, |
| "pass@2": 0.44478659369168433, |
| "pass@4": 0.5140218047166066, |
| "pass@8": 0.5798621738402842, |
| "pass@16": 0.6431315726707407, |
| "pass@32": 0.70473802819444, |
| "pass@64": 0.7626283934751199, |
| "pass@128": 0.8121387283236994 |
| }, |
| "16": { |
| "pass@1": 0.26948084677419354, |
| "pass@2": 0.3359740125730251, |
| "pass@4": 0.39538789385197826, |
| "pass@8": 0.44739721096125223, |
| "pass@16": 0.4947938819205465, |
| "pass@32": 0.5413986035897612, |
| "pass@64": 0.589901840216927, |
| "pass@128": 0.6419354838709678 |
| }, |
| "17": { |
| "pass@1": 0.2056990606936416, |
| "pass@2": 0.26698261344499574, |
| "pass@4": 0.3209079303129015, |
| "pass@8": 0.36330255189787325, |
| "pass@16": 0.39650658081994944, |
| "pass@32": 0.4274359223938896, |
| "pass@64": 0.4597385934956624, |
| "pass@128": 0.49421965317919075 |
| }, |
| "18": { |
| "pass@1": 0.1859375, |
| "pass@2": 0.2411670416197975, |
| "pass@4": 0.2920078555359149, |
| "pass@8": 0.336872345277439, |
| "pass@16": 0.37773670092641387, |
| "pass@32": 0.4160940812427355, |
| "pass@64": 0.4542224751783429, |
| "pass@128": 0.4942857142857143 |
| }, |
| "19": { |
| "pass@1": 0.15986298798798798, |
| "pass@2": 0.21596313340899953, |
| "pass@4": 0.2690571783819814, |
| "pass@8": 0.31513812996336793, |
| "pass@16": 0.3564166595066925, |
| "pass@32": 0.3952428050444897, |
| "pass@64": 0.431117494519804, |
| "pass@128": 0.4624624624624625 |
| }, |
| "20": { |
| "pass@1": 0.14562218468468469, |
| "pass@2": 0.19661282837956062, |
| "pass@4": 0.2482558511830165, |
| "pass@8": 0.2946614543100083, |
| "pass@16": 0.3349464870280812, |
| "pass@32": 0.3705912105513184, |
| "pass@64": 0.40289861349419565, |
| "pass@128": 0.43243243243243246 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 569336, |
| "answer_accuracy": 0.6933651597817615, |
| "avg_response_len": 260.64148845479343, |
| "resp_tokens_sum": 214017939, |
| "pass_at_k": { |
| "pass@1": 0.6933651597817615, |
| "pass@2": 0.7261213798245936, |
| "pass@4": 0.7533903509187178, |
| "pass@8": 0.7769189019371558, |
| "pass@16": 0.797639132484816, |
| "pass@32": 0.8162401138379485, |
| "pass@64": 0.83315086872702, |
| "pass@128": 0.8487918939984411 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9239120015337423, |
| "pass@2": 0.939297165595865, |
| "pass@4": 0.9488506887175913, |
| "pass@8": 0.9569262745580872, |
| "pass@16": 0.9649962417353435, |
| "pass@32": 0.9725742390334997, |
| "pass@64": 0.9782689457947833, |
| "pass@128": 0.9815950920245399 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9996448863636364, |
| "pass@2": 0.9999772578143641, |
| "pass@4": 0.999999905977662, |
| "pass@8": 0.9999999999996503, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9936141304347826, |
| "pass@2": 0.9961870934611436, |
| "pass@4": 0.9969917790167533, |
| "pass@8": 0.9970989423536831, |
| "pass@16": 0.9971014483515, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9970451289398281, |
| "pass@2": 0.9975231538930127, |
| "pass@4": 0.9978671852085824, |
| "pass@8": 0.9984282614536057, |
| "pass@16": 0.9991741079188153, |
| "pass@32": 0.999805115576346, |
| "pass@64": 0.9999958603534026, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9950550939306358, |
| "pass@2": 0.9971880547084794, |
| "pass@4": 0.9982026819443812, |
| "pass@8": 0.9989650771290374, |
| "pass@16": 0.9996567447528346, |
| "pass@32": 0.999971393889872, |
| "pass@64": 0.9999999651170723, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9884903169014084, |
| "pass@2": 0.9931303371409559, |
| "pass@4": 0.9948249148433909, |
| "pass@8": 0.9955238896865715, |
| "pass@16": 0.996246251477471, |
| "pass@32": 0.9969218386791731, |
| "pass@64": 0.9971743778471076, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.9761648577844312, |
| "pass@2": 0.9835428797915982, |
| "pass@4": 0.9870129656322897, |
| "pass@8": 0.9897489148240651, |
| "pass@16": 0.9925134760010502, |
| "pass@32": 0.9950162280649698, |
| "pass@64": 0.9965948475649766, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9205336972891566, |
| "pass@2": 0.9446788136799164, |
| "pass@4": 0.9570576899837824, |
| "pass@8": 0.9651102596060525, |
| "pass@16": 0.9717073808081004, |
| "pass@32": 0.9769240261038603, |
| "pass@64": 0.9809700488424706, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.8275862068965517, |
| "pass@2": 0.8680843957495128, |
| "pass@4": 0.8935928778886965, |
| "pass@8": 0.911868949057682, |
| "pass@16": 0.9255625424430537, |
| "pass@32": 0.9362526361307683, |
| "pass@64": 0.9446319528623792, |
| "pass@128": 0.9529780564263323 |
| }, |
| "13": { |
| "pass@1": 0.6444984243697479, |
| "pass@2": 0.7218009743267383, |
| "pass@4": 0.7742140536004424, |
| "pass@8": 0.8128013561369666, |
| "pass@16": 0.842701157933433, |
| "pass@32": 0.8678694408648038, |
| "pass@64": 0.8891193128300473, |
| "pass@128": 0.907563025210084 |
| }, |
| "14": { |
| "pass@1": 0.5190972222222222, |
| "pass@2": 0.6002775043744532, |
| "pass@4": 0.6643069353172953, |
| "pass@8": 0.7169930462826867, |
| "pass@16": 0.760271193323265, |
| "pass@32": 0.79558462220192, |
| "pass@64": 0.8256190463116261, |
| "pass@128": 0.8508771929824561 |
| }, |
| "15": { |
| "pass@1": 0.40400094696969696, |
| "pass@2": 0.4864915145549986, |
| "pass@4": 0.5567694691572643, |
| "pass@8": 0.616903250109242, |
| "pass@16": 0.6703134121801195, |
| "pass@32": 0.7171740587227702, |
| "pass@64": 0.7571307632153466, |
| "pass@128": 0.7909090909090909 |
| }, |
| "16": { |
| "pass@1": 0.23709045584045585, |
| "pass@2": 0.3004520268299797, |
| "pass@4": 0.3595409155586322, |
| "pass@8": 0.41233947053774117, |
| "pass@16": 0.4605785314517335, |
| "pass@32": 0.5059843984632991, |
| "pass@64": 0.5504399720240419, |
| "pass@128": 0.5982905982905983 |
| }, |
| "17": { |
| "pass@1": 0.2220496894409938, |
| "pass@2": 0.2798913043478263, |
| "pass@4": 0.33797175624786024, |
| "pass@8": 0.39376723283276, |
| "pass@16": 0.4472076138388002, |
| "pass@32": 0.49932642743328454, |
| "pass@64": 0.5500364059705539, |
| "pass@128": 0.5993788819875776 |
| }, |
| "18": { |
| "pass@1": 0.1990579044117647, |
| "pass@2": 0.25289196387216306, |
| "pass@4": 0.30604848281832403, |
| "pass@8": 0.3597687550560062, |
| "pass@16": 0.41296564197100694, |
| "pass@32": 0.46505016543385114, |
| "pass@64": 0.5153088393629462, |
| "pass@128": 0.5588235294117647 |
| }, |
| "19": { |
| "pass@1": 0.16792224702380953, |
| "pass@2": 0.22124102163010886, |
| "pass@4": 0.270145626606942, |
| "pass@8": 0.3122431638570926, |
| "pass@16": 0.349570755288023, |
| "pass@32": 0.3844204803698806, |
| "pass@64": 0.41783088336058016, |
| "pass@128": 0.4494047619047619 |
| }, |
| "20": { |
| "pass@1": 0.14513940868263472, |
| "pass@2": 0.20053065620727054, |
| "pass@4": 0.25832609724308414, |
| "pass@8": 0.31100387615247266, |
| "pass@16": 0.3534627112803187, |
| "pass@32": 0.38832412730611254, |
| "pass@64": 0.4201394475573823, |
| "pass@128": 0.45209580838323354 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 560754, |
| "answer_accuracy": 0.6999345941843745, |
| "avg_response_len": 279.93733648546095, |
| "resp_tokens_sum": 224272357, |
| "pass_at_k": { |
| "pass@1": 0.6999345941843745, |
| "pass@2": 0.7353753311766443, |
| "pass@4": 0.7633834243968002, |
| "pass@8": 0.7864388365542485, |
| "pass@16": 0.8063853285794542, |
| "pass@32": 0.8243676328013746, |
| "pass@64": 0.8406141304402708, |
| "pass@128": 0.8546093625179741 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9412546641791045, |
| "pass@2": 0.9593184422376307, |
| "pass@4": 0.9695254223819035, |
| "pass@8": 0.9766898090195009, |
| "pass@16": 0.9816499535087969, |
| "pass@32": 0.9847547376865038, |
| "pass@64": 0.9864757263960945, |
| "pass@128": 0.9880597014925373 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999771564327485, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9809283088235294, |
| "pass@2": 0.9837786591014358, |
| "pass@4": 0.9866885057566334, |
| "pass@8": 0.9899423620573605, |
| "pass@16": 0.9924910881977745, |
| "pass@32": 0.9941425372788092, |
| "pass@64": 0.9955034383225714, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9934690457413249, |
| "pass@2": 0.9953612856752527, |
| "pass@4": 0.9968462346189376, |
| "pass@8": 0.9976937615953517, |
| "pass@16": 0.9984002614157852, |
| "pass@32": 0.9992711833727619, |
| "pass@64": 0.9999090886509849, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9799495648734177, |
| "pass@2": 0.9876832234874913, |
| "pass@4": 0.9915690026484031, |
| "pass@8": 0.9939399622711271, |
| "pass@16": 0.9955733611518887, |
| "pass@32": 0.9965338404526095, |
| "pass@64": 0.9968256447422759, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9836477102102102, |
| "pass@2": 0.9904164745454117, |
| "pass@4": 0.9938962590261803, |
| "pass@8": 0.9964578922525354, |
| "pass@16": 0.9978828245350282, |
| "pass@32": 0.9987422746727578, |
| "pass@64": 0.9996334917575699, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9787479903536977, |
| "pass@2": 0.9892171399093606, |
| "pass@4": 0.9927080022352512, |
| "pass@8": 0.9947520503111771, |
| "pass@16": 0.9964767790739536, |
| "pass@32": 0.9975337860587778, |
| "pass@64": 0.9983922273210069, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9122023809523809, |
| "pass@2": 0.9376054555680542, |
| "pass@4": 0.9521722906660476, |
| "pass@8": 0.9628302788052732, |
| "pass@16": 0.9714406202872945, |
| "pass@32": 0.9782479921028684, |
| "pass@64": 0.9837419618796911, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.8394017584097859, |
| "pass@2": 0.8898446717956132, |
| "pass@4": 0.9196687294822091, |
| "pass@8": 0.9406661843540817, |
| "pass@16": 0.957395070528658, |
| "pass@32": 0.9697229701815303, |
| "pass@64": 0.9767958090558799, |
| "pass@128": 0.9785932721712538 |
| }, |
| "13": { |
| "pass@1": 0.6813271604938271, |
| "pass@2": 0.7613477568776124, |
| "pass@4": 0.8106106571747972, |
| "pass@8": 0.8424284971241444, |
| "pass@16": 0.8682623680108061, |
| "pass@32": 0.8914488914032701, |
| "pass@64": 0.9091182354162962, |
| "pass@128": 0.9197530864197531 |
| }, |
| "14": { |
| "pass@1": 0.5415893916913946, |
| "pass@2": 0.6222867356713943, |
| "pass@4": 0.6819231268198295, |
| "pass@8": 0.7293694585273083, |
| "pass@16": 0.7686858613810834, |
| "pass@32": 0.8009758388561122, |
| "pass@64": 0.8262900290340521, |
| "pass@128": 0.8456973293768546 |
| }, |
| "15": { |
| "pass@1": 0.4177517361111111, |
| "pass@2": 0.49934914819675263, |
| "pass@4": 0.5659743541895225, |
| "pass@8": 0.6206460292345172, |
| "pass@16": 0.666048752660521, |
| "pass@32": 0.7066372816638773, |
| "pass@64": 0.7450684795278228, |
| "pass@128": 0.7777777777777778 |
| }, |
| "16": { |
| "pass@1": 0.2724695796460177, |
| "pass@2": 0.3365477579959583, |
| "pass@4": 0.39392186900310017, |
| "pass@8": 0.44472361753946393, |
| "pass@16": 0.4923935211994492, |
| "pass@32": 0.5386040986843411, |
| "pass@64": 0.5829563684430491, |
| "pass@128": 0.6224188790560472 |
| }, |
| "17": { |
| "pass@1": 0.2198324548192771, |
| "pass@2": 0.2865492481737974, |
| "pass@4": 0.34748548214229247, |
| "pass@8": 0.40131828327066915, |
| "pass@16": 0.44859846046522056, |
| "pass@32": 0.49064898077399194, |
| "pass@64": 0.5289156903363321, |
| "pass@128": 0.5632530120481928 |
| }, |
| "18": { |
| "pass@1": 0.2242943548387097, |
| "pass@2": 0.28490324168148323, |
| "pass@4": 0.33709839284202386, |
| "pass@8": 0.3813736430183623, |
| "pass@16": 0.42205391987613056, |
| "pass@32": 0.4612959820640024, |
| "pass@64": 0.4983398388195886, |
| "pass@128": 0.5290322580645161 |
| }, |
| "19": { |
| "pass@1": 0.16691842900302115, |
| "pass@2": 0.2274053215024858, |
| "pass@4": 0.2858991231647406, |
| "pass@8": 0.3380087741026875, |
| "pass@16": 0.3833170541660391, |
| "pass@32": 0.423878954401669, |
| "pass@64": 0.4625552903139795, |
| "pass@128": 0.5015105740181269 |
| }, |
| "20": { |
| "pass@1": 0.14998592342342343, |
| "pass@2": 0.20855427277671373, |
| "pass@4": 0.2678397956449137, |
| "pass@8": 0.3218562416665546, |
| "pass@16": 0.37166987634583953, |
| "pass@32": 0.4221730459872243, |
| "pass@64": 0.4732634611370853, |
| "pass@128": 0.5225225225225225 |
| } |
| } |
| } |
| } |
| } |
| ] |
