[ { "checkpoint": "checkpoint-21414", "id_acc": 0.9849149305555556, "ood_acc": 0.43210078125, "total_acc": 0.693960115131579, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.25730208333334, "ood_avg_resp_len": 323.8433125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 557621, "answer_accuracy": 0.6886522387764781, "avg_response_len": 251.28136361840026, "resp_tokens_sum": 203469556, "pass_at_k": { "pass@1": 0.6886522387764781, "pass@2": 0.7237809107395733, "pass@4": 0.7521360810474842, "pass@8": 0.7753498817050547, "pass@16": 0.7949650075503566, "pass@32": 0.8127696664662291, "pass@64": 0.8294527524537995, "pass@128": 0.8450837812203604 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9307015117994101, "pass@2": 0.9492464230134947, "pass@4": 0.9603176585780757, "pass@8": 0.9672270820964995, "pass@16": 0.9719429842512252, "pass@32": 0.9760390588911826, "pass@64": 0.9796358752047998, "pass@128": 0.9823008849557522 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999761085626911, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9964111328125, "pass@2": 0.9968580831692913, "pass@4": 0.9969715208450506, "pass@8": 0.9970703123347953, "pass@16": 0.997265625, "pass@32": 0.99765625, "pass@64": 0.9984375, "pass@128": 1.0 }, "5": { "pass@1": 0.9848958333333333, "pass@2": 0.9892681383577052, "pass@4": 0.9919947530368227, "pass@8": 0.9940923063815486, "pass@16": 0.9957625810438835, "pass@32": 0.9966789269921299, "pass@64": 0.9968243884338579, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9963276571856288, "pass@2": 0.9970273527747655, "pass@4": 0.9970994958901095, "pass@8": 0.9971931137724341, "pass@16": 0.9973802395209581, "pass@32": 0.9977544910179641, "pass@64": 0.9985029940119761, "pass@128": 1.0 }, "7": { "pass@1": 0.9932969674556213, "pass@2": 0.9977625011647955, "pass@4": 0.9993333954557454, "pass@8": 0.9998893882949348, "pass@16": 0.9999969360251098, "pass@32": 0.999999999201496, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9801933092948718, "pass@2": 0.987775322405613, "pass@4": 0.9924649139410457, "pass@8": 0.9956971600694451, "pass@16": 0.9977654601169499, "pass@32": 0.9989849709320879, "pass@64": 0.999809098401515, "pass@128": 1.0 }, "9": { "pass@1": 0.977068661971831, "pass@2": 0.9877346262615062, "pass@4": 0.99279322892737, "pass@8": 0.9960030095287996, "pass@16": 0.9979625327781841, "pass@32": 0.9991010683792878, "pass@64": 0.9998322132410451, "pass@128": 1.0 }, "11": { "pass@1": 0.8986366855524079, "pass@2": 0.9266491796970843, "pass@4": 0.9446660811343338, "pass@8": 0.9557997755282336, "pass@16": 0.9619171555267607, "pass@32": 0.9669011364421911, "pass@64": 0.9723664309673853, "pass@128": 0.9773371104815864 }, "12": { "pass@1": 0.7938073799435028, "pass@2": 0.8475351300102323, "pass@4": 0.8779464293658206, "pass@8": 0.898811393022897, "pass@16": 0.9162384786765374, "pass@32": 0.9324306603335383, "pass@64": 0.9477865031402798, "pass@128": 0.96045197740113 }, "13": { "pass@1": 0.6753771551724138, "pass@2": 0.7510833713869619, "pass@4": 0.8050979867720295, "pass@8": 0.8438278075204682, "pass@16": 0.8700994175560868, "pass@32": 0.8896377354395928, "pass@64": 0.9044934104614416, "pass@128": 0.9184952978056427 }, "14": { "pass@1": 0.5608450155763239, "pass@2": 0.6548173154757525, "pass@4": 0.72600939485368, "pass@8": 0.7811643657505926, "pass@16": 0.8254414231401552, "pass@32": 0.8612872741153211, "pass@64": 0.8888848758542358, "pass@128": 0.9096573208722741 }, "15": { "pass@1": 0.36824873554913296, "pass@2": 0.44478659369168433, "pass@4": 0.5140218047166066, "pass@8": 0.5798621738402842, "pass@16": 0.6431315726707407, "pass@32": 0.70473802819444, "pass@64": 0.7626283934751199, "pass@128": 0.8121387283236994 }, "16": { "pass@1": 0.26948084677419354, "pass@2": 0.3359740125730251, "pass@4": 0.39538789385197826, "pass@8": 0.44739721096125223, "pass@16": 0.4947938819205465, "pass@32": 0.5413986035897612, "pass@64": 0.589901840216927, "pass@128": 0.6419354838709678 }, "17": { "pass@1": 0.2056990606936416, "pass@2": 0.26698261344499574, "pass@4": 0.3209079303129015, "pass@8": 0.36330255189787325, "pass@16": 0.39650658081994944, "pass@32": 0.4274359223938896, "pass@64": 0.4597385934956624, "pass@128": 0.49421965317919075 }, "18": { "pass@1": 0.1859375, "pass@2": 0.2411670416197975, "pass@4": 0.2920078555359149, "pass@8": 0.336872345277439, "pass@16": 0.37773670092641387, "pass@32": 0.4160940812427355, "pass@64": 0.4542224751783429, "pass@128": 0.4942857142857143 }, "19": { "pass@1": 0.15986298798798798, "pass@2": 0.21596313340899953, "pass@4": 0.2690571783819814, "pass@8": 0.31513812996336793, "pass@16": 0.3564166595066925, "pass@32": 0.3952428050444897, "pass@64": 0.431117494519804, "pass@128": 0.4624624624624625 }, "20": { "pass@1": 0.14562218468468469, "pass@2": 0.19661282837956062, "pass@4": 0.2482558511830165, "pass@8": 0.2946614543100083, "pass@16": 0.3349464870280812, "pass@32": 0.3705912105513184, "pass@64": 0.40289861349419565, "pass@128": 0.43243243243243246 } } }, "teachers_in_school": { "count": 821120, "correct": 569336, "answer_accuracy": 0.6933651597817615, "avg_response_len": 260.64148845479343, "resp_tokens_sum": 214017939, "pass_at_k": { "pass@1": 0.6933651597817615, "pass@2": 0.7261213798245936, "pass@4": 0.7533903509187178, "pass@8": 0.7769189019371558, "pass@16": 0.797639132484816, "pass@32": 0.8162401138379485, "pass@64": 0.83315086872702, "pass@128": 0.8487918939984411 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9239120015337423, "pass@2": 0.939297165595865, "pass@4": 0.9488506887175913, "pass@8": 0.9569262745580872, "pass@16": 0.9649962417353435, "pass@32": 0.9725742390334997, "pass@64": 0.9782689457947833, "pass@128": 0.9815950920245399 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9996448863636364, "pass@2": 0.9999772578143641, "pass@4": 0.999999905977662, "pass@8": 0.9999999999996503, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9936141304347826, "pass@2": 0.9961870934611436, "pass@4": 0.9969917790167533, "pass@8": 0.9970989423536831, "pass@16": 0.9971014483515, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9970451289398281, "pass@2": 0.9975231538930127, "pass@4": 0.9978671852085824, "pass@8": 0.9984282614536057, "pass@16": 0.9991741079188153, "pass@32": 0.999805115576346, "pass@64": 0.9999958603534026, "pass@128": 1.0 }, "7": { "pass@1": 0.9950550939306358, "pass@2": 0.9971880547084794, "pass@4": 0.9982026819443812, "pass@8": 0.9989650771290374, "pass@16": 0.9996567447528346, "pass@32": 0.999971393889872, "pass@64": 0.9999999651170723, "pass@128": 1.0 }, "8": { "pass@1": 0.9884903169014084, "pass@2": 0.9931303371409559, "pass@4": 0.9948249148433909, "pass@8": 0.9955238896865715, "pass@16": 0.996246251477471, "pass@32": 0.9969218386791731, "pass@64": 0.9971743778471076, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.9761648577844312, "pass@2": 0.9835428797915982, "pass@4": 0.9870129656322897, "pass@8": 0.9897489148240651, "pass@16": 0.9925134760010502, "pass@32": 0.9950162280649698, "pass@64": 0.9965948475649766, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9205336972891566, "pass@2": 0.9446788136799164, "pass@4": 0.9570576899837824, "pass@8": 0.9651102596060525, "pass@16": 0.9717073808081004, "pass@32": 0.9769240261038603, "pass@64": 0.9809700488424706, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.8275862068965517, "pass@2": 0.8680843957495128, "pass@4": 0.8935928778886965, "pass@8": 0.911868949057682, "pass@16": 0.9255625424430537, "pass@32": 0.9362526361307683, "pass@64": 0.9446319528623792, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.6444984243697479, "pass@2": 0.7218009743267383, "pass@4": 0.7742140536004424, "pass@8": 0.8128013561369666, "pass@16": 0.842701157933433, "pass@32": 0.8678694408648038, "pass@64": 0.8891193128300473, "pass@128": 0.907563025210084 }, "14": { "pass@1": 0.5190972222222222, "pass@2": 0.6002775043744532, "pass@4": 0.6643069353172953, "pass@8": 0.7169930462826867, "pass@16": 0.760271193323265, "pass@32": 0.79558462220192, "pass@64": 0.8256190463116261, "pass@128": 0.8508771929824561 }, "15": { "pass@1": 0.40400094696969696, "pass@2": 0.4864915145549986, "pass@4": 0.5567694691572643, "pass@8": 0.616903250109242, "pass@16": 0.6703134121801195, "pass@32": 0.7171740587227702, "pass@64": 0.7571307632153466, "pass@128": 0.7909090909090909 }, "16": { "pass@1": 0.23709045584045585, "pass@2": 0.3004520268299797, "pass@4": 0.3595409155586322, "pass@8": 0.41233947053774117, "pass@16": 0.4605785314517335, "pass@32": 0.5059843984632991, "pass@64": 0.5504399720240419, "pass@128": 0.5982905982905983 }, "17": { "pass@1": 0.2220496894409938, "pass@2": 0.2798913043478263, "pass@4": 0.33797175624786024, "pass@8": 0.39376723283276, "pass@16": 0.4472076138388002, "pass@32": 0.49932642743328454, "pass@64": 0.5500364059705539, "pass@128": 0.5993788819875776 }, "18": { "pass@1": 0.1990579044117647, "pass@2": 0.25289196387216306, "pass@4": 0.30604848281832403, "pass@8": 0.3597687550560062, "pass@16": 0.41296564197100694, "pass@32": 0.46505016543385114, "pass@64": 0.5153088393629462, "pass@128": 0.5588235294117647 }, "19": { "pass@1": 0.16792224702380953, "pass@2": 0.22124102163010886, "pass@4": 0.270145626606942, "pass@8": 0.3122431638570926, "pass@16": 0.349570755288023, "pass@32": 0.3844204803698806, "pass@64": 0.41783088336058016, "pass@128": 0.4494047619047619 }, "20": { "pass@1": 0.14513940868263472, "pass@2": 0.20053065620727054, "pass@4": 0.25832609724308414, "pass@8": 0.31100387615247266, "pass@16": 0.3534627112803187, "pass@32": 0.38832412730611254, "pass@64": 0.4201394475573823, "pass@128": 0.45209580838323354 } } }, "movie_festival_awards": { "count": 801152, "correct": 560754, "answer_accuracy": 0.6999345941843745, "avg_response_len": 279.93733648546095, "resp_tokens_sum": 224272357, "pass_at_k": { "pass@1": 0.6999345941843745, "pass@2": 0.7353753311766443, "pass@4": 0.7633834243968002, "pass@8": 0.7864388365542485, "pass@16": 0.8063853285794542, "pass@32": 0.8243676328013746, "pass@64": 0.8406141304402708, "pass@128": 0.8546093625179741 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9412546641791045, "pass@2": 0.9593184422376307, "pass@4": 0.9695254223819035, "pass@8": 0.9766898090195009, "pass@16": 0.9816499535087969, "pass@32": 0.9847547376865038, "pass@64": 0.9864757263960945, "pass@128": 0.9880597014925373 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999771564327485, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9809283088235294, "pass@2": 0.9837786591014358, "pass@4": 0.9866885057566334, "pass@8": 0.9899423620573605, "pass@16": 0.9924910881977745, "pass@32": 0.9941425372788092, "pass@64": 0.9955034383225714, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9934690457413249, "pass@2": 0.9953612856752527, "pass@4": 0.9968462346189376, "pass@8": 0.9976937615953517, "pass@16": 0.9984002614157852, "pass@32": 0.9992711833727619, "pass@64": 0.9999090886509849, "pass@128": 1.0 }, "7": { "pass@1": 0.9799495648734177, "pass@2": 0.9876832234874913, "pass@4": 0.9915690026484031, "pass@8": 0.9939399622711271, "pass@16": 0.9955733611518887, "pass@32": 0.9965338404526095, "pass@64": 0.9968256447422759, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9836477102102102, "pass@2": 0.9904164745454117, "pass@4": 0.9938962590261803, "pass@8": 0.9964578922525354, "pass@16": 0.9978828245350282, "pass@32": 0.9987422746727578, "pass@64": 0.9996334917575699, "pass@128": 1.0 }, "9": { "pass@1": 0.9787479903536977, "pass@2": 0.9892171399093606, "pass@4": 0.9927080022352512, "pass@8": 0.9947520503111771, "pass@16": 0.9964767790739536, "pass@32": 0.9975337860587778, "pass@64": 0.9983922273210069, "pass@128": 1.0 }, "11": { "pass@1": 0.9122023809523809, "pass@2": 0.9376054555680542, "pass@4": 0.9521722906660476, "pass@8": 0.9628302788052732, "pass@16": 0.9714406202872945, "pass@32": 0.9782479921028684, "pass@64": 0.9837419618796911, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.8394017584097859, "pass@2": 0.8898446717956132, "pass@4": 0.9196687294822091, "pass@8": 0.9406661843540817, "pass@16": 0.957395070528658, "pass@32": 0.9697229701815303, "pass@64": 0.9767958090558799, "pass@128": 0.9785932721712538 }, "13": { "pass@1": 0.6813271604938271, "pass@2": 0.7613477568776124, "pass@4": 0.8106106571747972, "pass@8": 0.8424284971241444, "pass@16": 0.8682623680108061, "pass@32": 0.8914488914032701, "pass@64": 0.9091182354162962, "pass@128": 0.9197530864197531 }, "14": { "pass@1": 0.5415893916913946, "pass@2": 0.6222867356713943, "pass@4": 0.6819231268198295, "pass@8": 0.7293694585273083, "pass@16": 0.7686858613810834, "pass@32": 0.8009758388561122, "pass@64": 0.8262900290340521, "pass@128": 0.8456973293768546 }, "15": { "pass@1": 0.4177517361111111, "pass@2": 0.49934914819675263, "pass@4": 0.5659743541895225, "pass@8": 0.6206460292345172, "pass@16": 0.666048752660521, "pass@32": 0.7066372816638773, "pass@64": 0.7450684795278228, "pass@128": 0.7777777777777778 }, "16": { "pass@1": 0.2724695796460177, "pass@2": 0.3365477579959583, "pass@4": 0.39392186900310017, "pass@8": 0.44472361753946393, "pass@16": 0.4923935211994492, "pass@32": 0.5386040986843411, "pass@64": 0.5829563684430491, "pass@128": 0.6224188790560472 }, "17": { "pass@1": 0.2198324548192771, "pass@2": 0.2865492481737974, "pass@4": 0.34748548214229247, "pass@8": 0.40131828327066915, "pass@16": 0.44859846046522056, "pass@32": 0.49064898077399194, "pass@64": 0.5289156903363321, "pass@128": 0.5632530120481928 }, "18": { "pass@1": 0.2242943548387097, "pass@2": 0.28490324168148323, "pass@4": 0.33709839284202386, "pass@8": 0.3813736430183623, "pass@16": 0.42205391987613056, "pass@32": 0.4612959820640024, "pass@64": 0.4983398388195886, "pass@128": 0.5290322580645161 }, "19": { "pass@1": 0.16691842900302115, "pass@2": 0.2274053215024858, "pass@4": 0.2858991231647406, "pass@8": 0.3380087741026875, "pass@16": 0.3833170541660391, "pass@32": 0.423878954401669, "pass@64": 0.4625552903139795, "pass@128": 0.5015105740181269 }, "20": { "pass@1": 0.14998592342342343, "pass@2": 0.20855427277671373, "pass@4": 0.2678397956449137, "pass@8": 0.3218562416665546, "pass@16": 0.37166987634583953, "pass@32": 0.4221730459872243, "pass@64": 0.4732634611370853, "pass@128": 0.5225225225225225 } } } } } ]