[ { "checkpoint": "SDAR-100M-block4-epoch4-op7-10", "id_acc": 0.9950512152777777, "ood_acc": 0.5002296875, "total_acc": 0.7346188322368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.65094791666667, "ood_avg_resp_len": 344.50245, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 592869, "answer_accuracy": 0.7321829058646854, "avg_response_len": 264.0533796534145, "resp_tokens_sum": 213811415, "pass_at_k": { "pass@1": 0.7321829058646854, "pass@2": 0.771732271017493, "pass@4": 0.805390515893657, "pass@8": 0.8332625061636697, "pass@16": 0.8558427228397145, "pass@32": 0.8742772288713363, "pass@64": 0.8899804869612601, "pass@128": 0.9042048687954474 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9712389380530974, "pass@2": 0.9810781914152326, "pass@4": 0.9877872917987017, "pass@8": 0.9923114067410685, "pass@16": 0.9953009685125475, "pass@32": 0.9976251357688136, "pass@64": 0.9992232974200413, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999267578125, "pass@2": 0.9999988465797245, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9967261904761905, "pass@2": 0.9979080739907511, "pass@4": 0.998781230917564, "pass@8": 0.9995512519293239, "pass@16": 0.9999456111182645, "pass@32": 0.999999550305024, "pass@64": 0.9999999999992847, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9986362795857988, "pass@2": 0.9996065176117038, "pass@4": 0.9999584012797217, "pass@8": 0.9999995539350579, "pass@16": 0.9999999999795047, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9996494391025641, "pass@2": 0.9999948736876642, "pass@4": 0.9999999996995569, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9967649647887324, "pass@2": 0.997464164910724, "pass@4": 0.99782990673701, "pass@8": 0.9983444563318933, "pass@16": 0.9990631538364027, "pass@32": 0.9997387400876238, "pass@64": 0.9999912792555583, "pass@128": 1.0 }, "11": { "pass@1": 0.9256595254957507, "pass@2": 0.9379046446655216, "pass@4": 0.9464527073103111, "pass@8": 0.9544125837859465, "pass@16": 0.9620550938845545, "pass@32": 0.9689466009864247, "pass@64": 0.9733538467202176, "pass@128": 0.9745042492917847 }, "12": { "pass@1": 0.8546963276836158, "pass@2": 0.8892605320521376, "pass@4": 0.9096035241357541, "pass@8": 0.9232561725273154, "pass@16": 0.9335660780959657, "pass@32": 0.9430045267513955, "pass@64": 0.9524606602559231, "pass@128": 0.96045197740113 }, "13": { "pass@1": 0.7480407523510971, "pass@2": 0.8101477457853037, "pass@4": 0.8495423074074989, "pass@8": 0.8771021172653639, "pass@16": 0.896003390136595, "pass@32": 0.9090301477413725, "pass@64": 0.9191436054892381, "pass@128": 0.9278996865203761 }, "14": { "pass@1": 0.682583722741433, "pass@2": 0.7649933616650716, "pass@4": 0.818672544436618, "pass@8": 0.852279360455284, "pass@16": 0.8766875266221614, "pass@32": 0.8978875992415474, "pass@64": 0.9145590851330114, "pass@128": 0.9252336448598131 }, "15": { "pass@1": 0.5563583815028902, "pass@2": 0.6612466433025347, "pass@4": 0.7391046363062996, "pass@8": 0.7995924425216887, "pass@16": 0.8472066555948774, "pass@32": 0.8814056698555999, "pass@64": 0.9055910973040048, "pass@128": 0.9248554913294798 }, "16": { "pass@1": 0.35776209677419357, "pass@2": 0.4704236252222501, "pass@4": 0.5702631176143302, "pass@8": 0.6486250520290218, "pass@16": 0.7058972081786804, "pass@32": 0.7489029331029314, "pass@64": 0.7864609089054653, "pass@128": 0.8225806451612904 }, "17": { "pass@1": 0.2822209176300578, "pass@2": 0.38103423262482355, "pass@4": 0.4747528679579794, "pass@8": 0.5555152783820172, "pass@16": 0.6224005820967119, "pass@32": 0.6793292843272832, "pass@64": 0.727092303936004, "pass@128": 0.7658959537572254 }, "18": { "pass@1": 0.21580357142857143, "pass@2": 0.29927622328458936, "pass@4": 0.3840758931919225, "pass@8": 0.46175944378352485, "pass@16": 0.5272364483452201, "pass@32": 0.5804867351029614, "pass@64": 0.6281698996492903, "pass@128": 0.68 }, "19": { "pass@1": 0.18041478978978978, "pass@2": 0.25600452815019736, "pass@4": 0.3330379324769089, "pass@8": 0.40319579360003893, "pass@16": 0.46454362983819264, "pass@32": 0.5196802274780482, "pass@64": 0.5703849188444975, "pass@128": 0.6186186186186187 }, "20": { "pass@1": 0.17896021021021022, "pass@2": 0.24850367099382842, "pass@4": 0.32194917217555025, "pass@8": 0.3922391954982608, "pass@16": 0.45386218477737117, "pass@32": 0.5048449014642736, "pass@64": 0.55023639343001, "pass@128": 0.5945945945945946 } } }, "teachers_in_school": { "count": 821120, "correct": 605793, "answer_accuracy": 0.737764273187841, "avg_response_len": 273.1703271141855, "resp_tokens_sum": 224305619, "pass_at_k": { "pass@1": 0.737764273187841, "pass@2": 0.7765919489262912, "pass@4": 0.8097011007636077, "pass@8": 0.8373497793981471, "pass@16": 0.8596389213374069, "pass@32": 0.8775147067749307, "pass@64": 0.8927725359259874, "pass@128": 0.9070927513639907 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9639810199386503, "pass@2": 0.9763013411187863, "pass@4": 0.9849948809121252, "pass@8": 0.9900287848894771, "pass@16": 0.9927408931048474, "pass@32": 0.9949717791311338, "pass@64": 0.9965136850356024, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9995018115942029, "pass@2": 0.9999390191715166, "pass@4": 0.9999989468707715, "pass@8": 0.9999999998467666, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9993955945558739, "pass@2": 0.9998854291225775, "pass@4": 0.9999959845635341, "pass@8": 0.9999999968689766, "pass@16": 0.9999999999999999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9995484104046243, "pass@2": 0.9999324393518729, "pass@4": 0.9999986873931221, "pass@8": 0.999999999745349, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9954225352112676, "pass@2": 0.9976322501940778, "pass@4": 0.9990125788853857, "pass@8": 0.9997359054080507, "pass@16": 0.9999800133300545, "pass@32": 0.9999999433844207, "pass@64": 0.9999999999999964, "pass@128": 1.0 }, "9": { "pass@1": 0.9961171407185628, "pass@2": 0.9976218822198123, "pass@4": 0.9984793601772832, "pass@8": 0.9992627421174618, "pass@16": 0.999836737839751, "pass@32": 0.9999945907477397, "pass@64": 0.9999999995090815, "pass@128": 1.0 }, "11": { "pass@1": 0.9549839984939759, "pass@2": 0.9656675422161087, "pass@4": 0.9699165173065714, "pass@8": 0.9719722005758548, "pass@16": 0.9736488700375456, "pass@32": 0.9754268332917806, "pass@64": 0.9774205766085304, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.8691222570532915, "pass@2": 0.8991303717325303, "pass@4": 0.9174640042055867, "pass@8": 0.9318582991709181, "pass@16": 0.9426732941451575, "pass@32": 0.949884708573006, "pass@64": 0.9548453788380237, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.7502626050420168, "pass@2": 0.8138757471492536, "pass@4": 0.8540254450861707, "pass@8": 0.883486764099256, "pass@16": 0.9069851064183747, "pass@32": 0.9246218794102262, "pass@64": 0.9372843805980681, "pass@128": 0.9495798319327731 }, "14": { "pass@1": 0.6877741228070176, "pass@2": 0.7709750713726572, "pass@4": 0.8255886059746912, "pass@8": 0.8616636529389289, "pass@16": 0.8884648994522174, "pass@32": 0.9102203876391841, "pass@64": 0.9280497250391677, "pass@128": 0.9415204678362573 }, "15": { "pass@1": 0.5294981060606061, "pass@2": 0.6371901843235506, "pass@4": 0.7218647379304856, "pass@8": 0.784792093362903, "pass@16": 0.8296342586994554, "pass@32": 0.8615023805667635, "pass@64": 0.8850014181350153, "pass@128": 0.9030303030303031 }, "16": { "pass@1": 0.38826566951566954, "pass@2": 0.49237134620992856, "pass@4": 0.5828217968480429, "pass@8": 0.6537934355879218, "pass@16": 0.7056221038183779, "pass@32": 0.7460727057171335, "pass@64": 0.784026806543782, "pass@128": 0.8233618233618234 }, "17": { "pass@1": 0.3012179736024845, "pass@2": 0.40523403494400156, "pass@4": 0.5060908428760066, "pass@8": 0.5919590185783913, "pass@16": 0.6587760745115138, "pass@32": 0.7099266521318334, "pass@64": 0.7537711568802992, "pass@128": 0.7950310559006211 }, "18": { "pass@1": 0.24391084558823528, "pass@2": 0.32872459761463624, "pass@4": 0.4134082233287015, "pass@8": 0.4939817269103554, "pass@16": 0.5646029865362203, "pass@32": 0.6231760486863247, "pass@64": 0.672253518753805, "pass@128": 0.7176470588235294 }, "19": { "pass@1": 0.17578125, "pass@2": 0.24372283054461924, "pass@4": 0.3149261088457693, "pass@8": 0.382738789933009, "pass@16": 0.44227697220974477, "pass@32": 0.49650961347677647, "pass@64": 0.5534303542394388, "pass@128": 0.6190476190476191 }, "20": { "pass@1": 0.14549026946107785, "pass@2": 0.21075847812249524, "pass@4": 0.28126393697419555, "pass@8": 0.35182762438965187, "pass@16": 0.41691692179414386, "pass@32": 0.47061910952622776, "pass@64": 0.511055493082766, "pass@128": 0.5419161676646707 } } }, "movie_festival_awards": { "count": 801152, "correct": 587931, "answer_accuracy": 0.7338569959258667, "avg_response_len": 292.07440535628695, "resp_tokens_sum": 233995994, "pass_at_k": { "pass@1": 0.7338569959258667, "pass@2": 0.7727384574087262, "pass@4": 0.80609744740726, "pass@8": 0.8340243399432995, "pass@16": 0.8570450005642073, "pass@32": 0.8759931171931754, "pass@64": 0.8914421187831943, "pass@128": 0.9042978111519412 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9648320895522388, "pass@2": 0.9742709924785523, "pass@4": 0.9802755158403708, "pass@8": 0.9853434213761023, "pass@16": 0.9897842102909985, "pass@32": 0.9931238035577333, "pass@64": 0.9955219923489672, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9962775735294118, "pass@2": 0.9990218996062991, "pass@4": 0.9999060312313901, "pass@8": 0.9999988032093555, "pass@16": 0.9999999998445908, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9995563880126183, "pass@2": 0.9999472167714053, "pass@4": 0.9999992962236186, "pass@8": 0.999999999946361, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9998763844936709, "pass@2": 0.9999988319794678, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9926097972972973, "pass@2": 0.9953824395261401, "pass@4": 0.9979162804536821, "pass@8": 0.9994617011862063, "pass@16": 0.9999566861229163, "pass@32": 0.9999997975032378, "pass@64": 0.9999999999999107, "pass@128": 1.0 }, "9": { "pass@1": 0.9963826366559485, "pass@2": 0.9985176880522572, "pass@4": 0.9996576828619894, "pass@8": 0.9999795024220703, "pass@16": 0.9999999337376294, "pass@32": 0.9999999999998973, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9442212301587302, "pass@2": 0.9586360298712663, "pass@4": 0.9687240315198693, "pass@8": 0.9762697265593556, "pass@16": 0.9817871018932806, "pass@32": 0.9860954944679531, "pass@64": 0.9896482633443495, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.896621750764526, "pass@2": 0.9290360350598379, "pass@4": 0.9457635553583317, "pass@8": 0.956214278217107, "pass@16": 0.9637589402015467, "pass@32": 0.9705687233059196, "pass@64": 0.977748568972364, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.7021846064814815, "pass@2": 0.7656755036696803, "pass@4": 0.8098888909140981, "pass@8": 0.8446585152568515, "pass@16": 0.8737353120262339, "pass@32": 0.8965907782767284, "pass@64": 0.9122739330215466, "pass@128": 0.9259259259259259 }, "14": { "pass@1": 0.6611646884272997, "pass@2": 0.7445344079300921, "pass@4": 0.8004453199841117, "pass@8": 0.8367972897354505, "pass@16": 0.86221953114493, "pass@32": 0.882909239208098, "pass@64": 0.9007521220726056, "pass@128": 0.9169139465875371 }, "15": { "pass@1": 0.5411120756172839, "pass@2": 0.6362866208564207, "pass@4": 0.709073353678012, "pass@8": 0.7639026736821625, "pass@16": 0.80479268808345, "pass@32": 0.8357947268190055, "pass@64": 0.8608328062716483, "pass@128": 0.8827160493827161 }, "16": { "pass@1": 0.37479258849557523, "pass@2": 0.4824867024365314, "pass@4": 0.5765447806302972, "pass@8": 0.6533844530467929, "pass@16": 0.7155828352187087, "pass@32": 0.7672687388232577, "pass@64": 0.8085037275060418, "pass@128": 0.8407079646017699 }, "17": { "pass@1": 0.26618975903614456, "pass@2": 0.3588547101792997, "pass@4": 0.4463049080461327, "pass@8": 0.5217500365258368, "pass@16": 0.5857671915433449, "pass@32": 0.6408562235399593, "pass@64": 0.6900266613904555, "pass@128": 0.7379518072289156 }, "18": { "pass@1": 0.24914314516129032, "pass@2": 0.33067968948437876, "pass@4": 0.41047757689562997, "pass@8": 0.4840626512960474, "pass@16": 0.5475132634197127, "pass@32": 0.5988138059798621, "pass@64": 0.6409601499492135, "pass@128": 0.6774193548387096 }, "19": { "pass@1": 0.18079682779456194, "pass@2": 0.2624990335894569, "pass@4": 0.34947638899744776, "pass@8": 0.4290194751124504, "pass@16": 0.49722958065940137, "pass@32": 0.555049453644149, "pass@64": 0.6008047174065532, "pass@128": 0.6344410876132931 }, "20": { "pass@1": 0.1629129129129129, "pass@2": 0.2328166010498687, "pass@4": 0.309161864620526, "pass@8": 0.3846533878317151, "pass@16": 0.451918426732032, "pass@32": 0.5077871637320823, "pass@64": 0.5519450366312206, "pass@128": 0.5825825825825826 } } } } } ]