| [ |
| { |
| "checkpoint": "SDAR-100M-block4-epoch4-op7-10", |
| "id_acc": 0.9950512152777777, |
| "ood_acc": 0.5002296875, |
| "total_acc": 0.7346188322368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.65094791666667, |
| "ood_avg_resp_len": 344.50245, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 592869, |
| "answer_accuracy": 0.7321829058646854, |
| "avg_response_len": 264.0533796534145, |
| "resp_tokens_sum": 213811415, |
| "pass_at_k": { |
| "pass@1": 0.7321829058646854, |
| "pass@2": 0.771732271017493, |
| "pass@4": 0.805390515893657, |
| "pass@8": 0.8332625061636697, |
| "pass@16": 0.8558427228397145, |
| "pass@32": 0.8742772288713363, |
| "pass@64": 0.8899804869612601, |
| "pass@128": 0.9042048687954474 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9712389380530974, |
| "pass@2": 0.9810781914152326, |
| "pass@4": 0.9877872917987017, |
| "pass@8": 0.9923114067410685, |
| "pass@16": 0.9953009685125475, |
| "pass@32": 0.9976251357688136, |
| "pass@64": 0.9992232974200413, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999267578125, |
| "pass@2": 0.9999988465797245, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9967261904761905, |
| "pass@2": 0.9979080739907511, |
| "pass@4": 0.998781230917564, |
| "pass@8": 0.9995512519293239, |
| "pass@16": 0.9999456111182645, |
| "pass@32": 0.999999550305024, |
| "pass@64": 0.9999999999992847, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9986362795857988, |
| "pass@2": 0.9996065176117038, |
| "pass@4": 0.9999584012797217, |
| "pass@8": 0.9999995539350579, |
| "pass@16": 0.9999999999795047, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9996494391025641, |
| "pass@2": 0.9999948736876642, |
| "pass@4": 0.9999999996995569, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9967649647887324, |
| "pass@2": 0.997464164910724, |
| "pass@4": 0.99782990673701, |
| "pass@8": 0.9983444563318933, |
| "pass@16": 0.9990631538364027, |
| "pass@32": 0.9997387400876238, |
| "pass@64": 0.9999912792555583, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9256595254957507, |
| "pass@2": 0.9379046446655216, |
| "pass@4": 0.9464527073103111, |
| "pass@8": 0.9544125837859465, |
| "pass@16": 0.9620550938845545, |
| "pass@32": 0.9689466009864247, |
| "pass@64": 0.9733538467202176, |
| "pass@128": 0.9745042492917847 |
| }, |
| "12": { |
| "pass@1": 0.8546963276836158, |
| "pass@2": 0.8892605320521376, |
| "pass@4": 0.9096035241357541, |
| "pass@8": 0.9232561725273154, |
| "pass@16": 0.9335660780959657, |
| "pass@32": 0.9430045267513955, |
| "pass@64": 0.9524606602559231, |
| "pass@128": 0.96045197740113 |
| }, |
| "13": { |
| "pass@1": 0.7480407523510971, |
| "pass@2": 0.8101477457853037, |
| "pass@4": 0.8495423074074989, |
| "pass@8": 0.8771021172653639, |
| "pass@16": 0.896003390136595, |
| "pass@32": 0.9090301477413725, |
| "pass@64": 0.9191436054892381, |
| "pass@128": 0.9278996865203761 |
| }, |
| "14": { |
| "pass@1": 0.682583722741433, |
| "pass@2": 0.7649933616650716, |
| "pass@4": 0.818672544436618, |
| "pass@8": 0.852279360455284, |
| "pass@16": 0.8766875266221614, |
| "pass@32": 0.8978875992415474, |
| "pass@64": 0.9145590851330114, |
| "pass@128": 0.9252336448598131 |
| }, |
| "15": { |
| "pass@1": 0.5563583815028902, |
| "pass@2": 0.6612466433025347, |
| "pass@4": 0.7391046363062996, |
| "pass@8": 0.7995924425216887, |
| "pass@16": 0.8472066555948774, |
| "pass@32": 0.8814056698555999, |
| "pass@64": 0.9055910973040048, |
| "pass@128": 0.9248554913294798 |
| }, |
| "16": { |
| "pass@1": 0.35776209677419357, |
| "pass@2": 0.4704236252222501, |
| "pass@4": 0.5702631176143302, |
| "pass@8": 0.6486250520290218, |
| "pass@16": 0.7058972081786804, |
| "pass@32": 0.7489029331029314, |
| "pass@64": 0.7864609089054653, |
| "pass@128": 0.8225806451612904 |
| }, |
| "17": { |
| "pass@1": 0.2822209176300578, |
| "pass@2": 0.38103423262482355, |
| "pass@4": 0.4747528679579794, |
| "pass@8": 0.5555152783820172, |
| "pass@16": 0.6224005820967119, |
| "pass@32": 0.6793292843272832, |
| "pass@64": 0.727092303936004, |
| "pass@128": 0.7658959537572254 |
| }, |
| "18": { |
| "pass@1": 0.21580357142857143, |
| "pass@2": 0.29927622328458936, |
| "pass@4": 0.3840758931919225, |
| "pass@8": 0.46175944378352485, |
| "pass@16": 0.5272364483452201, |
| "pass@32": 0.5804867351029614, |
| "pass@64": 0.6281698996492903, |
| "pass@128": 0.68 |
| }, |
| "19": { |
| "pass@1": 0.18041478978978978, |
| "pass@2": 0.25600452815019736, |
| "pass@4": 0.3330379324769089, |
| "pass@8": 0.40319579360003893, |
| "pass@16": 0.46454362983819264, |
| "pass@32": 0.5196802274780482, |
| "pass@64": 0.5703849188444975, |
| "pass@128": 0.6186186186186187 |
| }, |
| "20": { |
| "pass@1": 0.17896021021021022, |
| "pass@2": 0.24850367099382842, |
| "pass@4": 0.32194917217555025, |
| "pass@8": 0.3922391954982608, |
| "pass@16": 0.45386218477737117, |
| "pass@32": 0.5048449014642736, |
| "pass@64": 0.55023639343001, |
| "pass@128": 0.5945945945945946 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 605793, |
| "answer_accuracy": 0.737764273187841, |
| "avg_response_len": 273.1703271141855, |
| "resp_tokens_sum": 224305619, |
| "pass_at_k": { |
| "pass@1": 0.737764273187841, |
| "pass@2": 0.7765919489262912, |
| "pass@4": 0.8097011007636077, |
| "pass@8": 0.8373497793981471, |
| "pass@16": 0.8596389213374069, |
| "pass@32": 0.8775147067749307, |
| "pass@64": 0.8927725359259874, |
| "pass@128": 0.9070927513639907 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9639810199386503, |
| "pass@2": 0.9763013411187863, |
| "pass@4": 0.9849948809121252, |
| "pass@8": 0.9900287848894771, |
| "pass@16": 0.9927408931048474, |
| "pass@32": 0.9949717791311338, |
| "pass@64": 0.9965136850356024, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9995018115942029, |
| "pass@2": 0.9999390191715166, |
| "pass@4": 0.9999989468707715, |
| "pass@8": 0.9999999998467666, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9993955945558739, |
| "pass@2": 0.9998854291225775, |
| "pass@4": 0.9999959845635341, |
| "pass@8": 0.9999999968689766, |
| "pass@16": 0.9999999999999999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9995484104046243, |
| "pass@2": 0.9999324393518729, |
| "pass@4": 0.9999986873931221, |
| "pass@8": 0.999999999745349, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9954225352112676, |
| "pass@2": 0.9976322501940778, |
| "pass@4": 0.9990125788853857, |
| "pass@8": 0.9997359054080507, |
| "pass@16": 0.9999800133300545, |
| "pass@32": 0.9999999433844207, |
| "pass@64": 0.9999999999999964, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9961171407185628, |
| "pass@2": 0.9976218822198123, |
| "pass@4": 0.9984793601772832, |
| "pass@8": 0.9992627421174618, |
| "pass@16": 0.999836737839751, |
| "pass@32": 0.9999945907477397, |
| "pass@64": 0.9999999995090815, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9549839984939759, |
| "pass@2": 0.9656675422161087, |
| "pass@4": 0.9699165173065714, |
| "pass@8": 0.9719722005758548, |
| "pass@16": 0.9736488700375456, |
| "pass@32": 0.9754268332917806, |
| "pass@64": 0.9774205766085304, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.8691222570532915, |
| "pass@2": 0.8991303717325303, |
| "pass@4": 0.9174640042055867, |
| "pass@8": 0.9318582991709181, |
| "pass@16": 0.9426732941451575, |
| "pass@32": 0.949884708573006, |
| "pass@64": 0.9548453788380237, |
| "pass@128": 0.9592476489028213 |
| }, |
| "13": { |
| "pass@1": 0.7502626050420168, |
| "pass@2": 0.8138757471492536, |
| "pass@4": 0.8540254450861707, |
| "pass@8": 0.883486764099256, |
| "pass@16": 0.9069851064183747, |
| "pass@32": 0.9246218794102262, |
| "pass@64": 0.9372843805980681, |
| "pass@128": 0.9495798319327731 |
| }, |
| "14": { |
| "pass@1": 0.6877741228070176, |
| "pass@2": 0.7709750713726572, |
| "pass@4": 0.8255886059746912, |
| "pass@8": 0.8616636529389289, |
| "pass@16": 0.8884648994522174, |
| "pass@32": 0.9102203876391841, |
| "pass@64": 0.9280497250391677, |
| "pass@128": 0.9415204678362573 |
| }, |
| "15": { |
| "pass@1": 0.5294981060606061, |
| "pass@2": 0.6371901843235506, |
| "pass@4": 0.7218647379304856, |
| "pass@8": 0.784792093362903, |
| "pass@16": 0.8296342586994554, |
| "pass@32": 0.8615023805667635, |
| "pass@64": 0.8850014181350153, |
| "pass@128": 0.9030303030303031 |
| }, |
| "16": { |
| "pass@1": 0.38826566951566954, |
| "pass@2": 0.49237134620992856, |
| "pass@4": 0.5828217968480429, |
| "pass@8": 0.6537934355879218, |
| "pass@16": 0.7056221038183779, |
| "pass@32": 0.7460727057171335, |
| "pass@64": 0.784026806543782, |
| "pass@128": 0.8233618233618234 |
| }, |
| "17": { |
| "pass@1": 0.3012179736024845, |
| "pass@2": 0.40523403494400156, |
| "pass@4": 0.5060908428760066, |
| "pass@8": 0.5919590185783913, |
| "pass@16": 0.6587760745115138, |
| "pass@32": 0.7099266521318334, |
| "pass@64": 0.7537711568802992, |
| "pass@128": 0.7950310559006211 |
| }, |
| "18": { |
| "pass@1": 0.24391084558823528, |
| "pass@2": 0.32872459761463624, |
| "pass@4": 0.4134082233287015, |
| "pass@8": 0.4939817269103554, |
| "pass@16": 0.5646029865362203, |
| "pass@32": 0.6231760486863247, |
| "pass@64": 0.672253518753805, |
| "pass@128": 0.7176470588235294 |
| }, |
| "19": { |
| "pass@1": 0.17578125, |
| "pass@2": 0.24372283054461924, |
| "pass@4": 0.3149261088457693, |
| "pass@8": 0.382738789933009, |
| "pass@16": 0.44227697220974477, |
| "pass@32": 0.49650961347677647, |
| "pass@64": 0.5534303542394388, |
| "pass@128": 0.6190476190476191 |
| }, |
| "20": { |
| "pass@1": 0.14549026946107785, |
| "pass@2": 0.21075847812249524, |
| "pass@4": 0.28126393697419555, |
| "pass@8": 0.35182762438965187, |
| "pass@16": 0.41691692179414386, |
| "pass@32": 0.47061910952622776, |
| "pass@64": 0.511055493082766, |
| "pass@128": 0.5419161676646707 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 587931, |
| "answer_accuracy": 0.7338569959258667, |
| "avg_response_len": 292.07440535628695, |
| "resp_tokens_sum": 233995994, |
| "pass_at_k": { |
| "pass@1": 0.7338569959258667, |
| "pass@2": 0.7727384574087262, |
| "pass@4": 0.80609744740726, |
| "pass@8": 0.8340243399432995, |
| "pass@16": 0.8570450005642073, |
| "pass@32": 0.8759931171931754, |
| "pass@64": 0.8914421187831943, |
| "pass@128": 0.9042978111519412 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9648320895522388, |
| "pass@2": 0.9742709924785523, |
| "pass@4": 0.9802755158403708, |
| "pass@8": 0.9853434213761023, |
| "pass@16": 0.9897842102909985, |
| "pass@32": 0.9931238035577333, |
| "pass@64": 0.9955219923489672, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9962775735294118, |
| "pass@2": 0.9990218996062991, |
| "pass@4": 0.9999060312313901, |
| "pass@8": 0.9999988032093555, |
| "pass@16": 0.9999999998445908, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9995563880126183, |
| "pass@2": 0.9999472167714053, |
| "pass@4": 0.9999992962236186, |
| "pass@8": 0.999999999946361, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9998763844936709, |
| "pass@2": 0.9999988319794678, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9926097972972973, |
| "pass@2": 0.9953824395261401, |
| "pass@4": 0.9979162804536821, |
| "pass@8": 0.9994617011862063, |
| "pass@16": 0.9999566861229163, |
| "pass@32": 0.9999997975032378, |
| "pass@64": 0.9999999999999107, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9963826366559485, |
| "pass@2": 0.9985176880522572, |
| "pass@4": 0.9996576828619894, |
| "pass@8": 0.9999795024220703, |
| "pass@16": 0.9999999337376294, |
| "pass@32": 0.9999999999998973, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9442212301587302, |
| "pass@2": 0.9586360298712663, |
| "pass@4": 0.9687240315198693, |
| "pass@8": 0.9762697265593556, |
| "pass@16": 0.9817871018932806, |
| "pass@32": 0.9860954944679531, |
| "pass@64": 0.9896482633443495, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.896621750764526, |
| "pass@2": 0.9290360350598379, |
| "pass@4": 0.9457635553583317, |
| "pass@8": 0.956214278217107, |
| "pass@16": 0.9637589402015467, |
| "pass@32": 0.9705687233059196, |
| "pass@64": 0.977748568972364, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.7021846064814815, |
| "pass@2": 0.7656755036696803, |
| "pass@4": 0.8098888909140981, |
| "pass@8": 0.8446585152568515, |
| "pass@16": 0.8737353120262339, |
| "pass@32": 0.8965907782767284, |
| "pass@64": 0.9122739330215466, |
| "pass@128": 0.9259259259259259 |
| }, |
| "14": { |
| "pass@1": 0.6611646884272997, |
| "pass@2": 0.7445344079300921, |
| "pass@4": 0.8004453199841117, |
| "pass@8": 0.8367972897354505, |
| "pass@16": 0.86221953114493, |
| "pass@32": 0.882909239208098, |
| "pass@64": 0.9007521220726056, |
| "pass@128": 0.9169139465875371 |
| }, |
| "15": { |
| "pass@1": 0.5411120756172839, |
| "pass@2": 0.6362866208564207, |
| "pass@4": 0.709073353678012, |
| "pass@8": 0.7639026736821625, |
| "pass@16": 0.80479268808345, |
| "pass@32": 0.8357947268190055, |
| "pass@64": 0.8608328062716483, |
| "pass@128": 0.8827160493827161 |
| }, |
| "16": { |
| "pass@1": 0.37479258849557523, |
| "pass@2": 0.4824867024365314, |
| "pass@4": 0.5765447806302972, |
| "pass@8": 0.6533844530467929, |
| "pass@16": 0.7155828352187087, |
| "pass@32": 0.7672687388232577, |
| "pass@64": 0.8085037275060418, |
| "pass@128": 0.8407079646017699 |
| }, |
| "17": { |
| "pass@1": 0.26618975903614456, |
| "pass@2": 0.3588547101792997, |
| "pass@4": 0.4463049080461327, |
| "pass@8": 0.5217500365258368, |
| "pass@16": 0.5857671915433449, |
| "pass@32": 0.6408562235399593, |
| "pass@64": 0.6900266613904555, |
| "pass@128": 0.7379518072289156 |
| }, |
| "18": { |
| "pass@1": 0.24914314516129032, |
| "pass@2": 0.33067968948437876, |
| "pass@4": 0.41047757689562997, |
| "pass@8": 0.4840626512960474, |
| "pass@16": 0.5475132634197127, |
| "pass@32": 0.5988138059798621, |
| "pass@64": 0.6409601499492135, |
| "pass@128": 0.6774193548387096 |
| }, |
| "19": { |
| "pass@1": 0.18079682779456194, |
| "pass@2": 0.2624990335894569, |
| "pass@4": 0.34947638899744776, |
| "pass@8": 0.4290194751124504, |
| "pass@16": 0.49722958065940137, |
| "pass@32": 0.555049453644149, |
| "pass@64": 0.6008047174065532, |
| "pass@128": 0.6344410876132931 |
| }, |
| "20": { |
| "pass@1": 0.1629129129129129, |
| "pass@2": 0.2328166010498687, |
| "pass@4": 0.309161864620526, |
| "pass@8": 0.3846533878317151, |
| "pass@16": 0.451918426732032, |
| "pass@32": 0.5077871637320823, |
| "pass@64": 0.5519450366312206, |
| "pass@128": 0.5825825825825826 |
| } |
| } |
| } |
| } |
| } |
| ] |
